1 /*
2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <net/tc_act/tc_gact.h>
34 #include <linux/mlx5/fs.h>
35 #include <net/vxlan.h>
36 #include <net/geneve.h>
37 #include <linux/bpf.h>
38 #include <linux/if_bridge.h>
39 #include <linux/filter.h>
40 #include <net/page_pool.h>
41 #include <net/xdp_sock_drv.h>
42 #include "eswitch.h"
43 #include "en.h"
44 #include "en/txrx.h"
45 #include "en_tc.h"
46 #include "en_rep.h"
47 #include "en_accel/ipsec.h"
48 #include "en_accel/macsec.h"
49 #include "en_accel/en_accel.h"
50 #include "en_accel/ktls.h"
51 #include "lib/vxlan.h"
52 #include "lib/clock.h"
53 #include "en/port.h"
54 #include "en/xdp.h"
55 #include "lib/eq.h"
56 #include "en/monitor_stats.h"
57 #include "en/health.h"
58 #include "en/params.h"
59 #include "en/xsk/pool.h"
60 #include "en/xsk/setup.h"
61 #include "en/xsk/rx.h"
62 #include "en/xsk/tx.h"
63 #include "en/hv_vhca_stats.h"
64 #include "en/devlink.h"
65 #include "lib/mlx5.h"
66 #include "en/ptp.h"
67 #include "en/htb.h"
68 #include "qos.h"
69 #include "en/trap.h"
70
mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)71 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
72 enum mlx5e_mpwrq_umr_mode umr_mode)
73 {
74 u16 umr_wqebbs, max_wqebbs;
75 bool striding_rq_umr;
76
77 striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
78 MLX5_CAP_ETH(mdev, reg_umr_sq);
79 if (!striding_rq_umr)
80 return false;
81
82 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
83 max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
84 /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is
85 * calculated from mlx5e_get_max_sq_aligned_wqebbs.
86 */
87 if (WARN_ON(umr_wqebbs > max_wqebbs))
88 return false;
89
90 return true;
91 }
92
mlx5e_update_carrier(struct mlx5e_priv * priv)93 void mlx5e_update_carrier(struct mlx5e_priv *priv)
94 {
95 struct mlx5_core_dev *mdev = priv->mdev;
96 u8 port_state;
97 bool up;
98
99 port_state = mlx5_query_vport_state(mdev,
100 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT,
101 0);
102
103 up = port_state == VPORT_STATE_UP;
104 if (up == netif_carrier_ok(priv->netdev))
105 netif_carrier_event(priv->netdev);
106 if (up) {
107 netdev_info(priv->netdev, "Link up\n");
108 netif_carrier_on(priv->netdev);
109 } else {
110 netdev_info(priv->netdev, "Link down\n");
111 netif_carrier_off(priv->netdev);
112 }
113 }
114
mlx5e_update_carrier_work(struct work_struct * work)115 static void mlx5e_update_carrier_work(struct work_struct *work)
116 {
117 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
118 update_carrier_work);
119
120 mutex_lock(&priv->state_lock);
121 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
122 if (priv->profile->update_carrier)
123 priv->profile->update_carrier(priv);
124 mutex_unlock(&priv->state_lock);
125 }
126
mlx5e_update_stats_work(struct work_struct * work)127 static void mlx5e_update_stats_work(struct work_struct *work)
128 {
129 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
130 update_stats_work);
131
132 mutex_lock(&priv->state_lock);
133 priv->profile->update_stats(priv);
134 mutex_unlock(&priv->state_lock);
135 }
136
mlx5e_queue_update_stats(struct mlx5e_priv * priv)137 void mlx5e_queue_update_stats(struct mlx5e_priv *priv)
138 {
139 if (!priv->profile->update_stats)
140 return;
141
142 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state)))
143 return;
144
145 queue_work(priv->wq, &priv->update_stats_work);
146 }
147
async_event(struct notifier_block * nb,unsigned long event,void * data)148 static int async_event(struct notifier_block *nb, unsigned long event, void *data)
149 {
150 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
151 struct mlx5_eqe *eqe = data;
152
153 if (event != MLX5_EVENT_TYPE_PORT_CHANGE)
154 return NOTIFY_DONE;
155
156 switch (eqe->sub_type) {
157 case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
158 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
159 queue_work(priv->wq, &priv->update_carrier_work);
160 break;
161 default:
162 return NOTIFY_DONE;
163 }
164
165 return NOTIFY_OK;
166 }
167
mlx5e_enable_async_events(struct mlx5e_priv * priv)168 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
169 {
170 priv->events_nb.notifier_call = async_event;
171 mlx5_notifier_register(priv->mdev, &priv->events_nb);
172 }
173
mlx5e_disable_async_events(struct mlx5e_priv * priv)174 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
175 {
176 mlx5_notifier_unregister(priv->mdev, &priv->events_nb);
177 }
178
blocking_event(struct notifier_block * nb,unsigned long event,void * data)179 static int blocking_event(struct notifier_block *nb, unsigned long event, void *data)
180 {
181 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb);
182 int err;
183
184 switch (event) {
185 case MLX5_DRIVER_EVENT_TYPE_TRAP:
186 err = mlx5e_handle_trap_event(priv, data);
187 break;
188 default:
189 netdev_warn(priv->netdev, "Sync event: Unknown event %ld\n", event);
190 err = -EINVAL;
191 }
192 return err;
193 }
194
mlx5e_enable_blocking_events(struct mlx5e_priv * priv)195 static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv)
196 {
197 priv->blocking_events_nb.notifier_call = blocking_event;
198 mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb);
199 }
200
mlx5e_disable_blocking_events(struct mlx5e_priv * priv)201 static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
202 {
203 mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb);
204 }
205
mlx5e_mpwrq_umr_octowords(u32 entries,enum mlx5e_mpwrq_umr_mode umr_mode)206 static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode)
207 {
208 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
209 u32 sz;
210
211 sz = ALIGN(entries * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT);
212
213 return sz / MLX5_OCTWORD;
214 }
215
mlx5e_build_umr_wqe(struct mlx5e_rq * rq,struct mlx5e_icosq * sq,struct mlx5e_umr_wqe * wqe)216 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
217 struct mlx5e_icosq *sq,
218 struct mlx5e_umr_wqe *wqe)
219 {
220 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
221 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
222 u16 octowords;
223 u8 ds_cnt;
224
225 ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift,
226 rq->mpwqe.umr_mode),
227 MLX5_SEND_WQE_DS);
228
229 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
230 ds_cnt);
231 cseg->umr_mkey = rq->mpwqe.umr_mkey_be;
232
233 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
234 octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode);
235 ucseg->xlt_octowords = cpu_to_be16(octowords);
236 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
237 }
238
mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq * rq,int node)239 static int mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq *rq, int node)
240 {
241 rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo),
242 GFP_KERNEL, node);
243 if (!rq->mpwqe.shampo)
244 return -ENOMEM;
245 return 0;
246 }
247
mlx5e_rq_shampo_hd_free(struct mlx5e_rq * rq)248 static void mlx5e_rq_shampo_hd_free(struct mlx5e_rq *rq)
249 {
250 kvfree(rq->mpwqe.shampo);
251 }
252
mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq * rq,int node)253 static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node)
254 {
255 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
256
257 shampo->bitmap = bitmap_zalloc_node(shampo->hd_per_wq, GFP_KERNEL,
258 node);
259 if (!shampo->bitmap)
260 return -ENOMEM;
261
262 shampo->info = kvzalloc_node(array_size(shampo->hd_per_wq,
263 sizeof(*shampo->info)),
264 GFP_KERNEL, node);
265 if (!shampo->info) {
266 kvfree(shampo->bitmap);
267 return -ENOMEM;
268 }
269 return 0;
270 }
271
mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq * rq)272 static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
273 {
274 kvfree(rq->mpwqe.shampo->bitmap);
275 kvfree(rq->mpwqe.shampo->info);
276 }
277
mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq * rq,int node)278 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
279 {
280 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
281 size_t alloc_size;
282
283 alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, alloc_units,
284 rq->mpwqe.pages_per_wqe));
285
286 rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node);
287 if (!rq->mpwqe.info)
288 return -ENOMEM;
289
290 mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe);
291
292 return 0;
293 }
294
295
mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode)296 static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode)
297 {
298 switch (umr_mode) {
299 case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
300 return MLX5_MKC_ACCESS_MODE_MTT;
301 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
302 return MLX5_MKC_ACCESS_MODE_KSM;
303 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
304 return MLX5_MKC_ACCESS_MODE_KLMS;
305 case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
306 return MLX5_MKC_ACCESS_MODE_KSM;
307 }
308 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);
309 return 0;
310 }
311
mlx5e_create_umr_mkey(struct mlx5_core_dev * mdev,u32 npages,u8 page_shift,u32 * umr_mkey,dma_addr_t filler_addr,enum mlx5e_mpwrq_umr_mode umr_mode,u32 xsk_chunk_size)312 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
313 u32 npages, u8 page_shift, u32 *umr_mkey,
314 dma_addr_t filler_addr,
315 enum mlx5e_mpwrq_umr_mode umr_mode,
316 u32 xsk_chunk_size)
317 {
318 struct mlx5_mtt *mtt;
319 struct mlx5_ksm *ksm;
320 struct mlx5_klm *klm;
321 u32 octwords;
322 int inlen;
323 void *mkc;
324 u32 *in;
325 int err;
326 int i;
327
328 if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED ||
329 umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) &&
330 !MLX5_CAP_GEN(mdev, fixed_buffer_size)) {
331 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n");
332 return -EINVAL;
333 }
334
335 octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode);
336
337 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in),
338 MLX5_OCTWORD, octwords);
339 if (inlen < 0)
340 return inlen;
341
342 in = kvzalloc(inlen, GFP_KERNEL);
343 if (!in)
344 return -ENOMEM;
345
346 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
347
348 MLX5_SET(mkc, mkc, free, 1);
349 MLX5_SET(mkc, mkc, umr_en, 1);
350 MLX5_SET(mkc, mkc, lw, 1);
351 MLX5_SET(mkc, mkc, lr, 1);
352 MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode));
353 mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
354 MLX5_SET(mkc, mkc, qpn, 0xffffff);
355 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
356 MLX5_SET64(mkc, mkc, len, npages << page_shift);
357 MLX5_SET(mkc, mkc, translations_octword_size, octwords);
358 if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)
359 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2);
360 else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED)
361 MLX5_SET(mkc, mkc, log_page_size, page_shift);
362 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords);
363
364 /* Initialize the mkey with all MTTs pointing to a default
365 * page (filler_addr). When the channels are activated, UMR
366 * WQEs will redirect the RX WQEs to the actual memory from
367 * the RQ's pool, while the gaps (wqe_overflow) remain mapped
368 * to the default page.
369 */
370 switch (umr_mode) {
371 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
372 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
373 for (i = 0; i < npages; i++) {
374 klm[i << 1] = (struct mlx5_klm) {
375 .va = cpu_to_be64(filler_addr),
376 .bcount = cpu_to_be32(xsk_chunk_size),
377 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
378 };
379 klm[(i << 1) + 1] = (struct mlx5_klm) {
380 .va = cpu_to_be64(filler_addr),
381 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size),
382 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
383 };
384 }
385 break;
386 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
387 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
388 for (i = 0; i < npages; i++)
389 ksm[i] = (struct mlx5_ksm) {
390 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
391 .va = cpu_to_be64(filler_addr),
392 };
393 break;
394 case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
395 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
396 for (i = 0; i < npages; i++)
397 mtt[i] = (struct mlx5_mtt) {
398 .ptag = cpu_to_be64(filler_addr),
399 };
400 break;
401 case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
402 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
403 for (i = 0; i < npages * 4; i++) {
404 ksm[i] = (struct mlx5_ksm) {
405 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
406 .va = cpu_to_be64(filler_addr),
407 };
408 }
409 break;
410 }
411
412 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
413
414 kvfree(in);
415 return err;
416 }
417
mlx5e_create_umr_klm_mkey(struct mlx5_core_dev * mdev,u64 nentries,u32 * umr_mkey)418 static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev,
419 u64 nentries,
420 u32 *umr_mkey)
421 {
422 int inlen;
423 void *mkc;
424 u32 *in;
425 int err;
426
427 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
428
429 in = kvzalloc(inlen, GFP_KERNEL);
430 if (!in)
431 return -ENOMEM;
432
433 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
434
435 MLX5_SET(mkc, mkc, free, 1);
436 MLX5_SET(mkc, mkc, umr_en, 1);
437 MLX5_SET(mkc, mkc, lw, 1);
438 MLX5_SET(mkc, mkc, lr, 1);
439 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
440 mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
441 MLX5_SET(mkc, mkc, qpn, 0xffffff);
442 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
443 MLX5_SET(mkc, mkc, translations_octword_size, nentries);
444 MLX5_SET(mkc, mkc, length64, 1);
445 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
446
447 kvfree(in);
448 return err;
449 }
450
mlx5e_create_rq_umr_mkey(struct mlx5_core_dev * mdev,struct mlx5e_rq * rq)451 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
452 {
453 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0;
454 u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
455 u32 num_entries, max_num_entries;
456 u32 umr_mkey;
457 int err;
458
459 max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode);
460
461 /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */
462 if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe,
463 &num_entries) ||
464 num_entries > max_num_entries))
465 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n",
466 __func__, wq_size, rq->mpwqe.mtts_per_wqe,
467 max_num_entries);
468
469 err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift,
470 &umr_mkey, rq->wqe_overflow.addr,
471 rq->mpwqe.umr_mode, xsk_chunk_size);
472 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey);
473 return err;
474 }
475
mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev * mdev,struct mlx5e_rq * rq)476 static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev,
477 struct mlx5e_rq *rq)
478 {
479 u32 max_klm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size));
480
481 if (max_klm_size < rq->mpwqe.shampo->hd_per_wq) {
482 mlx5_core_err(mdev, "max klm list size 0x%x is smaller than shampo header buffer list size 0x%x\n",
483 max_klm_size, rq->mpwqe.shampo->hd_per_wq);
484 return -EINVAL;
485 }
486 return mlx5e_create_umr_klm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq,
487 &rq->mpwqe.shampo->mkey);
488 }
489
mlx5e_init_frags_partition(struct mlx5e_rq * rq)490 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
491 {
492 struct mlx5e_wqe_frag_info next_frag = {};
493 struct mlx5e_wqe_frag_info *prev = NULL;
494 int i;
495
496 if (rq->xsk_pool) {
497 /* Assumptions used by XSK batched allocator. */
498 WARN_ON(rq->wqe.info.num_frags != 1);
499 WARN_ON(rq->wqe.info.log_num_frags != 0);
500 WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE);
501 }
502
503 next_frag.au = &rq->wqe.alloc_units[0];
504
505 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
506 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
507 struct mlx5e_wqe_frag_info *frag =
508 &rq->wqe.frags[i << rq->wqe.info.log_num_frags];
509 int f;
510
511 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
512 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
513 next_frag.au++;
514 next_frag.offset = 0;
515 if (prev)
516 prev->last_in_page = true;
517 }
518 *frag = next_frag;
519
520 /* prepare next */
521 next_frag.offset += frag_info[f].frag_stride;
522 prev = frag;
523 }
524 }
525
526 if (prev)
527 prev->last_in_page = true;
528 }
529
mlx5e_init_au_list(struct mlx5e_rq * rq,int wq_sz,int node)530 static int mlx5e_init_au_list(struct mlx5e_rq *rq, int wq_sz, int node)
531 {
532 int len = wq_sz << rq->wqe.info.log_num_frags;
533
534 rq->wqe.alloc_units = kvzalloc_node(array_size(len, sizeof(*rq->wqe.alloc_units)),
535 GFP_KERNEL, node);
536 if (!rq->wqe.alloc_units)
537 return -ENOMEM;
538
539 mlx5e_init_frags_partition(rq);
540
541 return 0;
542 }
543
mlx5e_free_au_list(struct mlx5e_rq * rq)544 static void mlx5e_free_au_list(struct mlx5e_rq *rq)
545 {
546 kvfree(rq->wqe.alloc_units);
547 }
548
mlx5e_rq_err_cqe_work(struct work_struct * recover_work)549 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
550 {
551 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work);
552
553 mlx5e_reporter_rq_cqe_err(rq);
554 }
555
mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq * rq)556 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
557 {
558 rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
559 if (!rq->wqe_overflow.page)
560 return -ENOMEM;
561
562 rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
563 PAGE_SIZE, rq->buff.map_dir);
564 if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
565 __free_page(rq->wqe_overflow.page);
566 return -ENOMEM;
567 }
568 return 0;
569 }
570
mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq * rq)571 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
572 {
573 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
574 rq->buff.map_dir);
575 __free_page(rq->wqe_overflow.page);
576 }
577
mlx5e_init_rxq_rq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_rq * rq)578 static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
579 struct mlx5e_rq *rq)
580 {
581 struct mlx5_core_dev *mdev = c->mdev;
582 int err;
583
584 rq->wq_type = params->rq_wq_type;
585 rq->pdev = c->pdev;
586 rq->netdev = c->netdev;
587 rq->priv = c->priv;
588 rq->tstamp = c->tstamp;
589 rq->clock = &mdev->clock;
590 rq->icosq = &c->icosq;
591 rq->ix = c->ix;
592 rq->channel = c;
593 rq->mdev = mdev;
594 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
595 rq->xdpsq = &c->rq_xdpsq;
596 rq->stats = &c->priv->channel_stats[c->ix]->rq;
597 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
598 err = mlx5e_rq_set_handlers(rq, params, NULL);
599 if (err)
600 return err;
601
602 return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id);
603 }
604
mlx5_rq_shampo_alloc(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_rq_param * rqp,struct mlx5e_rq * rq,u32 * pool_size,int node)605 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
606 struct mlx5e_params *params,
607 struct mlx5e_rq_param *rqp,
608 struct mlx5e_rq *rq,
609 u32 *pool_size,
610 int node)
611 {
612 void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq);
613 int wq_size;
614 int err;
615
616 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
617 return 0;
618 err = mlx5e_rq_shampo_hd_alloc(rq, node);
619 if (err)
620 goto out;
621 rq->mpwqe.shampo->hd_per_wq =
622 mlx5e_shampo_hd_per_wq(mdev, params, rqp);
623 err = mlx5e_create_rq_hd_umr_mkey(mdev, rq);
624 if (err)
625 goto err_shampo_hd;
626 err = mlx5e_rq_shampo_hd_info_alloc(rq, node);
627 if (err)
628 goto err_shampo_info;
629 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node);
630 if (!rq->hw_gro_data) {
631 err = -ENOMEM;
632 goto err_hw_gro_data;
633 }
634 rq->mpwqe.shampo->key =
635 cpu_to_be32(rq->mpwqe.shampo->mkey);
636 rq->mpwqe.shampo->hd_per_wqe =
637 mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
638 wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
639 *pool_size += (rq->mpwqe.shampo->hd_per_wqe * wq_size) /
640 MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;
641 return 0;
642
643 err_hw_gro_data:
644 mlx5e_rq_shampo_hd_info_free(rq);
645 err_shampo_info:
646 mlx5_core_destroy_mkey(mdev, rq->mpwqe.shampo->mkey);
647 err_shampo_hd:
648 mlx5e_rq_shampo_hd_free(rq);
649 out:
650 return err;
651 }
652
mlx5e_rq_free_shampo(struct mlx5e_rq * rq)653 static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
654 {
655 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
656 return;
657
658 kvfree(rq->hw_gro_data);
659 mlx5e_rq_shampo_hd_info_free(rq);
660 mlx5_core_destroy_mkey(rq->mdev, rq->mpwqe.shampo->mkey);
661 mlx5e_rq_shampo_hd_free(rq);
662 }
663
mlx5e_alloc_rq(struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,struct mlx5e_rq_param * rqp,int node,struct mlx5e_rq * rq)664 static int mlx5e_alloc_rq(struct mlx5e_params *params,
665 struct mlx5e_xsk_param *xsk,
666 struct mlx5e_rq_param *rqp,
667 int node, struct mlx5e_rq *rq)
668 {
669 struct page_pool_params pp_params = { 0 };
670 struct mlx5_core_dev *mdev = rq->mdev;
671 void *rqc = rqp->rqc;
672 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
673 u32 pool_size;
674 int wq_sz;
675 int err;
676 int i;
677
678 rqp->wq.db_numa_node = node;
679 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
680
681 if (params->xdp_prog)
682 bpf_prog_inc(params->xdp_prog);
683 RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
684
685 rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
686 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
687 pool_size = 1 << params->log_rq_mtu_frames;
688
689 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
690
691 switch (rq->wq_type) {
692 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
693 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
694 &rq->wq_ctrl);
695 if (err)
696 goto err_rq_xdp_prog;
697
698 err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
699 if (err)
700 goto err_rq_wq_destroy;
701
702 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
703
704 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
705
706 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
707 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
708 rq->mpwqe.pages_per_wqe =
709 mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
710 rq->mpwqe.umr_mode);
711 rq->mpwqe.umr_wqebbs =
712 mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift,
713 rq->mpwqe.umr_mode);
714 rq->mpwqe.mtts_per_wqe =
715 mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift,
716 rq->mpwqe.umr_mode);
717
718 pool_size = rq->mpwqe.pages_per_wqe <<
719 mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
720
721 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
722 rq->mpwqe.num_strides =
723 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
724 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz);
725
726 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
727
728 err = mlx5e_create_rq_umr_mkey(mdev, rq);
729 if (err)
730 goto err_rq_drop_page;
731
732 err = mlx5e_rq_alloc_mpwqe_info(rq, node);
733 if (err)
734 goto err_rq_mkey;
735
736 err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node);
737 if (err)
738 goto err_free_mpwqe_info;
739
740 break;
741 default: /* MLX5_WQ_TYPE_CYCLIC */
742 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
743 &rq->wq_ctrl);
744 if (err)
745 goto err_rq_xdp_prog;
746
747 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
748
749 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
750
751 rq->wqe.info = rqp->frags_info;
752 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
753
754 rq->wqe.frags =
755 kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
756 (wq_sz << rq->wqe.info.log_num_frags)),
757 GFP_KERNEL, node);
758 if (!rq->wqe.frags) {
759 err = -ENOMEM;
760 goto err_rq_wq_destroy;
761 }
762
763 err = mlx5e_init_au_list(rq, wq_sz, node);
764 if (err)
765 goto err_rq_frags;
766 }
767
768 if (xsk) {
769 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
770 MEM_TYPE_XSK_BUFF_POOL, NULL);
771 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq);
772 } else {
773 /* Create a page_pool and register it with rxq */
774 pp_params.order = 0;
775 pp_params.flags = 0; /* No-internal DMA mapping in page_pool */
776 pp_params.pool_size = pool_size;
777 pp_params.nid = node;
778 pp_params.dev = rq->pdev;
779 pp_params.dma_dir = rq->buff.map_dir;
780
781 /* page_pool can be used even when there is no rq->xdp_prog,
782 * given page_pool does not handle DMA mapping there is no
783 * required state to clear. And page_pool gracefully handle
784 * elevated refcnt.
785 */
786 rq->page_pool = page_pool_create(&pp_params);
787 if (IS_ERR(rq->page_pool)) {
788 err = PTR_ERR(rq->page_pool);
789 rq->page_pool = NULL;
790 goto err_free_by_rq_type;
791 }
792 if (xdp_rxq_info_is_reg(&rq->xdp_rxq))
793 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
794 MEM_TYPE_PAGE_POOL, rq->page_pool);
795 }
796 if (err)
797 goto err_destroy_page_pool;
798
799 for (i = 0; i < wq_sz; i++) {
800 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
801 struct mlx5e_rx_wqe_ll *wqe =
802 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
803 u32 byte_count =
804 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
805 u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) <<
806 rq->mpwqe.page_shift;
807 u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ?
808 0 : rq->buff.headroom;
809
810 wqe->data[0].addr = cpu_to_be64(dma_offset + headroom);
811 wqe->data[0].byte_count = cpu_to_be32(byte_count);
812 wqe->data[0].lkey = rq->mpwqe.umr_mkey_be;
813 } else {
814 struct mlx5e_rx_wqe_cyc *wqe =
815 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
816 int f;
817
818 for (f = 0; f < rq->wqe.info.num_frags; f++) {
819 u32 frag_size = rq->wqe.info.arr[f].frag_size |
820 MLX5_HW_START_PADDING;
821
822 wqe->data[f].byte_count = cpu_to_be32(frag_size);
823 wqe->data[f].lkey = rq->mkey_be;
824 }
825 /* check if num_frags is not a pow of two */
826 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
827 wqe->data[f].byte_count = 0;
828 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
829 wqe->data[f].addr = 0;
830 }
831 }
832 }
833
834 INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
835
836 switch (params->rx_cq_moderation.cq_period_mode) {
837 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
838 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
839 break;
840 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
841 default:
842 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
843 }
844
845 rq->page_cache.head = 0;
846 rq->page_cache.tail = 0;
847
848 return 0;
849
850 err_destroy_page_pool:
851 page_pool_destroy(rq->page_pool);
852 err_free_by_rq_type:
853 switch (rq->wq_type) {
854 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
855 mlx5e_rq_free_shampo(rq);
856 err_free_mpwqe_info:
857 kvfree(rq->mpwqe.info);
858 err_rq_mkey:
859 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
860 err_rq_drop_page:
861 mlx5e_free_mpwqe_rq_drop_page(rq);
862 break;
863 default: /* MLX5_WQ_TYPE_CYCLIC */
864 mlx5e_free_au_list(rq);
865 err_rq_frags:
866 kvfree(rq->wqe.frags);
867 }
868 err_rq_wq_destroy:
869 mlx5_wq_destroy(&rq->wq_ctrl);
870 err_rq_xdp_prog:
871 if (params->xdp_prog)
872 bpf_prog_put(params->xdp_prog);
873
874 return err;
875 }
876
mlx5e_free_rq(struct mlx5e_rq * rq)877 static void mlx5e_free_rq(struct mlx5e_rq *rq)
878 {
879 struct bpf_prog *old_prog;
880 int i;
881
882 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) {
883 old_prog = rcu_dereference_protected(rq->xdp_prog,
884 lockdep_is_held(&rq->priv->state_lock));
885 if (old_prog)
886 bpf_prog_put(old_prog);
887 }
888
889 switch (rq->wq_type) {
890 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
891 kvfree(rq->mpwqe.info);
892 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
893 mlx5e_free_mpwqe_rq_drop_page(rq);
894 mlx5e_rq_free_shampo(rq);
895 break;
896 default: /* MLX5_WQ_TYPE_CYCLIC */
897 kvfree(rq->wqe.frags);
898 mlx5e_free_au_list(rq);
899 }
900
901 for (i = rq->page_cache.head; i != rq->page_cache.tail;
902 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
903 /* With AF_XDP, page_cache is not used, so this loop is not
904 * entered, and it's safe to call mlx5e_page_release_dynamic
905 * directly.
906 */
907 mlx5e_page_release_dynamic(rq, rq->page_cache.page_cache[i], false);
908 }
909
910 xdp_rxq_info_unreg(&rq->xdp_rxq);
911 page_pool_destroy(rq->page_pool);
912 mlx5_wq_destroy(&rq->wq_ctrl);
913 }
914
mlx5e_create_rq(struct mlx5e_rq * rq,struct mlx5e_rq_param * param)915 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
916 {
917 struct mlx5_core_dev *mdev = rq->mdev;
918 u8 ts_format;
919 void *in;
920 void *rqc;
921 void *wq;
922 int inlen;
923 int err;
924
925 inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
926 sizeof(u64) * rq->wq_ctrl.buf.npages;
927 in = kvzalloc(inlen, GFP_KERNEL);
928 if (!in)
929 return -ENOMEM;
930
931 ts_format = mlx5_is_real_time_rq(mdev) ?
932 MLX5_TIMESTAMP_FORMAT_REAL_TIME :
933 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
934 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
935 wq = MLX5_ADDR_OF(rqc, rqc, wq);
936
937 memcpy(rqc, param->rqc, sizeof(param->rqc));
938
939 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
940 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
941 MLX5_SET(rqc, rqc, ts_format, ts_format);
942 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
943 MLX5_ADAPTER_PAGE_SHIFT);
944 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
945
946 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
947 MLX5_SET(wq, wq, log_headers_buffer_entry_num,
948 order_base_2(rq->mpwqe.shampo->hd_per_wq));
949 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey);
950 }
951
952 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
953 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
954
955 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
956
957 kvfree(in);
958
959 return err;
960 }
961
mlx5e_modify_rq_state(struct mlx5e_rq * rq,int curr_state,int next_state)962 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
963 {
964 struct mlx5_core_dev *mdev = rq->mdev;
965
966 void *in;
967 void *rqc;
968 int inlen;
969 int err;
970
971 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
972 in = kvzalloc(inlen, GFP_KERNEL);
973 if (!in)
974 return -ENOMEM;
975
976 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY)
977 mlx5e_rqwq_reset(rq);
978
979 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
980
981 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
982 MLX5_SET(rqc, rqc, state, next_state);
983
984 err = mlx5_core_modify_rq(mdev, rq->rqn, in);
985
986 kvfree(in);
987
988 return err;
989 }
990
mlx5e_rq_to_ready(struct mlx5e_rq * rq,int curr_state)991 static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state)
992 {
993 struct net_device *dev = rq->netdev;
994 int err;
995
996 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST);
997 if (err) {
998 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn);
999 return err;
1000 }
1001 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1002 if (err) {
1003 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn);
1004 return err;
1005 }
1006
1007 return 0;
1008 }
1009
mlx5e_flush_rq(struct mlx5e_rq * rq,int curr_state)1010 int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state)
1011 {
1012 mlx5e_free_rx_descs(rq);
1013
1014 return mlx5e_rq_to_ready(rq, curr_state);
1015 }
1016
mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq * rq,bool enable)1017 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
1018 {
1019 struct mlx5_core_dev *mdev = rq->mdev;
1020
1021 void *in;
1022 void *rqc;
1023 int inlen;
1024 int err;
1025
1026 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1027 in = kvzalloc(inlen, GFP_KERNEL);
1028 if (!in)
1029 return -ENOMEM;
1030
1031 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1032
1033 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
1034 MLX5_SET64(modify_rq_in, in, modify_bitmask,
1035 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS);
1036 MLX5_SET(rqc, rqc, scatter_fcs, enable);
1037 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
1038
1039 err = mlx5_core_modify_rq(mdev, rq->rqn, in);
1040
1041 kvfree(in);
1042
1043 return err;
1044 }
1045
mlx5e_modify_rq_vsd(struct mlx5e_rq * rq,bool vsd)1046 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
1047 {
1048 struct mlx5_core_dev *mdev = rq->mdev;
1049 void *in;
1050 void *rqc;
1051 int inlen;
1052 int err;
1053
1054 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1055 in = kvzalloc(inlen, GFP_KERNEL);
1056 if (!in)
1057 return -ENOMEM;
1058
1059 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1060
1061 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
1062 MLX5_SET64(modify_rq_in, in, modify_bitmask,
1063 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
1064 MLX5_SET(rqc, rqc, vsd, vsd);
1065 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
1066
1067 err = mlx5_core_modify_rq(mdev, rq->rqn, in);
1068
1069 kvfree(in);
1070
1071 return err;
1072 }
1073
mlx5e_destroy_rq(struct mlx5e_rq * rq)1074 void mlx5e_destroy_rq(struct mlx5e_rq *rq)
1075 {
1076 mlx5_core_destroy_rq(rq->mdev, rq->rqn);
1077 }
1078
mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq * rq,int wait_time)1079 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
1080 {
1081 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
1082
1083 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
1084
1085 do {
1086 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
1087 return 0;
1088
1089 msleep(20);
1090 } while (time_before(jiffies, exp_time));
1091
1092 netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
1093 rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
1094
1095 mlx5e_reporter_rx_timeout(rq);
1096 return -ETIMEDOUT;
1097 }
1098
mlx5e_free_rx_in_progress_descs(struct mlx5e_rq * rq)1099 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq)
1100 {
1101 struct mlx5_wq_ll *wq;
1102 u16 head;
1103 int i;
1104
1105 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
1106 return;
1107
1108 wq = &rq->mpwqe.wq;
1109 head = wq->head;
1110
1111 /* Outstanding UMR WQEs (in progress) start at wq->head */
1112 for (i = 0; i < rq->mpwqe.umr_in_progress; i++) {
1113 rq->dealloc_wqe(rq, head);
1114 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
1115 }
1116
1117 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
1118 u16 len;
1119
1120 len = (rq->mpwqe.shampo->pi - rq->mpwqe.shampo->ci) &
1121 (rq->mpwqe.shampo->hd_per_wq - 1);
1122 mlx5e_shampo_dealloc_hd(rq, len, rq->mpwqe.shampo->ci, false);
1123 rq->mpwqe.shampo->pi = rq->mpwqe.shampo->ci;
1124 }
1125
1126 rq->mpwqe.actual_wq_head = wq->head;
1127 rq->mpwqe.umr_in_progress = 0;
1128 rq->mpwqe.umr_completed = 0;
1129 }
1130
mlx5e_free_rx_descs(struct mlx5e_rq * rq)1131 void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
1132 {
1133 __be16 wqe_ix_be;
1134 u16 wqe_ix;
1135
1136 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
1137 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
1138
1139 mlx5e_free_rx_in_progress_descs(rq);
1140
1141 while (!mlx5_wq_ll_is_empty(wq)) {
1142 struct mlx5e_rx_wqe_ll *wqe;
1143
1144 wqe_ix_be = *wq->tail_next;
1145 wqe_ix = be16_to_cpu(wqe_ix_be);
1146 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix);
1147 rq->dealloc_wqe(rq, wqe_ix);
1148 mlx5_wq_ll_pop(wq, wqe_ix_be,
1149 &wqe->next.next_wqe_index);
1150 }
1151
1152 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
1153 mlx5e_shampo_dealloc_hd(rq, rq->mpwqe.shampo->hd_per_wq,
1154 0, true);
1155 } else {
1156 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1157
1158 while (!mlx5_wq_cyc_is_empty(wq)) {
1159 wqe_ix = mlx5_wq_cyc_get_tail(wq);
1160 rq->dealloc_wqe(rq, wqe_ix);
1161 mlx5_wq_cyc_pop(wq);
1162 }
1163 }
1164
1165 }
1166
mlx5e_open_rq(struct mlx5e_params * params,struct mlx5e_rq_param * param,struct mlx5e_xsk_param * xsk,int node,struct mlx5e_rq * rq)1167 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
1168 struct mlx5e_xsk_param *xsk, int node,
1169 struct mlx5e_rq *rq)
1170 {
1171 struct mlx5_core_dev *mdev = rq->mdev;
1172 int err;
1173
1174 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
1175 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state);
1176
1177 err = mlx5e_alloc_rq(params, xsk, param, node, rq);
1178 if (err)
1179 return err;
1180
1181 err = mlx5e_create_rq(rq, param);
1182 if (err)
1183 goto err_free_rq;
1184
1185 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1186 if (err)
1187 goto err_destroy_rq;
1188
1189 if (MLX5_CAP_ETH(mdev, cqe_checksum_full))
1190 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state);
1191
1192 if (params->rx_dim_enabled)
1193 __set_bit(MLX5E_RQ_STATE_AM, &rq->state);
1194
1195 /* We disable csum_complete when XDP is enabled since
1196 * XDP programs might manipulate packets which will render
1197 * skb->checksum incorrect.
1198 */
1199 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog)
1200 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state);
1201
1202 /* For CQE compression on striding RQ, use stride index provided by
1203 * HW if capability is supported.
1204 */
1205 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) &&
1206 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index))
1207 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state);
1208
1209 return 0;
1210
1211 err_destroy_rq:
1212 mlx5e_destroy_rq(rq);
1213 err_free_rq:
1214 mlx5e_free_rq(rq);
1215
1216 return err;
1217 }
1218
mlx5e_activate_rq(struct mlx5e_rq * rq)1219 void mlx5e_activate_rq(struct mlx5e_rq *rq)
1220 {
1221 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
1222 }
1223
mlx5e_deactivate_rq(struct mlx5e_rq * rq)1224 void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
1225 {
1226 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
1227 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
1228 }
1229
mlx5e_close_rq(struct mlx5e_rq * rq)1230 void mlx5e_close_rq(struct mlx5e_rq *rq)
1231 {
1232 cancel_work_sync(&rq->dim.work);
1233 cancel_work_sync(&rq->recover_work);
1234 mlx5e_destroy_rq(rq);
1235 mlx5e_free_rx_descs(rq);
1236 mlx5e_free_rq(rq);
1237 }
1238
mlx5e_free_xdpsq_db(struct mlx5e_xdpsq * sq)1239 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
1240 {
1241 kvfree(sq->db.xdpi_fifo.xi);
1242 kvfree(sq->db.wqe_info);
1243 }
1244
mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq * sq,int numa)1245 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
1246 {
1247 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
1248 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1249 int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1250 size_t size;
1251
1252 size = array_size(sizeof(*xdpi_fifo->xi), dsegs_per_wq);
1253 xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa);
1254 if (!xdpi_fifo->xi)
1255 return -ENOMEM;
1256
1257 xdpi_fifo->pc = &sq->xdpi_fifo_pc;
1258 xdpi_fifo->cc = &sq->xdpi_fifo_cc;
1259 xdpi_fifo->mask = dsegs_per_wq - 1;
1260
1261 return 0;
1262 }
1263
mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq * sq,int numa)1264 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
1265 {
1266 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1267 size_t size;
1268 int err;
1269
1270 size = array_size(sizeof(*sq->db.wqe_info), wq_sz);
1271 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1272 if (!sq->db.wqe_info)
1273 return -ENOMEM;
1274
1275 err = mlx5e_alloc_xdpsq_fifo(sq, numa);
1276 if (err) {
1277 mlx5e_free_xdpsq_db(sq);
1278 return err;
1279 }
1280
1281 return 0;
1282 }
1283
mlx5e_alloc_xdpsq(struct mlx5e_channel * c,struct mlx5e_params * params,struct xsk_buff_pool * xsk_pool,struct mlx5e_sq_param * param,struct mlx5e_xdpsq * sq,bool is_redirect)1284 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
1285 struct mlx5e_params *params,
1286 struct xsk_buff_pool *xsk_pool,
1287 struct mlx5e_sq_param *param,
1288 struct mlx5e_xdpsq *sq,
1289 bool is_redirect)
1290 {
1291 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
1292 struct mlx5_core_dev *mdev = c->mdev;
1293 struct mlx5_wq_cyc *wq = &sq->wq;
1294 int err;
1295
1296 sq->pdev = c->pdev;
1297 sq->mkey_be = c->mkey_be;
1298 sq->channel = c;
1299 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
1300 sq->min_inline_mode = params->tx_min_inline_mode;
1301 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN;
1302 sq->xsk_pool = xsk_pool;
1303
1304 sq->stats = sq->xsk_pool ?
1305 &c->priv->channel_stats[c->ix]->xsksq :
1306 is_redirect ?
1307 &c->priv->channel_stats[c->ix]->xdpsq :
1308 &c->priv->channel_stats[c->ix]->rq_xdpsq;
1309 sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) :
1310 mlx5e_stop_room_for_max_wqe(mdev);
1311 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
1312
1313 param->wq.db_numa_node = cpu_to_node(c->cpu);
1314 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl);
1315 if (err)
1316 return err;
1317 wq->db = &wq->db[MLX5_SND_DBR];
1318
1319 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
1320 if (err)
1321 goto err_sq_wq_destroy;
1322
1323 return 0;
1324
1325 err_sq_wq_destroy:
1326 mlx5_wq_destroy(&sq->wq_ctrl);
1327
1328 return err;
1329 }
1330
mlx5e_free_xdpsq(struct mlx5e_xdpsq * sq)1331 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1332 {
1333 mlx5e_free_xdpsq_db(sq);
1334 mlx5_wq_destroy(&sq->wq_ctrl);
1335 }
1336
mlx5e_free_icosq_db(struct mlx5e_icosq * sq)1337 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
1338 {
1339 kvfree(sq->db.wqe_info);
1340 }
1341
mlx5e_alloc_icosq_db(struct mlx5e_icosq * sq,int numa)1342 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1343 {
1344 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1345 size_t size;
1346
1347 size = array_size(wq_sz, sizeof(*sq->db.wqe_info));
1348 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1349 if (!sq->db.wqe_info)
1350 return -ENOMEM;
1351
1352 return 0;
1353 }
1354
mlx5e_icosq_err_cqe_work(struct work_struct * recover_work)1355 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
1356 {
1357 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1358 recover_work);
1359
1360 mlx5e_reporter_icosq_cqe_err(sq);
1361 }
1362
mlx5e_async_icosq_err_cqe_work(struct work_struct * recover_work)1363 static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work)
1364 {
1365 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1366 recover_work);
1367
1368 /* Not implemented yet. */
1369
1370 netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n");
1371 }
1372
mlx5e_alloc_icosq(struct mlx5e_channel * c,struct mlx5e_sq_param * param,struct mlx5e_icosq * sq,work_func_t recover_work_func)1373 static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
1374 struct mlx5e_sq_param *param,
1375 struct mlx5e_icosq *sq,
1376 work_func_t recover_work_func)
1377 {
1378 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
1379 struct mlx5_core_dev *mdev = c->mdev;
1380 struct mlx5_wq_cyc *wq = &sq->wq;
1381 int err;
1382
1383 sq->channel = c;
1384 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
1385 sq->reserved_room = param->stop_room;
1386
1387 param->wq.db_numa_node = cpu_to_node(c->cpu);
1388 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl);
1389 if (err)
1390 return err;
1391 wq->db = &wq->db[MLX5_SND_DBR];
1392
1393 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
1394 if (err)
1395 goto err_sq_wq_destroy;
1396
1397 INIT_WORK(&sq->recover_work, recover_work_func);
1398
1399 return 0;
1400
1401 err_sq_wq_destroy:
1402 mlx5_wq_destroy(&sq->wq_ctrl);
1403
1404 return err;
1405 }
1406
mlx5e_free_icosq(struct mlx5e_icosq * sq)1407 static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
1408 {
1409 mlx5e_free_icosq_db(sq);
1410 mlx5_wq_destroy(&sq->wq_ctrl);
1411 }
1412
mlx5e_free_txqsq_db(struct mlx5e_txqsq * sq)1413 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
1414 {
1415 kvfree(sq->db.wqe_info);
1416 kvfree(sq->db.skb_fifo.fifo);
1417 kvfree(sq->db.dma_fifo);
1418 }
1419
mlx5e_alloc_txqsq_db(struct mlx5e_txqsq * sq,int numa)1420 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
1421 {
1422 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1423 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1424
1425 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1426 sizeof(*sq->db.dma_fifo)),
1427 GFP_KERNEL, numa);
1428 sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz,
1429 sizeof(*sq->db.skb_fifo.fifo)),
1430 GFP_KERNEL, numa);
1431 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1432 sizeof(*sq->db.wqe_info)),
1433 GFP_KERNEL, numa);
1434 if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) {
1435 mlx5e_free_txqsq_db(sq);
1436 return -ENOMEM;
1437 }
1438
1439 sq->dma_fifo_mask = df_sz - 1;
1440
1441 sq->db.skb_fifo.pc = &sq->skb_fifo_pc;
1442 sq->db.skb_fifo.cc = &sq->skb_fifo_cc;
1443 sq->db.skb_fifo.mask = df_sz - 1;
1444
1445 return 0;
1446 }
1447
mlx5e_alloc_txqsq(struct mlx5e_channel * c,int txq_ix,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_txqsq * sq,int tc)1448 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
1449 int txq_ix,
1450 struct mlx5e_params *params,
1451 struct mlx5e_sq_param *param,
1452 struct mlx5e_txqsq *sq,
1453 int tc)
1454 {
1455 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
1456 struct mlx5_core_dev *mdev = c->mdev;
1457 struct mlx5_wq_cyc *wq = &sq->wq;
1458 int err;
1459
1460 sq->pdev = c->pdev;
1461 sq->clock = &mdev->clock;
1462 sq->mkey_be = c->mkey_be;
1463 sq->netdev = c->netdev;
1464 sq->mdev = c->mdev;
1465 sq->priv = c->priv;
1466 sq->ch_ix = c->ix;
1467 sq->txq_ix = txq_ix;
1468 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
1469 sq->min_inline_mode = params->tx_min_inline_mode;
1470 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1471 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
1472 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
1473 if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
1474 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
1475 if (mlx5_ipsec_device_caps(c->priv->mdev))
1476 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
1477 if (param->is_mpw)
1478 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state);
1479 sq->stop_room = param->stop_room;
1480 sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev);
1481
1482 param->wq.db_numa_node = cpu_to_node(c->cpu);
1483 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl);
1484 if (err)
1485 return err;
1486 wq->db = &wq->db[MLX5_SND_DBR];
1487
1488 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
1489 if (err)
1490 goto err_sq_wq_destroy;
1491
1492 INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
1493 sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
1494
1495 return 0;
1496
1497 err_sq_wq_destroy:
1498 mlx5_wq_destroy(&sq->wq_ctrl);
1499
1500 return err;
1501 }
1502
mlx5e_free_txqsq(struct mlx5e_txqsq * sq)1503 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
1504 {
1505 mlx5e_free_txqsq_db(sq);
1506 mlx5_wq_destroy(&sq->wq_ctrl);
1507 }
1508
mlx5e_create_sq(struct mlx5_core_dev * mdev,struct mlx5e_sq_param * param,struct mlx5e_create_sq_param * csp,u32 * sqn)1509 static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
1510 struct mlx5e_sq_param *param,
1511 struct mlx5e_create_sq_param *csp,
1512 u32 *sqn)
1513 {
1514 u8 ts_format;
1515 void *in;
1516 void *sqc;
1517 void *wq;
1518 int inlen;
1519 int err;
1520
1521 inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1522 sizeof(u64) * csp->wq_ctrl->buf.npages;
1523 in = kvzalloc(inlen, GFP_KERNEL);
1524 if (!in)
1525 return -ENOMEM;
1526
1527 ts_format = mlx5_is_real_time_sq(mdev) ?
1528 MLX5_TIMESTAMP_FORMAT_REAL_TIME :
1529 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1530 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1531 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1532
1533 memcpy(sqc, param->sqc, sizeof(param->sqc));
1534 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz);
1535 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn);
1536 MLX5_SET(sqc, sqc, cqn, csp->cqn);
1537 MLX5_SET(sqc, sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn);
1538 MLX5_SET(sqc, sqc, ts_format, ts_format);
1539
1540
1541 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
1542 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode);
1543
1544 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1545 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1546
1547 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1548 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index);
1549 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift -
1550 MLX5_ADAPTER_PAGE_SHIFT);
1551 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma);
1552
1553 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1554 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1555
1556 err = mlx5_core_create_sq(mdev, in, inlen, sqn);
1557
1558 kvfree(in);
1559
1560 return err;
1561 }
1562
mlx5e_modify_sq(struct mlx5_core_dev * mdev,u32 sqn,struct mlx5e_modify_sq_param * p)1563 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1564 struct mlx5e_modify_sq_param *p)
1565 {
1566 u64 bitmask = 0;
1567 void *in;
1568 void *sqc;
1569 int inlen;
1570 int err;
1571
1572 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1573 in = kvzalloc(inlen, GFP_KERNEL);
1574 if (!in)
1575 return -ENOMEM;
1576
1577 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1578
1579 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1580 MLX5_SET(sqc, sqc, state, p->next_state);
1581 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
1582 bitmask |= 1;
1583 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
1584 }
1585 if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) {
1586 bitmask |= 1 << 2;
1587 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id);
1588 }
1589 MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask);
1590
1591 err = mlx5_core_modify_sq(mdev, sqn, in);
1592
1593 kvfree(in);
1594
1595 return err;
1596 }
1597
mlx5e_destroy_sq(struct mlx5_core_dev * mdev,u32 sqn)1598 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
1599 {
1600 mlx5_core_destroy_sq(mdev, sqn);
1601 }
1602
mlx5e_create_sq_rdy(struct mlx5_core_dev * mdev,struct mlx5e_sq_param * param,struct mlx5e_create_sq_param * csp,u16 qos_queue_group_id,u32 * sqn)1603 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1604 struct mlx5e_sq_param *param,
1605 struct mlx5e_create_sq_param *csp,
1606 u16 qos_queue_group_id,
1607 u32 *sqn)
1608 {
1609 struct mlx5e_modify_sq_param msp = {0};
1610 int err;
1611
1612 err = mlx5e_create_sq(mdev, param, csp, sqn);
1613 if (err)
1614 return err;
1615
1616 msp.curr_state = MLX5_SQC_STATE_RST;
1617 msp.next_state = MLX5_SQC_STATE_RDY;
1618 if (qos_queue_group_id) {
1619 msp.qos_update = true;
1620 msp.qos_queue_group_id = qos_queue_group_id;
1621 }
1622 err = mlx5e_modify_sq(mdev, *sqn, &msp);
1623 if (err)
1624 mlx5e_destroy_sq(mdev, *sqn);
1625
1626 return err;
1627 }
1628
1629 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1630 struct mlx5e_txqsq *sq, u32 rate);
1631
mlx5e_open_txqsq(struct mlx5e_channel * c,u32 tisn,int txq_ix,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_txqsq * sq,int tc,u16 qos_queue_group_id,struct mlx5e_sq_stats * sq_stats)1632 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1633 struct mlx5e_params *params, struct mlx5e_sq_param *param,
1634 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
1635 struct mlx5e_sq_stats *sq_stats)
1636 {
1637 struct mlx5e_create_sq_param csp = {};
1638 u32 tx_rate;
1639 int err;
1640
1641 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
1642 if (err)
1643 return err;
1644
1645 sq->stats = sq_stats;
1646
1647 csp.tisn = tisn;
1648 csp.tis_lst_sz = 1;
1649 csp.cqn = sq->cq.mcq.cqn;
1650 csp.wq_ctrl = &sq->wq_ctrl;
1651 csp.min_inline_mode = sq->min_inline_mode;
1652 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn);
1653 if (err)
1654 goto err_free_txqsq;
1655
1656 tx_rate = c->priv->tx_rates[sq->txq_ix];
1657 if (tx_rate)
1658 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
1659
1660 if (params->tx_dim_enabled)
1661 sq->state |= BIT(MLX5E_SQ_STATE_AM);
1662
1663 return 0;
1664
1665 err_free_txqsq:
1666 mlx5e_free_txqsq(sq);
1667
1668 return err;
1669 }
1670
mlx5e_activate_txqsq(struct mlx5e_txqsq * sq)1671 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1672 {
1673 sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix);
1674 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1675 netdev_tx_reset_queue(sq->txq);
1676 netif_tx_start_queue(sq->txq);
1677 }
1678
mlx5e_tx_disable_queue(struct netdev_queue * txq)1679 void mlx5e_tx_disable_queue(struct netdev_queue *txq)
1680 {
1681 __netif_tx_lock_bh(txq);
1682 netif_tx_stop_queue(txq);
1683 __netif_tx_unlock_bh(txq);
1684 }
1685
mlx5e_deactivate_txqsq(struct mlx5e_txqsq * sq)1686 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
1687 {
1688 struct mlx5_wq_cyc *wq = &sq->wq;
1689
1690 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1691 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
1692
1693 mlx5e_tx_disable_queue(sq->txq);
1694
1695 /* last doorbell out, godspeed .. */
1696 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1697 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
1698 struct mlx5e_tx_wqe *nop;
1699
1700 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) {
1701 .num_wqebbs = 1,
1702 };
1703
1704 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1705 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
1706 }
1707 }
1708
mlx5e_close_txqsq(struct mlx5e_txqsq * sq)1709 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1710 {
1711 struct mlx5_core_dev *mdev = sq->mdev;
1712 struct mlx5_rate_limit rl = {0};
1713
1714 cancel_work_sync(&sq->dim.work);
1715 cancel_work_sync(&sq->recover_work);
1716 mlx5e_destroy_sq(mdev, sq->sqn);
1717 if (sq->rate_limit) {
1718 rl.rate = sq->rate_limit;
1719 mlx5_rl_remove_rate(mdev, &rl);
1720 }
1721 mlx5e_free_txqsq_descs(sq);
1722 mlx5e_free_txqsq(sq);
1723 }
1724
mlx5e_tx_err_cqe_work(struct work_struct * recover_work)1725 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
1726 {
1727 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq,
1728 recover_work);
1729
1730 mlx5e_reporter_tx_err_cqe(sq);
1731 }
1732
mlx5e_open_icosq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_icosq * sq,work_func_t recover_work_func)1733 static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
1734 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq,
1735 work_func_t recover_work_func)
1736 {
1737 struct mlx5e_create_sq_param csp = {};
1738 int err;
1739
1740 err = mlx5e_alloc_icosq(c, param, sq, recover_work_func);
1741 if (err)
1742 return err;
1743
1744 csp.cqn = sq->cq.mcq.cqn;
1745 csp.wq_ctrl = &sq->wq_ctrl;
1746 csp.min_inline_mode = params->tx_min_inline_mode;
1747 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1748 if (err)
1749 goto err_free_icosq;
1750
1751 if (param->is_tls) {
1752 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list();
1753 if (IS_ERR(sq->ktls_resync)) {
1754 err = PTR_ERR(sq->ktls_resync);
1755 goto err_destroy_icosq;
1756 }
1757 }
1758 return 0;
1759
1760 err_destroy_icosq:
1761 mlx5e_destroy_sq(c->mdev, sq->sqn);
1762 err_free_icosq:
1763 mlx5e_free_icosq(sq);
1764
1765 return err;
1766 }
1767
mlx5e_activate_icosq(struct mlx5e_icosq * icosq)1768 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
1769 {
1770 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1771 }
1772
mlx5e_deactivate_icosq(struct mlx5e_icosq * icosq)1773 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
1774 {
1775 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1776 synchronize_net(); /* Sync with NAPI. */
1777 }
1778
mlx5e_close_icosq(struct mlx5e_icosq * sq)1779 static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1780 {
1781 struct mlx5e_channel *c = sq->channel;
1782
1783 if (sq->ktls_resync)
1784 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync);
1785 mlx5e_destroy_sq(c->mdev, sq->sqn);
1786 mlx5e_free_icosq_descs(sq);
1787 mlx5e_free_icosq(sq);
1788 }
1789
mlx5e_open_xdpsq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct xsk_buff_pool * xsk_pool,struct mlx5e_xdpsq * sq,bool is_redirect)1790 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1791 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
1792 struct mlx5e_xdpsq *sq, bool is_redirect)
1793 {
1794 struct mlx5e_create_sq_param csp = {};
1795 int err;
1796
1797 err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect);
1798 if (err)
1799 return err;
1800
1801 csp.tis_lst_sz = 1;
1802 csp.tisn = c->priv->tisn[c->lag_port][0]; /* tc = 0 */
1803 csp.cqn = sq->cq.mcq.cqn;
1804 csp.wq_ctrl = &sq->wq_ctrl;
1805 csp.min_inline_mode = sq->min_inline_mode;
1806 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1807
1808 /* Don't enable multi buffer on XDP_REDIRECT SQ, as it's not yet
1809 * supported by upstream, and there is no defined trigger to allow
1810 * transmitting redirected multi-buffer frames.
1811 */
1812 if (param->is_xdp_mb && !is_redirect)
1813 set_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state);
1814
1815 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1816 if (err)
1817 goto err_free_xdpsq;
1818
1819 mlx5e_set_xmit_fp(sq, param->is_mpw);
1820
1821 if (!param->is_mpw && !test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state)) {
1822 unsigned int ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + 1;
1823 unsigned int inline_hdr_sz = 0;
1824 int i;
1825
1826 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1827 inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1828 ds_cnt++;
1829 }
1830
1831 /* Pre initialize fixed WQE fields */
1832 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1833 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1834 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1835 struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
1836 struct mlx5_wqe_data_seg *dseg;
1837
1838 sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) {
1839 .num_wqebbs = 1,
1840 .num_pkts = 1,
1841 };
1842
1843 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1844 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1845
1846 dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
1847 dseg->lkey = sq->mkey_be;
1848 }
1849 }
1850
1851 return 0;
1852
1853 err_free_xdpsq:
1854 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1855 mlx5e_free_xdpsq(sq);
1856
1857 return err;
1858 }
1859
mlx5e_close_xdpsq(struct mlx5e_xdpsq * sq)1860 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1861 {
1862 struct mlx5e_channel *c = sq->channel;
1863
1864 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1865 synchronize_net(); /* Sync with NAPI. */
1866
1867 mlx5e_destroy_sq(c->mdev, sq->sqn);
1868 mlx5e_free_xdpsq_descs(sq);
1869 mlx5e_free_xdpsq(sq);
1870 }
1871
mlx5e_alloc_cq_common(struct mlx5e_priv * priv,struct mlx5e_cq_param * param,struct mlx5e_cq * cq)1872 static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv,
1873 struct mlx5e_cq_param *param,
1874 struct mlx5e_cq *cq)
1875 {
1876 struct mlx5_core_dev *mdev = priv->mdev;
1877 struct mlx5_core_cq *mcq = &cq->mcq;
1878 int err;
1879 u32 i;
1880
1881 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq,
1882 &cq->wq_ctrl);
1883 if (err)
1884 return err;
1885
1886 mcq->cqe_sz = 64;
1887 mcq->set_ci_db = cq->wq_ctrl.db.db;
1888 mcq->arm_db = cq->wq_ctrl.db.db + 1;
1889 *mcq->set_ci_db = 0;
1890 *mcq->arm_db = 0;
1891 mcq->vector = param->eq_ix;
1892 mcq->comp = mlx5e_completion_event;
1893 mcq->event = mlx5e_cq_error_event;
1894
1895 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1896 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1897
1898 cqe->op_own = 0xf1;
1899 }
1900
1901 cq->mdev = mdev;
1902 cq->netdev = priv->netdev;
1903 cq->priv = priv;
1904
1905 return 0;
1906 }
1907
mlx5e_alloc_cq(struct mlx5e_priv * priv,struct mlx5e_cq_param * param,struct mlx5e_create_cq_param * ccp,struct mlx5e_cq * cq)1908 static int mlx5e_alloc_cq(struct mlx5e_priv *priv,
1909 struct mlx5e_cq_param *param,
1910 struct mlx5e_create_cq_param *ccp,
1911 struct mlx5e_cq *cq)
1912 {
1913 int err;
1914
1915 param->wq.buf_numa_node = ccp->node;
1916 param->wq.db_numa_node = ccp->node;
1917 param->eq_ix = ccp->ix;
1918
1919 err = mlx5e_alloc_cq_common(priv, param, cq);
1920
1921 cq->napi = ccp->napi;
1922 cq->ch_stats = ccp->ch_stats;
1923
1924 return err;
1925 }
1926
mlx5e_free_cq(struct mlx5e_cq * cq)1927 static void mlx5e_free_cq(struct mlx5e_cq *cq)
1928 {
1929 mlx5_wq_destroy(&cq->wq_ctrl);
1930 }
1931
mlx5e_create_cq(struct mlx5e_cq * cq,struct mlx5e_cq_param * param)1932 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1933 {
1934 u32 out[MLX5_ST_SZ_DW(create_cq_out)];
1935 struct mlx5_core_dev *mdev = cq->mdev;
1936 struct mlx5_core_cq *mcq = &cq->mcq;
1937
1938 void *in;
1939 void *cqc;
1940 int inlen;
1941 int eqn;
1942 int err;
1943
1944 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn);
1945 if (err)
1946 return err;
1947
1948 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1949 sizeof(u64) * cq->wq_ctrl.buf.npages;
1950 in = kvzalloc(inlen, GFP_KERNEL);
1951 if (!in)
1952 return -ENOMEM;
1953
1954 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1955
1956 memcpy(cqc, param->cqc, sizeof(param->cqc));
1957
1958 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
1959 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1960
1961 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode);
1962 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
1963 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
1964 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1965 MLX5_ADAPTER_PAGE_SHIFT);
1966 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1967
1968 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
1969
1970 kvfree(in);
1971
1972 if (err)
1973 return err;
1974
1975 mlx5e_cq_arm(cq);
1976
1977 return 0;
1978 }
1979
mlx5e_destroy_cq(struct mlx5e_cq * cq)1980 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1981 {
1982 mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
1983 }
1984
mlx5e_open_cq(struct mlx5e_priv * priv,struct dim_cq_moder moder,struct mlx5e_cq_param * param,struct mlx5e_create_cq_param * ccp,struct mlx5e_cq * cq)1985 int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder,
1986 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
1987 struct mlx5e_cq *cq)
1988 {
1989 struct mlx5_core_dev *mdev = priv->mdev;
1990 int err;
1991
1992 err = mlx5e_alloc_cq(priv, param, ccp, cq);
1993 if (err)
1994 return err;
1995
1996 err = mlx5e_create_cq(cq, param);
1997 if (err)
1998 goto err_free_cq;
1999
2000 if (MLX5_CAP_GEN(mdev, cq_moderation))
2001 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
2002 return 0;
2003
2004 err_free_cq:
2005 mlx5e_free_cq(cq);
2006
2007 return err;
2008 }
2009
mlx5e_close_cq(struct mlx5e_cq * cq)2010 void mlx5e_close_cq(struct mlx5e_cq *cq)
2011 {
2012 mlx5e_destroy_cq(cq);
2013 mlx5e_free_cq(cq);
2014 }
2015
mlx5e_open_tx_cqs(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_create_cq_param * ccp,struct mlx5e_channel_param * cparam)2016 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2017 struct mlx5e_params *params,
2018 struct mlx5e_create_cq_param *ccp,
2019 struct mlx5e_channel_param *cparam)
2020 {
2021 int err;
2022 int tc;
2023
2024 for (tc = 0; tc < c->num_tc; tc++) {
2025 err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->txq_sq.cqp,
2026 ccp, &c->sq[tc].cq);
2027 if (err)
2028 goto err_close_tx_cqs;
2029 }
2030
2031 return 0;
2032
2033 err_close_tx_cqs:
2034 for (tc--; tc >= 0; tc--)
2035 mlx5e_close_cq(&c->sq[tc].cq);
2036
2037 return err;
2038 }
2039
mlx5e_close_tx_cqs(struct mlx5e_channel * c)2040 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2041 {
2042 int tc;
2043
2044 for (tc = 0; tc < c->num_tc; tc++)
2045 mlx5e_close_cq(&c->sq[tc].cq);
2046 }
2047
mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq * tc_to_txq,unsigned int txq)2048 static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq)
2049 {
2050 int tc;
2051
2052 for (tc = 0; tc < TC_MAX_QUEUE; tc++)
2053 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count)
2054 return tc;
2055
2056 WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq);
2057 return -ENOENT;
2058 }
2059
mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params * params,int txq_ix,u32 * hw_id)2060 static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix,
2061 u32 *hw_id)
2062 {
2063 int tc;
2064
2065 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) {
2066 *hw_id = 0;
2067 return 0;
2068 }
2069
2070 tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix);
2071 if (tc < 0)
2072 return tc;
2073
2074 if (tc >= params->mqprio.num_tc) {
2075 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u",
2076 tc, params->mqprio.num_tc);
2077 return -EINVAL;
2078 }
2079
2080 *hw_id = params->mqprio.channel.hw_id[tc];
2081 return 0;
2082 }
2083
mlx5e_open_sqs(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_channel_param * cparam)2084 static int mlx5e_open_sqs(struct mlx5e_channel *c,
2085 struct mlx5e_params *params,
2086 struct mlx5e_channel_param *cparam)
2087 {
2088 int err, tc;
2089
2090 for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) {
2091 int txq_ix = c->ix + tc * params->num_channels;
2092 u32 qos_queue_group_id;
2093
2094 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id);
2095 if (err)
2096 goto err_close_sqs;
2097
2098 err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix,
2099 params, &cparam->txq_sq, &c->sq[tc], tc,
2100 qos_queue_group_id,
2101 &c->priv->channel_stats[c->ix]->sq[tc]);
2102 if (err)
2103 goto err_close_sqs;
2104 }
2105
2106 return 0;
2107
2108 err_close_sqs:
2109 for (tc--; tc >= 0; tc--)
2110 mlx5e_close_txqsq(&c->sq[tc]);
2111
2112 return err;
2113 }
2114
mlx5e_close_sqs(struct mlx5e_channel * c)2115 static void mlx5e_close_sqs(struct mlx5e_channel *c)
2116 {
2117 int tc;
2118
2119 for (tc = 0; tc < c->num_tc; tc++)
2120 mlx5e_close_txqsq(&c->sq[tc]);
2121 }
2122
mlx5e_set_sq_maxrate(struct net_device * dev,struct mlx5e_txqsq * sq,u32 rate)2123 static int mlx5e_set_sq_maxrate(struct net_device *dev,
2124 struct mlx5e_txqsq *sq, u32 rate)
2125 {
2126 struct mlx5e_priv *priv = netdev_priv(dev);
2127 struct mlx5_core_dev *mdev = priv->mdev;
2128 struct mlx5e_modify_sq_param msp = {0};
2129 struct mlx5_rate_limit rl = {0};
2130 u16 rl_index = 0;
2131 int err;
2132
2133 if (rate == sq->rate_limit)
2134 /* nothing to do */
2135 return 0;
2136
2137 if (sq->rate_limit) {
2138 rl.rate = sq->rate_limit;
2139 /* remove current rl index to free space to next ones */
2140 mlx5_rl_remove_rate(mdev, &rl);
2141 }
2142
2143 sq->rate_limit = 0;
2144
2145 if (rate) {
2146 rl.rate = rate;
2147 err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
2148 if (err) {
2149 netdev_err(dev, "Failed configuring rate %u: %d\n",
2150 rate, err);
2151 return err;
2152 }
2153 }
2154
2155 msp.curr_state = MLX5_SQC_STATE_RDY;
2156 msp.next_state = MLX5_SQC_STATE_RDY;
2157 msp.rl_index = rl_index;
2158 msp.rl_update = true;
2159 err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
2160 if (err) {
2161 netdev_err(dev, "Failed configuring rate %u: %d\n",
2162 rate, err);
2163 /* remove the rate from the table */
2164 if (rate)
2165 mlx5_rl_remove_rate(mdev, &rl);
2166 return err;
2167 }
2168
2169 sq->rate_limit = rate;
2170 return 0;
2171 }
2172
mlx5e_set_tx_maxrate(struct net_device * dev,int index,u32 rate)2173 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
2174 {
2175 struct mlx5e_priv *priv = netdev_priv(dev);
2176 struct mlx5_core_dev *mdev = priv->mdev;
2177 struct mlx5e_txqsq *sq = priv->txq2sq[index];
2178 int err = 0;
2179
2180 if (!mlx5_rl_is_supported(mdev)) {
2181 netdev_err(dev, "Rate limiting is not supported on this device\n");
2182 return -EINVAL;
2183 }
2184
2185 /* rate is given in Mb/sec, HW config is in Kb/sec */
2186 rate = rate << 10;
2187
2188 /* Check whether rate in valid range, 0 is always valid */
2189 if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
2190 netdev_err(dev, "TX rate %u, is not in range\n", rate);
2191 return -ERANGE;
2192 }
2193
2194 mutex_lock(&priv->state_lock);
2195 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
2196 err = mlx5e_set_sq_maxrate(dev, sq, rate);
2197 if (!err)
2198 priv->tx_rates[index] = rate;
2199 mutex_unlock(&priv->state_lock);
2200
2201 return err;
2202 }
2203
mlx5e_open_rxq_rq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_rq_param * rq_params)2204 static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
2205 struct mlx5e_rq_param *rq_params)
2206 {
2207 int err;
2208
2209 err = mlx5e_init_rxq_rq(c, params, &c->rq);
2210 if (err)
2211 return err;
2212
2213 return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), &c->rq);
2214 }
2215
mlx5e_open_queues(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_channel_param * cparam)2216 static int mlx5e_open_queues(struct mlx5e_channel *c,
2217 struct mlx5e_params *params,
2218 struct mlx5e_channel_param *cparam)
2219 {
2220 struct dim_cq_moder icocq_moder = {0, 0};
2221 struct mlx5e_create_cq_param ccp;
2222 int err;
2223
2224 mlx5e_build_create_cq_param(&ccp, c);
2225
2226 err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->async_icosq.cqp, &ccp,
2227 &c->async_icosq.cq);
2228 if (err)
2229 return err;
2230
2231 err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->icosq.cqp, &ccp,
2232 &c->icosq.cq);
2233 if (err)
2234 goto err_close_async_icosq_cq;
2235
2236 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam);
2237 if (err)
2238 goto err_close_icosq_cq;
2239
2240 err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
2241 &c->xdpsq.cq);
2242 if (err)
2243 goto err_close_tx_cqs;
2244
2245 err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
2246 &c->rq.cq);
2247 if (err)
2248 goto err_close_xdp_tx_cqs;
2249
2250 err = c->xdp ? mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp,
2251 &ccp, &c->rq_xdpsq.cq) : 0;
2252 if (err)
2253 goto err_close_rx_cq;
2254
2255 spin_lock_init(&c->async_icosq_lock);
2256
2257 err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq,
2258 mlx5e_async_icosq_err_cqe_work);
2259 if (err)
2260 goto err_close_xdpsq_cq;
2261
2262 mutex_init(&c->icosq_recovery_lock);
2263
2264 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq,
2265 mlx5e_icosq_err_cqe_work);
2266 if (err)
2267 goto err_close_async_icosq;
2268
2269 err = mlx5e_open_sqs(c, params, cparam);
2270 if (err)
2271 goto err_close_icosq;
2272
2273 err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
2274 if (err)
2275 goto err_close_sqs;
2276
2277 if (c->xdp) {
2278 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL,
2279 &c->rq_xdpsq, false);
2280 if (err)
2281 goto err_close_rq;
2282 }
2283
2284 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true);
2285 if (err)
2286 goto err_close_xdp_sq;
2287
2288 return 0;
2289
2290 err_close_xdp_sq:
2291 if (c->xdp)
2292 mlx5e_close_xdpsq(&c->rq_xdpsq);
2293
2294 err_close_rq:
2295 mlx5e_close_rq(&c->rq);
2296
2297 err_close_sqs:
2298 mlx5e_close_sqs(c);
2299
2300 err_close_icosq:
2301 mlx5e_close_icosq(&c->icosq);
2302
2303 err_close_async_icosq:
2304 mlx5e_close_icosq(&c->async_icosq);
2305
2306 err_close_xdpsq_cq:
2307 if (c->xdp)
2308 mlx5e_close_cq(&c->rq_xdpsq.cq);
2309
2310 err_close_rx_cq:
2311 mlx5e_close_cq(&c->rq.cq);
2312
2313 err_close_xdp_tx_cqs:
2314 mlx5e_close_cq(&c->xdpsq.cq);
2315
2316 err_close_tx_cqs:
2317 mlx5e_close_tx_cqs(c);
2318
2319 err_close_icosq_cq:
2320 mlx5e_close_cq(&c->icosq.cq);
2321
2322 err_close_async_icosq_cq:
2323 mlx5e_close_cq(&c->async_icosq.cq);
2324
2325 return err;
2326 }
2327
mlx5e_close_queues(struct mlx5e_channel * c)2328 static void mlx5e_close_queues(struct mlx5e_channel *c)
2329 {
2330 mlx5e_close_xdpsq(&c->xdpsq);
2331 if (c->xdp)
2332 mlx5e_close_xdpsq(&c->rq_xdpsq);
2333 /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */
2334 cancel_work_sync(&c->icosq.recover_work);
2335 mlx5e_close_rq(&c->rq);
2336 mlx5e_close_sqs(c);
2337 mlx5e_close_icosq(&c->icosq);
2338 mutex_destroy(&c->icosq_recovery_lock);
2339 mlx5e_close_icosq(&c->async_icosq);
2340 if (c->xdp)
2341 mlx5e_close_cq(&c->rq_xdpsq.cq);
2342 mlx5e_close_cq(&c->rq.cq);
2343 mlx5e_close_cq(&c->xdpsq.cq);
2344 mlx5e_close_tx_cqs(c);
2345 mlx5e_close_cq(&c->icosq.cq);
2346 mlx5e_close_cq(&c->async_icosq.cq);
2347 }
2348
mlx5e_enumerate_lag_port(struct mlx5_core_dev * mdev,int ix)2349 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
2350 {
2351 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id);
2352
2353 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev);
2354 }
2355
mlx5e_channel_stats_alloc(struct mlx5e_priv * priv,int ix,int cpu)2356 static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu)
2357 {
2358 if (ix > priv->stats_nch) {
2359 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix,
2360 priv->stats_nch);
2361 return -EINVAL;
2362 }
2363
2364 if (priv->channel_stats[ix])
2365 return 0;
2366
2367 /* Asymmetric dynamic memory allocation.
2368 * Freed in mlx5e_priv_arrays_free, not on channel closure.
2369 */
2370 mlx5e_dbg(DRV, priv, "Creating channel stats %d\n", ix);
2371 priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats),
2372 GFP_KERNEL, cpu_to_node(cpu));
2373 if (!priv->channel_stats[ix])
2374 return -ENOMEM;
2375 priv->stats_nch++;
2376
2377 return 0;
2378 }
2379
mlx5e_trigger_napi_icosq(struct mlx5e_channel * c)2380 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
2381 {
2382 spin_lock_bh(&c->async_icosq_lock);
2383 mlx5e_trigger_irq(&c->async_icosq);
2384 spin_unlock_bh(&c->async_icosq_lock);
2385 }
2386
mlx5e_trigger_napi_sched(struct napi_struct * napi)2387 void mlx5e_trigger_napi_sched(struct napi_struct *napi)
2388 {
2389 local_bh_disable();
2390 napi_schedule(napi);
2391 local_bh_enable();
2392 }
2393
mlx5e_open_channel(struct mlx5e_priv * priv,int ix,struct mlx5e_params * params,struct mlx5e_channel_param * cparam,struct xsk_buff_pool * xsk_pool,struct mlx5e_channel ** cp)2394 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
2395 struct mlx5e_params *params,
2396 struct mlx5e_channel_param *cparam,
2397 struct xsk_buff_pool *xsk_pool,
2398 struct mlx5e_channel **cp)
2399 {
2400 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix));
2401 struct net_device *netdev = priv->netdev;
2402 struct mlx5e_xsk_param xsk;
2403 struct mlx5e_channel *c;
2404 unsigned int irq;
2405 int err;
2406
2407 err = mlx5_vector2irqn(priv->mdev, ix, &irq);
2408 if (err)
2409 return err;
2410
2411 err = mlx5e_channel_stats_alloc(priv, ix, cpu);
2412 if (err)
2413 return err;
2414
2415 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
2416 if (!c)
2417 return -ENOMEM;
2418
2419 c->priv = priv;
2420 c->mdev = priv->mdev;
2421 c->tstamp = &priv->tstamp;
2422 c->ix = ix;
2423 c->cpu = cpu;
2424 c->pdev = mlx5_core_dma_dev(priv->mdev);
2425 c->netdev = priv->netdev;
2426 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey);
2427 c->num_tc = mlx5e_get_dcb_num_tc(params);
2428 c->xdp = !!params->xdp_prog;
2429 c->stats = &priv->channel_stats[ix]->ch;
2430 c->aff_mask = irq_get_effective_affinity_mask(irq);
2431 c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix);
2432
2433 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll);
2434
2435 err = mlx5e_open_queues(c, params, cparam);
2436 if (unlikely(err))
2437 goto err_napi_del;
2438
2439 if (xsk_pool) {
2440 mlx5e_build_xsk_param(xsk_pool, &xsk);
2441 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
2442 if (unlikely(err))
2443 goto err_close_queues;
2444 }
2445
2446 *cp = c;
2447
2448 return 0;
2449
2450 err_close_queues:
2451 mlx5e_close_queues(c);
2452
2453 err_napi_del:
2454 netif_napi_del(&c->napi);
2455
2456 kvfree(c);
2457
2458 return err;
2459 }
2460
mlx5e_activate_channel(struct mlx5e_channel * c)2461 static void mlx5e_activate_channel(struct mlx5e_channel *c)
2462 {
2463 int tc;
2464
2465 napi_enable(&c->napi);
2466
2467 for (tc = 0; tc < c->num_tc; tc++)
2468 mlx5e_activate_txqsq(&c->sq[tc]);
2469 mlx5e_activate_icosq(&c->icosq);
2470 mlx5e_activate_icosq(&c->async_icosq);
2471
2472 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2473 mlx5e_activate_xsk(c);
2474 else
2475 mlx5e_activate_rq(&c->rq);
2476
2477 mlx5e_trigger_napi_icosq(c);
2478 }
2479
mlx5e_deactivate_channel(struct mlx5e_channel * c)2480 static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2481 {
2482 int tc;
2483
2484 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2485 mlx5e_deactivate_xsk(c);
2486 else
2487 mlx5e_deactivate_rq(&c->rq);
2488
2489 mlx5e_deactivate_icosq(&c->async_icosq);
2490 mlx5e_deactivate_icosq(&c->icosq);
2491 for (tc = 0; tc < c->num_tc; tc++)
2492 mlx5e_deactivate_txqsq(&c->sq[tc]);
2493 mlx5e_qos_deactivate_queues(c);
2494
2495 napi_disable(&c->napi);
2496 }
2497
mlx5e_close_channel(struct mlx5e_channel * c)2498 static void mlx5e_close_channel(struct mlx5e_channel *c)
2499 {
2500 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2501 mlx5e_close_xsk(c);
2502 mlx5e_close_queues(c);
2503 mlx5e_qos_close_queues(c);
2504 netif_napi_del(&c->napi);
2505
2506 kvfree(c);
2507 }
2508
mlx5e_open_channels(struct mlx5e_priv * priv,struct mlx5e_channels * chs)2509 int mlx5e_open_channels(struct mlx5e_priv *priv,
2510 struct mlx5e_channels *chs)
2511 {
2512 struct mlx5e_channel_param *cparam;
2513 int err = -ENOMEM;
2514 int i;
2515
2516 chs->num = chs->params.num_channels;
2517
2518 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
2519 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
2520 if (!chs->c || !cparam)
2521 goto err_free;
2522
2523 err = mlx5e_build_channel_param(priv->mdev, &chs->params, priv->q_counter, cparam);
2524 if (err)
2525 goto err_free;
2526
2527 for (i = 0; i < chs->num; i++) {
2528 struct xsk_buff_pool *xsk_pool = NULL;
2529
2530 if (chs->params.xdp_prog)
2531 xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
2532
2533 err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]);
2534 if (err)
2535 goto err_close_channels;
2536 }
2537
2538 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) {
2539 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp);
2540 if (err)
2541 goto err_close_channels;
2542 }
2543
2544 if (priv->htb) {
2545 err = mlx5e_qos_open_queues(priv, chs);
2546 if (err)
2547 goto err_close_ptp;
2548 }
2549
2550 mlx5e_health_channels_update(priv);
2551 kvfree(cparam);
2552 return 0;
2553
2554 err_close_ptp:
2555 if (chs->ptp)
2556 mlx5e_ptp_close(chs->ptp);
2557
2558 err_close_channels:
2559 for (i--; i >= 0; i--)
2560 mlx5e_close_channel(chs->c[i]);
2561
2562 err_free:
2563 kfree(chs->c);
2564 kvfree(cparam);
2565 chs->num = 0;
2566 return err;
2567 }
2568
mlx5e_activate_channels(struct mlx5e_channels * chs)2569 static void mlx5e_activate_channels(struct mlx5e_channels *chs)
2570 {
2571 int i;
2572
2573 for (i = 0; i < chs->num; i++)
2574 mlx5e_activate_channel(chs->c[i]);
2575
2576 if (chs->ptp)
2577 mlx5e_ptp_activate_channel(chs->ptp);
2578 }
2579
mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels * chs)2580 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2581 {
2582 int err = 0;
2583 int i;
2584
2585 for (i = 0; i < chs->num; i++) {
2586 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2587 struct mlx5e_channel *c = chs->c[i];
2588
2589 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2590 continue;
2591
2592 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout);
2593
2594 /* Don't wait on the XSK RQ, because the newer xdpsock sample
2595 * doesn't provide any Fill Ring entries at the setup stage.
2596 */
2597 }
2598
2599 return err ? -ETIMEDOUT : 0;
2600 }
2601
mlx5e_deactivate_channels(struct mlx5e_channels * chs)2602 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2603 {
2604 int i;
2605
2606 if (chs->ptp)
2607 mlx5e_ptp_deactivate_channel(chs->ptp);
2608
2609 for (i = 0; i < chs->num; i++)
2610 mlx5e_deactivate_channel(chs->c[i]);
2611 }
2612
mlx5e_close_channels(struct mlx5e_channels * chs)2613 void mlx5e_close_channels(struct mlx5e_channels *chs)
2614 {
2615 int i;
2616
2617 if (chs->ptp) {
2618 mlx5e_ptp_close(chs->ptp);
2619 chs->ptp = NULL;
2620 }
2621 for (i = 0; i < chs->num; i++)
2622 mlx5e_close_channel(chs->c[i]);
2623
2624 kfree(chs->c);
2625 chs->num = 0;
2626 }
2627
mlx5e_modify_tirs_packet_merge(struct mlx5e_priv * priv)2628 static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv)
2629 {
2630 struct mlx5e_rx_res *res = priv->rx_res;
2631
2632 return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge);
2633 }
2634
2635 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge);
2636
mlx5e_set_mtu(struct mlx5_core_dev * mdev,struct mlx5e_params * params,u16 mtu)2637 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2638 struct mlx5e_params *params, u16 mtu)
2639 {
2640 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
2641 int err;
2642
2643 err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2644 if (err)
2645 return err;
2646
2647 /* Update vport context MTU */
2648 mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2649 return 0;
2650 }
2651
mlx5e_query_mtu(struct mlx5_core_dev * mdev,struct mlx5e_params * params,u16 * mtu)2652 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2653 struct mlx5e_params *params, u16 *mtu)
2654 {
2655 u16 hw_mtu = 0;
2656 int err;
2657
2658 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2659 if (err || !hw_mtu) /* fallback to port oper mtu */
2660 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2661
2662 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
2663 }
2664
mlx5e_set_dev_port_mtu(struct mlx5e_priv * priv)2665 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
2666 {
2667 struct mlx5e_params *params = &priv->channels.params;
2668 struct net_device *netdev = priv->netdev;
2669 struct mlx5_core_dev *mdev = priv->mdev;
2670 u16 mtu;
2671 int err;
2672
2673 err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
2674 if (err)
2675 return err;
2676
2677 mlx5e_query_mtu(mdev, params, &mtu);
2678 if (mtu != params->sw_mtu)
2679 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2680 __func__, mtu, params->sw_mtu);
2681
2682 params->sw_mtu = mtu;
2683 return 0;
2684 }
2685
2686 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu);
2687
mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv * priv)2688 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv)
2689 {
2690 struct mlx5e_params *params = &priv->channels.params;
2691 struct net_device *netdev = priv->netdev;
2692 struct mlx5_core_dev *mdev = priv->mdev;
2693 u16 max_mtu;
2694
2695 /* MTU range: 68 - hw-specific max */
2696 netdev->min_mtu = ETH_MIN_MTU;
2697
2698 mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2699 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu),
2700 ETH_MAX_MTU);
2701 }
2702
mlx5e_netdev_set_tcs(struct net_device * netdev,u16 nch,u8 ntc,struct netdev_tc_txq * tc_to_txq)2703 static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc,
2704 struct netdev_tc_txq *tc_to_txq)
2705 {
2706 int tc, err;
2707
2708 netdev_reset_tc(netdev);
2709
2710 if (ntc == 1)
2711 return 0;
2712
2713 err = netdev_set_num_tc(netdev, ntc);
2714 if (err) {
2715 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc);
2716 return err;
2717 }
2718
2719 for (tc = 0; tc < ntc; tc++) {
2720 u16 count, offset;
2721
2722 count = tc_to_txq[tc].count;
2723 offset = tc_to_txq[tc].offset;
2724 netdev_set_tc_queue(netdev, tc, count, offset);
2725 }
2726
2727 return 0;
2728 }
2729
mlx5e_update_tx_netdev_queues(struct mlx5e_priv * priv)2730 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv)
2731 {
2732 int nch, ntc, num_txqs, err;
2733 int qos_queues = 0;
2734
2735 if (priv->htb)
2736 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb);
2737
2738 nch = priv->channels.params.num_channels;
2739 ntc = mlx5e_get_dcb_num_tc(&priv->channels.params);
2740 num_txqs = nch * ntc + qos_queues;
2741 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS))
2742 num_txqs += ntc;
2743
2744 mlx5e_dbg(DRV, priv, "Setting num_txqs %d\n", num_txqs);
2745 err = netif_set_real_num_tx_queues(priv->netdev, num_txqs);
2746 if (err)
2747 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err);
2748
2749 return err;
2750 }
2751
mlx5e_update_netdev_queues(struct mlx5e_priv * priv)2752 static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
2753 {
2754 struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq;
2755 struct net_device *netdev = priv->netdev;
2756 int old_num_txqs, old_ntc;
2757 int nch, ntc;
2758 int err;
2759 int i;
2760
2761 old_num_txqs = netdev->real_num_tx_queues;
2762 old_ntc = netdev->num_tc ? : 1;
2763 for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++)
2764 old_tc_to_txq[i] = netdev->tc_to_txq[i];
2765
2766 nch = priv->channels.params.num_channels;
2767 ntc = priv->channels.params.mqprio.num_tc;
2768 tc_to_txq = priv->channels.params.mqprio.tc_to_txq;
2769
2770 err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq);
2771 if (err)
2772 goto err_out;
2773 err = mlx5e_update_tx_netdev_queues(priv);
2774 if (err)
2775 goto err_tcs;
2776 err = netif_set_real_num_rx_queues(netdev, nch);
2777 if (err) {
2778 netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err);
2779 goto err_txqs;
2780 }
2781
2782 return 0;
2783
2784 err_txqs:
2785 /* netif_set_real_num_rx_queues could fail only when nch increased. Only
2786 * one of nch and ntc is changed in this function. That means, the call
2787 * to netif_set_real_num_tx_queues below should not fail, because it
2788 * decreases the number of TX queues.
2789 */
2790 WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs));
2791
2792 err_tcs:
2793 WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc,
2794 old_tc_to_txq));
2795 err_out:
2796 return err;
2797 }
2798
2799 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_netdev_queues);
2800
mlx5e_set_default_xps_cpumasks(struct mlx5e_priv * priv,struct mlx5e_params * params)2801 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv,
2802 struct mlx5e_params *params)
2803 {
2804 struct mlx5_core_dev *mdev = priv->mdev;
2805 int num_comp_vectors, ix, irq;
2806
2807 num_comp_vectors = mlx5_comp_vectors_count(mdev);
2808
2809 for (ix = 0; ix < params->num_channels; ix++) {
2810 cpumask_clear(priv->scratchpad.cpumask);
2811
2812 for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) {
2813 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev, irq));
2814
2815 cpumask_set_cpu(cpu, priv->scratchpad.cpumask);
2816 }
2817
2818 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix);
2819 }
2820 }
2821
mlx5e_num_channels_changed(struct mlx5e_priv * priv)2822 static int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
2823 {
2824 u16 count = priv->channels.params.num_channels;
2825 int err;
2826
2827 err = mlx5e_update_netdev_queues(priv);
2828 if (err)
2829 return err;
2830
2831 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params);
2832
2833 /* This function may be called on attach, before priv->rx_res is created. */
2834 if (!netif_is_rxfh_configured(priv->netdev) && priv->rx_res)
2835 mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count);
2836
2837 return 0;
2838 }
2839
2840 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed);
2841
mlx5e_build_txq_maps(struct mlx5e_priv * priv)2842 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
2843 {
2844 int i, ch, tc, num_tc;
2845
2846 ch = priv->channels.num;
2847 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params);
2848
2849 for (i = 0; i < ch; i++) {
2850 for (tc = 0; tc < num_tc; tc++) {
2851 struct mlx5e_channel *c = priv->channels.c[i];
2852 struct mlx5e_txqsq *sq = &c->sq[tc];
2853
2854 priv->txq2sq[sq->txq_ix] = sq;
2855 }
2856 }
2857
2858 if (!priv->channels.ptp)
2859 goto out;
2860
2861 if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state))
2862 goto out;
2863
2864 for (tc = 0; tc < num_tc; tc++) {
2865 struct mlx5e_ptp *c = priv->channels.ptp;
2866 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq;
2867
2868 priv->txq2sq[sq->txq_ix] = sq;
2869 }
2870
2871 out:
2872 /* Make the change to txq2sq visible before the queue is started.
2873 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
2874 * which pairs with this barrier.
2875 */
2876 smp_wmb();
2877 }
2878
mlx5e_activate_priv_channels(struct mlx5e_priv * priv)2879 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
2880 {
2881 mlx5e_build_txq_maps(priv);
2882 mlx5e_activate_channels(&priv->channels);
2883 if (priv->htb)
2884 mlx5e_qos_activate_queues(priv);
2885 mlx5e_xdp_tx_enable(priv);
2886
2887 /* dev_watchdog() wants all TX queues to be started when the carrier is
2888 * OK, including the ones in range real_num_tx_queues..num_tx_queues-1.
2889 * Make it happy to avoid TX timeout false alarms.
2890 */
2891 netif_tx_start_all_queues(priv->netdev);
2892
2893 if (mlx5e_is_vport_rep(priv))
2894 mlx5e_rep_activate_channels(priv);
2895
2896 mlx5e_wait_channels_min_rx_wqes(&priv->channels);
2897
2898 if (priv->rx_res)
2899 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels);
2900 }
2901
mlx5e_deactivate_priv_channels(struct mlx5e_priv * priv)2902 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
2903 {
2904 if (priv->rx_res)
2905 mlx5e_rx_res_channels_deactivate(priv->rx_res);
2906
2907 if (mlx5e_is_vport_rep(priv))
2908 mlx5e_rep_deactivate_channels(priv);
2909
2910 /* The results of ndo_select_queue are unreliable, while netdev config
2911 * is being changed (real_num_tx_queues, num_tc). Stop all queues to
2912 * prevent ndo_start_xmit from being called, so that it can assume that
2913 * the selected queue is always valid.
2914 */
2915 netif_tx_disable(priv->netdev);
2916
2917 mlx5e_xdp_tx_disable(priv);
2918 mlx5e_deactivate_channels(&priv->channels);
2919 }
2920
mlx5e_switch_priv_params(struct mlx5e_priv * priv,struct mlx5e_params * new_params,mlx5e_fp_preactivate preactivate,void * context)2921 static int mlx5e_switch_priv_params(struct mlx5e_priv *priv,
2922 struct mlx5e_params *new_params,
2923 mlx5e_fp_preactivate preactivate,
2924 void *context)
2925 {
2926 struct mlx5e_params old_params;
2927
2928 old_params = priv->channels.params;
2929 priv->channels.params = *new_params;
2930
2931 if (preactivate) {
2932 int err;
2933
2934 err = preactivate(priv, context);
2935 if (err) {
2936 priv->channels.params = old_params;
2937 return err;
2938 }
2939 }
2940
2941 return 0;
2942 }
2943
mlx5e_switch_priv_channels(struct mlx5e_priv * priv,struct mlx5e_channels * new_chs,mlx5e_fp_preactivate preactivate,void * context)2944 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
2945 struct mlx5e_channels *new_chs,
2946 mlx5e_fp_preactivate preactivate,
2947 void *context)
2948 {
2949 struct net_device *netdev = priv->netdev;
2950 struct mlx5e_channels old_chs;
2951 int carrier_ok;
2952 int err = 0;
2953
2954 carrier_ok = netif_carrier_ok(netdev);
2955 netif_carrier_off(netdev);
2956
2957 mlx5e_deactivate_priv_channels(priv);
2958
2959 old_chs = priv->channels;
2960 priv->channels = *new_chs;
2961
2962 /* New channels are ready to roll, call the preactivate hook if needed
2963 * to modify HW settings or update kernel parameters.
2964 */
2965 if (preactivate) {
2966 err = preactivate(priv, context);
2967 if (err) {
2968 priv->channels = old_chs;
2969 goto out;
2970 }
2971 }
2972
2973 mlx5e_close_channels(&old_chs);
2974 priv->profile->update_rx(priv);
2975
2976 mlx5e_selq_apply(&priv->selq);
2977 out:
2978 mlx5e_activate_priv_channels(priv);
2979
2980 /* return carrier back if needed */
2981 if (carrier_ok)
2982 netif_carrier_on(netdev);
2983
2984 return err;
2985 }
2986
mlx5e_safe_switch_params(struct mlx5e_priv * priv,struct mlx5e_params * params,mlx5e_fp_preactivate preactivate,void * context,bool reset)2987 int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
2988 struct mlx5e_params *params,
2989 mlx5e_fp_preactivate preactivate,
2990 void *context, bool reset)
2991 {
2992 struct mlx5e_channels new_chs = {};
2993 int err;
2994
2995 reset &= test_bit(MLX5E_STATE_OPENED, &priv->state);
2996 if (!reset)
2997 return mlx5e_switch_priv_params(priv, params, preactivate, context);
2998
2999 new_chs.params = *params;
3000
3001 mlx5e_selq_prepare_params(&priv->selq, &new_chs.params);
3002
3003 err = mlx5e_open_channels(priv, &new_chs);
3004 if (err)
3005 goto err_cancel_selq;
3006
3007 err = mlx5e_switch_priv_channels(priv, &new_chs, preactivate, context);
3008 if (err)
3009 goto err_close;
3010
3011 return 0;
3012
3013 err_close:
3014 mlx5e_close_channels(&new_chs);
3015
3016 err_cancel_selq:
3017 mlx5e_selq_cancel(&priv->selq);
3018 return err;
3019 }
3020
mlx5e_safe_reopen_channels(struct mlx5e_priv * priv)3021 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv)
3022 {
3023 return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true);
3024 }
3025
mlx5e_timestamp_init(struct mlx5e_priv * priv)3026 void mlx5e_timestamp_init(struct mlx5e_priv *priv)
3027 {
3028 priv->tstamp.tx_type = HWTSTAMP_TX_OFF;
3029 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
3030 }
3031
mlx5e_modify_admin_state(struct mlx5_core_dev * mdev,enum mlx5_port_status state)3032 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev,
3033 enum mlx5_port_status state)
3034 {
3035 struct mlx5_eswitch *esw = mdev->priv.eswitch;
3036 int vport_admin_state;
3037
3038 mlx5_set_port_admin_status(mdev, state);
3039
3040 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS ||
3041 !MLX5_CAP_GEN(mdev, uplink_follow))
3042 return;
3043
3044 if (state == MLX5_PORT_UP)
3045 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO;
3046 else
3047 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3048
3049 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state);
3050 }
3051
mlx5e_open_locked(struct net_device * netdev)3052 int mlx5e_open_locked(struct net_device *netdev)
3053 {
3054 struct mlx5e_priv *priv = netdev_priv(netdev);
3055 int err;
3056
3057 mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params);
3058
3059 set_bit(MLX5E_STATE_OPENED, &priv->state);
3060
3061 err = mlx5e_open_channels(priv, &priv->channels);
3062 if (err)
3063 goto err_clear_state_opened_flag;
3064
3065 priv->profile->update_rx(priv);
3066 mlx5e_selq_apply(&priv->selq);
3067 mlx5e_activate_priv_channels(priv);
3068 mlx5e_apply_traps(priv, true);
3069 if (priv->profile->update_carrier)
3070 priv->profile->update_carrier(priv);
3071
3072 mlx5e_queue_update_stats(priv);
3073 return 0;
3074
3075 err_clear_state_opened_flag:
3076 clear_bit(MLX5E_STATE_OPENED, &priv->state);
3077 mlx5e_selq_cancel(&priv->selq);
3078 return err;
3079 }
3080
mlx5e_open(struct net_device * netdev)3081 int mlx5e_open(struct net_device *netdev)
3082 {
3083 struct mlx5e_priv *priv = netdev_priv(netdev);
3084 int err;
3085
3086 mutex_lock(&priv->state_lock);
3087 err = mlx5e_open_locked(netdev);
3088 if (!err)
3089 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP);
3090 mutex_unlock(&priv->state_lock);
3091
3092 return err;
3093 }
3094
mlx5e_close_locked(struct net_device * netdev)3095 int mlx5e_close_locked(struct net_device *netdev)
3096 {
3097 struct mlx5e_priv *priv = netdev_priv(netdev);
3098
3099 /* May already be CLOSED in case a previous configuration operation
3100 * (e.g RX/TX queue size change) that involves close&open failed.
3101 */
3102 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3103 return 0;
3104
3105 mlx5e_apply_traps(priv, false);
3106 clear_bit(MLX5E_STATE_OPENED, &priv->state);
3107
3108 netif_carrier_off(priv->netdev);
3109 mlx5e_deactivate_priv_channels(priv);
3110 mlx5e_close_channels(&priv->channels);
3111
3112 return 0;
3113 }
3114
mlx5e_close(struct net_device * netdev)3115 int mlx5e_close(struct net_device *netdev)
3116 {
3117 struct mlx5e_priv *priv = netdev_priv(netdev);
3118 int err;
3119
3120 if (!netif_device_present(netdev))
3121 return -ENODEV;
3122
3123 mutex_lock(&priv->state_lock);
3124 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN);
3125 err = mlx5e_close_locked(netdev);
3126 mutex_unlock(&priv->state_lock);
3127
3128 return err;
3129 }
3130
mlx5e_free_drop_rq(struct mlx5e_rq * rq)3131 static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
3132 {
3133 mlx5_wq_destroy(&rq->wq_ctrl);
3134 }
3135
mlx5e_alloc_drop_rq(struct mlx5_core_dev * mdev,struct mlx5e_rq * rq,struct mlx5e_rq_param * param)3136 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3137 struct mlx5e_rq *rq,
3138 struct mlx5e_rq_param *param)
3139 {
3140 void *rqc = param->rqc;
3141 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
3142 int err;
3143
3144 param->wq.db_numa_node = param->wq.buf_numa_node;
3145
3146 err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq,
3147 &rq->wq_ctrl);
3148 if (err)
3149 return err;
3150
3151 /* Mark as unused given "Drop-RQ" packets never reach XDP */
3152 xdp_rxq_info_unused(&rq->xdp_rxq);
3153
3154 rq->mdev = mdev;
3155
3156 return 0;
3157 }
3158
mlx5e_alloc_drop_cq(struct mlx5e_priv * priv,struct mlx5e_cq * cq,struct mlx5e_cq_param * param)3159 static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
3160 struct mlx5e_cq *cq,
3161 struct mlx5e_cq_param *param)
3162 {
3163 struct mlx5_core_dev *mdev = priv->mdev;
3164
3165 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
3166 param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
3167
3168 return mlx5e_alloc_cq_common(priv, param, cq);
3169 }
3170
mlx5e_open_drop_rq(struct mlx5e_priv * priv,struct mlx5e_rq * drop_rq)3171 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
3172 struct mlx5e_rq *drop_rq)
3173 {
3174 struct mlx5_core_dev *mdev = priv->mdev;
3175 struct mlx5e_cq_param cq_param = {};
3176 struct mlx5e_rq_param rq_param = {};
3177 struct mlx5e_cq *cq = &drop_rq->cq;
3178 int err;
3179
3180 mlx5e_build_drop_rq_param(mdev, priv->drop_rq_q_counter, &rq_param);
3181
3182 err = mlx5e_alloc_drop_cq(priv, cq, &cq_param);
3183 if (err)
3184 return err;
3185
3186 err = mlx5e_create_cq(cq, &cq_param);
3187 if (err)
3188 goto err_free_cq;
3189
3190 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
3191 if (err)
3192 goto err_destroy_cq;
3193
3194 err = mlx5e_create_rq(drop_rq, &rq_param);
3195 if (err)
3196 goto err_free_rq;
3197
3198 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3199 if (err)
3200 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3201
3202 return 0;
3203
3204 err_free_rq:
3205 mlx5e_free_drop_rq(drop_rq);
3206
3207 err_destroy_cq:
3208 mlx5e_destroy_cq(cq);
3209
3210 err_free_cq:
3211 mlx5e_free_cq(cq);
3212
3213 return err;
3214 }
3215
mlx5e_close_drop_rq(struct mlx5e_rq * drop_rq)3216 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
3217 {
3218 mlx5e_destroy_rq(drop_rq);
3219 mlx5e_free_drop_rq(drop_rq);
3220 mlx5e_destroy_cq(&drop_rq->cq);
3221 mlx5e_free_cq(&drop_rq->cq);
3222 }
3223
mlx5e_create_tis(struct mlx5_core_dev * mdev,void * in,u32 * tisn)3224 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn)
3225 {
3226 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3227
3228 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn);
3229
3230 if (MLX5_GET(tisc, tisc, tls_en))
3231 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn);
3232
3233 if (mlx5_lag_is_lacp_owner(mdev))
3234 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
3235
3236 return mlx5_core_create_tis(mdev, in, tisn);
3237 }
3238
mlx5e_destroy_tis(struct mlx5_core_dev * mdev,u32 tisn)3239 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
3240 {
3241 mlx5_core_destroy_tis(mdev, tisn);
3242 }
3243
mlx5e_destroy_tises(struct mlx5e_priv * priv)3244 void mlx5e_destroy_tises(struct mlx5e_priv *priv)
3245 {
3246 int tc, i;
3247
3248 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++)
3249 for (tc = 0; tc < priv->profile->max_tc; tc++)
3250 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3251 }
3252
mlx5e_lag_should_assign_affinity(struct mlx5_core_dev * mdev)3253 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev)
3254 {
3255 return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1;
3256 }
3257
mlx5e_create_tises(struct mlx5e_priv * priv)3258 int mlx5e_create_tises(struct mlx5e_priv *priv)
3259 {
3260 int tc, i;
3261 int err;
3262
3263 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) {
3264 for (tc = 0; tc < priv->profile->max_tc; tc++) {
3265 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
3266 void *tisc;
3267
3268 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3269
3270 MLX5_SET(tisc, tisc, prio, tc << 1);
3271
3272 if (mlx5e_lag_should_assign_affinity(priv->mdev))
3273 MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1);
3274
3275 err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]);
3276 if (err)
3277 goto err_close_tises;
3278 }
3279 }
3280
3281 return 0;
3282
3283 err_close_tises:
3284 for (; i >= 0; i--) {
3285 for (tc--; tc >= 0; tc--)
3286 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3287 tc = priv->profile->max_tc;
3288 }
3289
3290 return err;
3291 }
3292
mlx5e_cleanup_nic_tx(struct mlx5e_priv * priv)3293 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
3294 {
3295 if (priv->mqprio_rl) {
3296 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3297 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3298 priv->mqprio_rl = NULL;
3299 }
3300 mlx5e_accel_cleanup_tx(priv);
3301 mlx5e_destroy_tises(priv);
3302 }
3303
mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels * chs,bool enable)3304 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
3305 {
3306 int err = 0;
3307 int i;
3308
3309 for (i = 0; i < chs->num; i++) {
3310 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable);
3311 if (err)
3312 return err;
3313 }
3314
3315 return 0;
3316 }
3317
mlx5e_modify_channels_vsd(struct mlx5e_channels * chs,bool vsd)3318 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
3319 {
3320 int err;
3321 int i;
3322
3323 for (i = 0; i < chs->num; i++) {
3324 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
3325 if (err)
3326 return err;
3327 }
3328 if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state))
3329 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd);
3330
3331 return 0;
3332 }
3333
mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq * tc_to_txq,int ntc,int nch)3334 static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3335 int ntc, int nch)
3336 {
3337 int tc;
3338
3339 memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE);
3340
3341 /* Map netdev TCs to offset 0.
3342 * We have our own UP to TXQ mapping for DCB mode of QoS
3343 */
3344 for (tc = 0; tc < ntc; tc++) {
3345 tc_to_txq[tc] = (struct netdev_tc_txq) {
3346 .count = nch,
3347 .offset = 0,
3348 };
3349 }
3350 }
3351
mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq * tc_to_txq,struct tc_mqprio_qopt * qopt)3352 static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3353 struct tc_mqprio_qopt *qopt)
3354 {
3355 int tc;
3356
3357 for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
3358 tc_to_txq[tc] = (struct netdev_tc_txq) {
3359 .count = qopt->count[tc],
3360 .offset = qopt->offset[tc],
3361 };
3362 }
3363 }
3364
mlx5e_params_mqprio_dcb_set(struct mlx5e_params * params,u8 num_tc)3365 static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc)
3366 {
3367 params->mqprio.mode = TC_MQPRIO_MODE_DCB;
3368 params->mqprio.num_tc = num_tc;
3369 mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc,
3370 params->num_channels);
3371 }
3372
mlx5e_mqprio_rl_update_params(struct mlx5e_params * params,struct mlx5e_mqprio_rl * rl)3373 static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params,
3374 struct mlx5e_mqprio_rl *rl)
3375 {
3376 int tc;
3377
3378 for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
3379 u32 hw_id = 0;
3380
3381 if (rl)
3382 mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id);
3383 params->mqprio.channel.hw_id[tc] = hw_id;
3384 }
3385 }
3386
mlx5e_params_mqprio_channel_set(struct mlx5e_params * params,struct tc_mqprio_qopt_offload * mqprio,struct mlx5e_mqprio_rl * rl)3387 static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params,
3388 struct tc_mqprio_qopt_offload *mqprio,
3389 struct mlx5e_mqprio_rl *rl)
3390 {
3391 int tc;
3392
3393 params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL;
3394 params->mqprio.num_tc = mqprio->qopt.num_tc;
3395
3396 for (tc = 0; tc < TC_MAX_QUEUE; tc++)
3397 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc];
3398
3399 mlx5e_mqprio_rl_update_params(params, rl);
3400 mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt);
3401 }
3402
mlx5e_params_mqprio_reset(struct mlx5e_params * params)3403 static void mlx5e_params_mqprio_reset(struct mlx5e_params *params)
3404 {
3405 mlx5e_params_mqprio_dcb_set(params, 1);
3406 }
3407
mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv * priv,struct tc_mqprio_qopt * mqprio)3408 static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv,
3409 struct tc_mqprio_qopt *mqprio)
3410 {
3411 struct mlx5e_params new_params;
3412 u8 tc = mqprio->num_tc;
3413 int err;
3414
3415 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3416
3417 if (tc && tc != MLX5E_MAX_NUM_TC)
3418 return -EINVAL;
3419
3420 new_params = priv->channels.params;
3421 mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1);
3422
3423 err = mlx5e_safe_switch_params(priv, &new_params,
3424 mlx5e_num_channels_changed_ctx, NULL, true);
3425
3426 if (!err && priv->mqprio_rl) {
3427 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3428 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3429 priv->mqprio_rl = NULL;
3430 }
3431
3432 priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
3433 mlx5e_get_dcb_num_tc(&priv->channels.params));
3434 return err;
3435 }
3436
mlx5e_mqprio_channel_validate(struct mlx5e_priv * priv,struct tc_mqprio_qopt_offload * mqprio)3437 static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv,
3438 struct tc_mqprio_qopt_offload *mqprio)
3439 {
3440 struct net_device *netdev = priv->netdev;
3441 struct mlx5e_ptp *ptp_channel;
3442 int agg_count = 0;
3443 int i;
3444
3445 ptp_channel = priv->channels.ptp;
3446 if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) {
3447 netdev_err(netdev,
3448 "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n");
3449 return -EINVAL;
3450 }
3451
3452 if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 ||
3453 mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC)
3454 return -EINVAL;
3455
3456 for (i = 0; i < mqprio->qopt.num_tc; i++) {
3457 if (!mqprio->qopt.count[i]) {
3458 netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i);
3459 return -EINVAL;
3460 }
3461 if (mqprio->min_rate[i]) {
3462 netdev_err(netdev, "Min tx rate is not supported\n");
3463 return -EINVAL;
3464 }
3465
3466 if (mqprio->max_rate[i]) {
3467 int err;
3468
3469 err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]);
3470 if (err)
3471 return err;
3472 }
3473
3474 if (mqprio->qopt.offset[i] != agg_count) {
3475 netdev_err(netdev, "Discontinuous queues config is not supported\n");
3476 return -EINVAL;
3477 }
3478 agg_count += mqprio->qopt.count[i];
3479 }
3480
3481 if (priv->channels.params.num_channels != agg_count) {
3482 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n",
3483 agg_count, priv->channels.params.num_channels);
3484 return -EINVAL;
3485 }
3486
3487 return 0;
3488 }
3489
mlx5e_mqprio_rate_limit(u8 num_tc,u64 max_rate[])3490 static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[])
3491 {
3492 int tc;
3493
3494 for (tc = 0; tc < num_tc; tc++)
3495 if (max_rate[tc])
3496 return true;
3497 return false;
3498 }
3499
mlx5e_mqprio_rl_create(struct mlx5_core_dev * mdev,u8 num_tc,u64 max_rate[])3500 static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev,
3501 u8 num_tc, u64 max_rate[])
3502 {
3503 struct mlx5e_mqprio_rl *rl;
3504 int err;
3505
3506 if (!mlx5e_mqprio_rate_limit(num_tc, max_rate))
3507 return NULL;
3508
3509 rl = mlx5e_mqprio_rl_alloc();
3510 if (!rl)
3511 return ERR_PTR(-ENOMEM);
3512
3513 err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate);
3514 if (err) {
3515 mlx5e_mqprio_rl_free(rl);
3516 return ERR_PTR(err);
3517 }
3518
3519 return rl;
3520 }
3521
mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv * priv,struct tc_mqprio_qopt_offload * mqprio)3522 static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv,
3523 struct tc_mqprio_qopt_offload *mqprio)
3524 {
3525 mlx5e_fp_preactivate preactivate;
3526 struct mlx5e_params new_params;
3527 struct mlx5e_mqprio_rl *rl;
3528 bool nch_changed;
3529 int err;
3530
3531 err = mlx5e_mqprio_channel_validate(priv, mqprio);
3532 if (err)
3533 return err;
3534
3535 rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate);
3536 if (IS_ERR(rl))
3537 return PTR_ERR(rl);
3538
3539 new_params = priv->channels.params;
3540 mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl);
3541
3542 nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1;
3543 preactivate = nch_changed ? mlx5e_num_channels_changed_ctx :
3544 mlx5e_update_netdev_queues_ctx;
3545 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true);
3546 if (err) {
3547 if (rl) {
3548 mlx5e_mqprio_rl_cleanup(rl);
3549 mlx5e_mqprio_rl_free(rl);
3550 }
3551 return err;
3552 }
3553
3554 if (priv->mqprio_rl) {
3555 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3556 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3557 }
3558 priv->mqprio_rl = rl;
3559
3560 return 0;
3561 }
3562
mlx5e_setup_tc_mqprio(struct mlx5e_priv * priv,struct tc_mqprio_qopt_offload * mqprio)3563 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv,
3564 struct tc_mqprio_qopt_offload *mqprio)
3565 {
3566 /* MQPRIO is another toplevel qdisc that can't be attached
3567 * simultaneously with the offloaded HTB.
3568 */
3569 if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq)))
3570 return -EINVAL;
3571
3572 switch (mqprio->mode) {
3573 case TC_MQPRIO_MODE_DCB:
3574 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt);
3575 case TC_MQPRIO_MODE_CHANNEL:
3576 return mlx5e_setup_tc_mqprio_channel(priv, mqprio);
3577 default:
3578 return -EOPNOTSUPP;
3579 }
3580 }
3581
3582 static LIST_HEAD(mlx5e_block_cb_list);
3583
mlx5e_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)3584 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3585 void *type_data)
3586 {
3587 struct mlx5e_priv *priv = netdev_priv(dev);
3588 bool tc_unbind = false;
3589 int err;
3590
3591 if (type == TC_SETUP_BLOCK &&
3592 ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND)
3593 tc_unbind = true;
3594
3595 if (!netif_device_present(dev) && !tc_unbind)
3596 return -ENODEV;
3597
3598 switch (type) {
3599 case TC_SETUP_BLOCK: {
3600 struct flow_block_offload *f = type_data;
3601
3602 f->unlocked_driver_cb = true;
3603 return flow_block_cb_setup_simple(type_data,
3604 &mlx5e_block_cb_list,
3605 mlx5e_setup_tc_block_cb,
3606 priv, priv, true);
3607 }
3608 case TC_SETUP_QDISC_MQPRIO:
3609 mutex_lock(&priv->state_lock);
3610 err = mlx5e_setup_tc_mqprio(priv, type_data);
3611 mutex_unlock(&priv->state_lock);
3612 return err;
3613 case TC_SETUP_QDISC_HTB:
3614 mutex_lock(&priv->state_lock);
3615 err = mlx5e_htb_setup_tc(priv, type_data);
3616 mutex_unlock(&priv->state_lock);
3617 return err;
3618 default:
3619 return -EOPNOTSUPP;
3620 }
3621 }
3622
mlx5e_fold_sw_stats64(struct mlx5e_priv * priv,struct rtnl_link_stats64 * s)3623 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
3624 {
3625 int i;
3626
3627 for (i = 0; i < priv->stats_nch; i++) {
3628 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i];
3629 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
3630 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
3631 int j;
3632
3633 s->rx_packets += rq_stats->packets + xskrq_stats->packets;
3634 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes;
3635 s->multicast += rq_stats->mcast_packets + xskrq_stats->mcast_packets;
3636
3637 for (j = 0; j < priv->max_opened_tc; j++) {
3638 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
3639
3640 s->tx_packets += sq_stats->packets;
3641 s->tx_bytes += sq_stats->bytes;
3642 s->tx_dropped += sq_stats->dropped;
3643 }
3644 }
3645 if (priv->tx_ptp_opened) {
3646 for (i = 0; i < priv->max_opened_tc; i++) {
3647 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i];
3648
3649 s->tx_packets += sq_stats->packets;
3650 s->tx_bytes += sq_stats->bytes;
3651 s->tx_dropped += sq_stats->dropped;
3652 }
3653 }
3654 if (priv->rx_ptp_opened) {
3655 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq;
3656
3657 s->rx_packets += rq_stats->packets;
3658 s->rx_bytes += rq_stats->bytes;
3659 s->multicast += rq_stats->mcast_packets;
3660 }
3661 }
3662
3663 void
mlx5e_get_stats(struct net_device * dev,struct rtnl_link_stats64 * stats)3664 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3665 {
3666 struct mlx5e_priv *priv = netdev_priv(dev);
3667 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
3668
3669 if (!netif_device_present(dev))
3670 return;
3671
3672 /* In switchdev mode, monitor counters doesn't monitor
3673 * rx/tx stats of 802_3. The update stats mechanism
3674 * should keep the 802_3 layout counters updated
3675 */
3676 if (!mlx5e_monitor_counter_supported(priv) ||
3677 mlx5e_is_uplink_rep(priv)) {
3678 /* update HW stats in background for next time */
3679 mlx5e_queue_update_stats(priv);
3680 }
3681
3682 if (mlx5e_is_uplink_rep(priv)) {
3683 struct mlx5e_vport_stats *vstats = &priv->stats.vport;
3684
3685 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3686 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok);
3687 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3688 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3689
3690 /* vport multicast also counts packets that are dropped due to steering
3691 * or rx out of buffer
3692 */
3693 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
3694 } else {
3695 mlx5e_fold_sw_stats64(priv, stats);
3696 }
3697
3698 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
3699
3700 stats->rx_length_errors =
3701 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3702 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3703 PPORT_802_3_GET(pstats, a_frame_too_long_errors) +
3704 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small);
3705 stats->rx_crc_errors =
3706 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3707 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3708 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
3709 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3710 stats->rx_frame_errors;
3711 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3712 }
3713
mlx5e_nic_set_rx_mode(struct mlx5e_priv * priv)3714 static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv)
3715 {
3716 if (mlx5e_is_uplink_rep(priv))
3717 return; /* no rx mode for uplink rep */
3718
3719 queue_work(priv->wq, &priv->set_rx_mode_work);
3720 }
3721
mlx5e_set_rx_mode(struct net_device * dev)3722 static void mlx5e_set_rx_mode(struct net_device *dev)
3723 {
3724 struct mlx5e_priv *priv = netdev_priv(dev);
3725
3726 mlx5e_nic_set_rx_mode(priv);
3727 }
3728
mlx5e_set_mac(struct net_device * netdev,void * addr)3729 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3730 {
3731 struct mlx5e_priv *priv = netdev_priv(netdev);
3732 struct sockaddr *saddr = addr;
3733
3734 if (!is_valid_ether_addr(saddr->sa_data))
3735 return -EADDRNOTAVAIL;
3736
3737 netif_addr_lock_bh(netdev);
3738 eth_hw_addr_set(netdev, saddr->sa_data);
3739 netif_addr_unlock_bh(netdev);
3740
3741 mlx5e_nic_set_rx_mode(priv);
3742
3743 return 0;
3744 }
3745
3746 #define MLX5E_SET_FEATURE(features, feature, enable) \
3747 do { \
3748 if (enable) \
3749 *features |= feature; \
3750 else \
3751 *features &= ~feature; \
3752 } while (0)
3753
3754 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3755
set_feature_lro(struct net_device * netdev,bool enable)3756 static int set_feature_lro(struct net_device *netdev, bool enable)
3757 {
3758 struct mlx5e_priv *priv = netdev_priv(netdev);
3759 struct mlx5_core_dev *mdev = priv->mdev;
3760 struct mlx5e_params *cur_params;
3761 struct mlx5e_params new_params;
3762 bool reset = true;
3763 int err = 0;
3764
3765 mutex_lock(&priv->state_lock);
3766
3767 cur_params = &priv->channels.params;
3768 new_params = *cur_params;
3769
3770 if (enable)
3771 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO;
3772 else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)
3773 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
3774 else
3775 goto out;
3776
3777 if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO &&
3778 new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) {
3779 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3780 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
3781 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL))
3782 reset = false;
3783 }
3784 }
3785
3786 err = mlx5e_safe_switch_params(priv, &new_params,
3787 mlx5e_modify_tirs_packet_merge_ctx, NULL, reset);
3788 out:
3789 mutex_unlock(&priv->state_lock);
3790 return err;
3791 }
3792
set_feature_hw_gro(struct net_device * netdev,bool enable)3793 static int set_feature_hw_gro(struct net_device *netdev, bool enable)
3794 {
3795 struct mlx5e_priv *priv = netdev_priv(netdev);
3796 struct mlx5e_params new_params;
3797 bool reset = true;
3798 int err = 0;
3799
3800 mutex_lock(&priv->state_lock);
3801 new_params = priv->channels.params;
3802
3803 if (enable) {
3804 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO;
3805 new_params.packet_merge.shampo.match_criteria_type =
3806 MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED;
3807 new_params.packet_merge.shampo.alignment_granularity =
3808 MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE;
3809 } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) {
3810 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
3811 } else {
3812 goto out;
3813 }
3814
3815 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
3816 out:
3817 mutex_unlock(&priv->state_lock);
3818 return err;
3819 }
3820
set_feature_cvlan_filter(struct net_device * netdev,bool enable)3821 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
3822 {
3823 struct mlx5e_priv *priv = netdev_priv(netdev);
3824
3825 if (enable)
3826 mlx5e_enable_cvlan_filter(priv->fs,
3827 !!(priv->netdev->flags & IFF_PROMISC));
3828 else
3829 mlx5e_disable_cvlan_filter(priv->fs,
3830 !!(priv->netdev->flags & IFF_PROMISC));
3831
3832 return 0;
3833 }
3834
set_feature_hw_tc(struct net_device * netdev,bool enable)3835 static int set_feature_hw_tc(struct net_device *netdev, bool enable)
3836 {
3837 struct mlx5e_priv *priv = netdev_priv(netdev);
3838 int err = 0;
3839
3840 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3841 int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) :
3842 MLX5_TC_FLAG(NIC_OFFLOAD);
3843 if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) {
3844 netdev_err(netdev,
3845 "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3846 return -EINVAL;
3847 }
3848 #endif
3849
3850 mutex_lock(&priv->state_lock);
3851 if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) {
3852 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n");
3853 err = -EINVAL;
3854 }
3855 mutex_unlock(&priv->state_lock);
3856
3857 return err;
3858 }
3859
set_feature_rx_all(struct net_device * netdev,bool enable)3860 static int set_feature_rx_all(struct net_device *netdev, bool enable)
3861 {
3862 struct mlx5e_priv *priv = netdev_priv(netdev);
3863 struct mlx5_core_dev *mdev = priv->mdev;
3864
3865 return mlx5_set_port_fcs(mdev, !enable);
3866 }
3867
mlx5e_set_rx_port_ts(struct mlx5_core_dev * mdev,bool enable)3868 static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable)
3869 {
3870 u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {};
3871 bool supported, curr_state;
3872 int err;
3873
3874 if (!MLX5_CAP_GEN(mdev, ports_check))
3875 return 0;
3876
3877 err = mlx5_query_ports_check(mdev, in, sizeof(in));
3878 if (err)
3879 return err;
3880
3881 supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap);
3882 curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc);
3883
3884 if (!supported || enable == curr_state)
3885 return 0;
3886
3887 MLX5_SET(pcmr_reg, in, local_port, 1);
3888 MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable);
3889
3890 return mlx5_set_ports_check(mdev, in, sizeof(in));
3891 }
3892
set_feature_rx_fcs(struct net_device * netdev,bool enable)3893 static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3894 {
3895 struct mlx5e_priv *priv = netdev_priv(netdev);
3896 struct mlx5e_channels *chs = &priv->channels;
3897 struct mlx5_core_dev *mdev = priv->mdev;
3898 int err;
3899
3900 mutex_lock(&priv->state_lock);
3901
3902 if (enable) {
3903 err = mlx5e_set_rx_port_ts(mdev, false);
3904 if (err)
3905 goto out;
3906
3907 chs->params.scatter_fcs_en = true;
3908 err = mlx5e_modify_channels_scatter_fcs(chs, true);
3909 if (err) {
3910 chs->params.scatter_fcs_en = false;
3911 mlx5e_set_rx_port_ts(mdev, true);
3912 }
3913 } else {
3914 chs->params.scatter_fcs_en = false;
3915 err = mlx5e_modify_channels_scatter_fcs(chs, false);
3916 if (err) {
3917 chs->params.scatter_fcs_en = true;
3918 goto out;
3919 }
3920 err = mlx5e_set_rx_port_ts(mdev, true);
3921 if (err) {
3922 mlx5_core_warn(mdev, "Failed to set RX port timestamp %d\n", err);
3923 err = 0;
3924 }
3925 }
3926
3927 out:
3928 mutex_unlock(&priv->state_lock);
3929 return err;
3930 }
3931
set_feature_rx_vlan(struct net_device * netdev,bool enable)3932 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3933 {
3934 struct mlx5e_priv *priv = netdev_priv(netdev);
3935 int err = 0;
3936
3937 mutex_lock(&priv->state_lock);
3938
3939 mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable);
3940 priv->channels.params.vlan_strip_disable = !enable;
3941
3942 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3943 goto unlock;
3944
3945 err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
3946 if (err) {
3947 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable);
3948 priv->channels.params.vlan_strip_disable = enable;
3949 }
3950 unlock:
3951 mutex_unlock(&priv->state_lock);
3952
3953 return err;
3954 }
3955
mlx5e_vlan_rx_add_vid(struct net_device * dev,__be16 proto,u16 vid)3956 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
3957 {
3958 struct mlx5e_priv *priv = netdev_priv(dev);
3959 struct mlx5e_flow_steering *fs = priv->fs;
3960
3961 if (mlx5e_is_uplink_rep(priv))
3962 return 0; /* no vlan table for uplink rep */
3963
3964 return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid);
3965 }
3966
mlx5e_vlan_rx_kill_vid(struct net_device * dev,__be16 proto,u16 vid)3967 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
3968 {
3969 struct mlx5e_priv *priv = netdev_priv(dev);
3970 struct mlx5e_flow_steering *fs = priv->fs;
3971
3972 if (mlx5e_is_uplink_rep(priv))
3973 return 0; /* no vlan table for uplink rep */
3974
3975 return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid);
3976 }
3977
3978 #ifdef CONFIG_MLX5_EN_ARFS
set_feature_arfs(struct net_device * netdev,bool enable)3979 static int set_feature_arfs(struct net_device *netdev, bool enable)
3980 {
3981 struct mlx5e_priv *priv = netdev_priv(netdev);
3982 int err;
3983
3984 if (enable)
3985 err = mlx5e_arfs_enable(priv->fs);
3986 else
3987 err = mlx5e_arfs_disable(priv->fs);
3988
3989 return err;
3990 }
3991 #endif
3992
mlx5e_handle_feature(struct net_device * netdev,netdev_features_t * features,netdev_features_t feature,mlx5e_feature_handler feature_handler)3993 static int mlx5e_handle_feature(struct net_device *netdev,
3994 netdev_features_t *features,
3995 netdev_features_t feature,
3996 mlx5e_feature_handler feature_handler)
3997 {
3998 netdev_features_t changes = *features ^ netdev->features;
3999 bool enable = !!(*features & feature);
4000 int err;
4001
4002 if (!(changes & feature))
4003 return 0;
4004
4005 err = feature_handler(netdev, enable);
4006 if (err) {
4007 MLX5E_SET_FEATURE(features, feature, !enable);
4008 netdev_err(netdev, "%s feature %pNF failed, err %d\n",
4009 enable ? "Enable" : "Disable", &feature, err);
4010 return err;
4011 }
4012
4013 return 0;
4014 }
4015
mlx5e_set_features(struct net_device * netdev,netdev_features_t features)4016 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features)
4017 {
4018 netdev_features_t oper_features = features;
4019 int err = 0;
4020
4021 #define MLX5E_HANDLE_FEATURE(feature, handler) \
4022 mlx5e_handle_feature(netdev, &oper_features, feature, handler)
4023
4024 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
4025 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro);
4026 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
4027 set_feature_cvlan_filter);
4028 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc);
4029 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all);
4030 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
4031 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
4032 #ifdef CONFIG_MLX5_EN_ARFS
4033 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs);
4034 #endif
4035 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx);
4036
4037 if (err) {
4038 netdev->features = oper_features;
4039 return -EINVAL;
4040 }
4041
4042 return 0;
4043 }
4044
mlx5e_fix_uplink_rep_features(struct net_device * netdev,netdev_features_t features)4045 static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev,
4046 netdev_features_t features)
4047 {
4048 features &= ~NETIF_F_HW_TLS_RX;
4049 if (netdev->features & NETIF_F_HW_TLS_RX)
4050 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n");
4051
4052 features &= ~NETIF_F_HW_TLS_TX;
4053 if (netdev->features & NETIF_F_HW_TLS_TX)
4054 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n");
4055
4056 features &= ~NETIF_F_NTUPLE;
4057 if (netdev->features & NETIF_F_NTUPLE)
4058 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n");
4059
4060 features &= ~NETIF_F_GRO_HW;
4061 if (netdev->features & NETIF_F_GRO_HW)
4062 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n");
4063
4064 return features;
4065 }
4066
mlx5e_fix_features(struct net_device * netdev,netdev_features_t features)4067 static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
4068 netdev_features_t features)
4069 {
4070 struct mlx5e_priv *priv = netdev_priv(netdev);
4071 struct mlx5e_vlan_table *vlan;
4072 struct mlx5e_params *params;
4073
4074 if (!netif_device_present(netdev))
4075 return features;
4076
4077 vlan = mlx5e_fs_get_vlan(priv->fs);
4078 mutex_lock(&priv->state_lock);
4079 params = &priv->channels.params;
4080 if (!vlan ||
4081 !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) {
4082 /* HW strips the outer C-tag header, this is a problem
4083 * for S-tag traffic.
4084 */
4085 features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4086 if (!params->vlan_strip_disable)
4087 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
4088 }
4089
4090 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
4091 if (features & NETIF_F_LRO) {
4092 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n");
4093 features &= ~NETIF_F_LRO;
4094 }
4095 if (features & NETIF_F_GRO_HW) {
4096 netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n");
4097 features &= ~NETIF_F_GRO_HW;
4098 }
4099 }
4100
4101 if (params->xdp_prog) {
4102 if (features & NETIF_F_LRO) {
4103 netdev_warn(netdev, "LRO is incompatible with XDP\n");
4104 features &= ~NETIF_F_LRO;
4105 }
4106 if (features & NETIF_F_GRO_HW) {
4107 netdev_warn(netdev, "HW GRO is incompatible with XDP\n");
4108 features &= ~NETIF_F_GRO_HW;
4109 }
4110 }
4111
4112 if (priv->xsk.refcnt) {
4113 if (features & NETIF_F_LRO) {
4114 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n",
4115 priv->xsk.refcnt);
4116 features &= ~NETIF_F_LRO;
4117 }
4118 if (features & NETIF_F_GRO_HW) {
4119 netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n",
4120 priv->xsk.refcnt);
4121 features &= ~NETIF_F_GRO_HW;
4122 }
4123 }
4124
4125 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
4126 features &= ~NETIF_F_RXHASH;
4127 if (netdev->features & NETIF_F_RXHASH)
4128 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n");
4129
4130 if (features & NETIF_F_GRO_HW) {
4131 netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n");
4132 features &= ~NETIF_F_GRO_HW;
4133 }
4134 }
4135
4136 if (mlx5e_is_uplink_rep(priv))
4137 features = mlx5e_fix_uplink_rep_features(netdev, features);
4138
4139 mutex_unlock(&priv->state_lock);
4140
4141 return features;
4142 }
4143
mlx5e_xsk_validate_mtu(struct net_device * netdev,struct mlx5e_channels * chs,struct mlx5e_params * new_params,struct mlx5_core_dev * mdev)4144 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
4145 struct mlx5e_channels *chs,
4146 struct mlx5e_params *new_params,
4147 struct mlx5_core_dev *mdev)
4148 {
4149 u16 ix;
4150
4151 for (ix = 0; ix < chs->params.num_channels; ix++) {
4152 struct xsk_buff_pool *xsk_pool =
4153 mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
4154 struct mlx5e_xsk_param xsk;
4155
4156 if (!xsk_pool)
4157 continue;
4158
4159 mlx5e_build_xsk_param(xsk_pool, &xsk);
4160
4161 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) {
4162 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
4163 int max_mtu_frame, max_mtu_page, max_mtu;
4164
4165 /* Two criteria must be met:
4166 * 1. HW MTU + all headrooms <= XSK frame size.
4167 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
4168 */
4169 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr);
4170 max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0));
4171 max_mtu = min(max_mtu_frame, max_mtu_page);
4172
4173 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u. Try MTU <= %d\n",
4174 new_params->sw_mtu, ix, max_mtu);
4175 return false;
4176 }
4177 }
4178
4179 return true;
4180 }
4181
mlx5e_params_validate_xdp(struct net_device * netdev,struct mlx5_core_dev * mdev,struct mlx5e_params * params)4182 static bool mlx5e_params_validate_xdp(struct net_device *netdev,
4183 struct mlx5_core_dev *mdev,
4184 struct mlx5e_params *params)
4185 {
4186 bool is_linear;
4187
4188 /* No XSK params: AF_XDP can't be enabled yet at the point of setting
4189 * the XDP program.
4190 */
4191 is_linear = mlx5e_rx_is_linear_skb(mdev, params, NULL);
4192
4193 if (!is_linear && params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {
4194 netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n",
4195 params->sw_mtu,
4196 mlx5e_xdp_max_mtu(params, NULL));
4197 return false;
4198 }
4199 if (!is_linear && !params->xdp_prog->aux->xdp_has_frags) {
4200 netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n",
4201 params->sw_mtu,
4202 mlx5e_xdp_max_mtu(params, NULL));
4203 return false;
4204 }
4205
4206 return true;
4207 }
4208
mlx5e_change_mtu(struct net_device * netdev,int new_mtu,mlx5e_fp_preactivate preactivate)4209 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
4210 mlx5e_fp_preactivate preactivate)
4211 {
4212 struct mlx5e_priv *priv = netdev_priv(netdev);
4213 struct mlx5e_params new_params;
4214 struct mlx5e_params *params;
4215 bool reset = true;
4216 int err = 0;
4217
4218 mutex_lock(&priv->state_lock);
4219
4220 params = &priv->channels.params;
4221
4222 new_params = *params;
4223 new_params.sw_mtu = new_mtu;
4224 err = mlx5e_validate_params(priv->mdev, &new_params);
4225 if (err)
4226 goto out;
4227
4228 if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev,
4229 &new_params)) {
4230 err = -EINVAL;
4231 goto out;
4232 }
4233
4234 if (priv->xsk.refcnt &&
4235 !mlx5e_xsk_validate_mtu(netdev, &priv->channels,
4236 &new_params, priv->mdev)) {
4237 err = -EINVAL;
4238 goto out;
4239 }
4240
4241 if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO)
4242 reset = false;
4243
4244 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
4245 params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) {
4246 bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL);
4247 bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev,
4248 &new_params, NULL);
4249 u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL);
4250 u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL);
4251
4252 /* Always reset in linear mode - hw_mtu is used in data path.
4253 * Check that the mode was non-linear and didn't change.
4254 * If XSK is active, XSK RQs are linear.
4255 * Reset if the RQ size changed, even if it's non-linear.
4256 */
4257 if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt &&
4258 sz_old == sz_new)
4259 reset = false;
4260 }
4261
4262 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, reset);
4263
4264 out:
4265 netdev->mtu = params->sw_mtu;
4266 mutex_unlock(&priv->state_lock);
4267 return err;
4268 }
4269
mlx5e_change_nic_mtu(struct net_device * netdev,int new_mtu)4270 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
4271 {
4272 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx);
4273 }
4274
mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv * priv,void * ctx)4275 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx)
4276 {
4277 bool set = *(bool *)ctx;
4278
4279 return mlx5e_ptp_rx_manage_fs(priv, set);
4280 }
4281
mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv * priv,bool rx_filter)4282 static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter)
4283 {
4284 bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
4285 int err;
4286
4287 if (!rx_filter)
4288 /* Reset CQE compression to Admin default */
4289 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false);
4290
4291 if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS))
4292 return 0;
4293
4294 /* Disable CQE compression */
4295 netdev_warn(priv->netdev, "Disabling RX cqe compression\n");
4296 err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true);
4297 if (err)
4298 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
4299
4300 return err;
4301 }
4302
mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv * priv,bool ptp_rx)4303 static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx)
4304 {
4305 struct mlx5e_params new_params;
4306
4307 if (ptp_rx == priv->channels.params.ptp_rx)
4308 return 0;
4309
4310 new_params = priv->channels.params;
4311 new_params.ptp_rx = ptp_rx;
4312 return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx,
4313 &new_params.ptp_rx, true);
4314 }
4315
mlx5e_hwstamp_set(struct mlx5e_priv * priv,struct ifreq * ifr)4316 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
4317 {
4318 struct hwtstamp_config config;
4319 bool rx_cqe_compress_def;
4320 bool ptp_rx;
4321 int err;
4322
4323 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) ||
4324 (mlx5_clock_get_ptp_index(priv->mdev) == -1))
4325 return -EOPNOTSUPP;
4326
4327 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
4328 return -EFAULT;
4329
4330 /* TX HW timestamp */
4331 switch (config.tx_type) {
4332 case HWTSTAMP_TX_OFF:
4333 case HWTSTAMP_TX_ON:
4334 break;
4335 default:
4336 return -ERANGE;
4337 }
4338
4339 mutex_lock(&priv->state_lock);
4340 rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
4341
4342 /* RX HW timestamp */
4343 switch (config.rx_filter) {
4344 case HWTSTAMP_FILTER_NONE:
4345 ptp_rx = false;
4346 break;
4347 case HWTSTAMP_FILTER_ALL:
4348 case HWTSTAMP_FILTER_SOME:
4349 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
4350 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
4351 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
4352 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
4353 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
4354 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
4355 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
4356 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
4357 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
4358 case HWTSTAMP_FILTER_PTP_V2_EVENT:
4359 case HWTSTAMP_FILTER_PTP_V2_SYNC:
4360 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
4361 case HWTSTAMP_FILTER_NTP_ALL:
4362 config.rx_filter = HWTSTAMP_FILTER_ALL;
4363 /* ptp_rx is set if both HW TS is set and CQE
4364 * compression is set
4365 */
4366 ptp_rx = rx_cqe_compress_def;
4367 break;
4368 default:
4369 err = -ERANGE;
4370 goto err_unlock;
4371 }
4372
4373 if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX))
4374 err = mlx5e_hwstamp_config_no_ptp_rx(priv,
4375 config.rx_filter != HWTSTAMP_FILTER_NONE);
4376 else
4377 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx);
4378 if (err)
4379 goto err_unlock;
4380
4381 memcpy(&priv->tstamp, &config, sizeof(config));
4382 mutex_unlock(&priv->state_lock);
4383
4384 /* might need to fix some features */
4385 netdev_update_features(priv->netdev);
4386
4387 return copy_to_user(ifr->ifr_data, &config,
4388 sizeof(config)) ? -EFAULT : 0;
4389 err_unlock:
4390 mutex_unlock(&priv->state_lock);
4391 return err;
4392 }
4393
mlx5e_hwstamp_get(struct mlx5e_priv * priv,struct ifreq * ifr)4394 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
4395 {
4396 struct hwtstamp_config *cfg = &priv->tstamp;
4397
4398 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
4399 return -EOPNOTSUPP;
4400
4401 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
4402 }
4403
mlx5e_ioctl(struct net_device * dev,struct ifreq * ifr,int cmd)4404 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
4405 {
4406 struct mlx5e_priv *priv = netdev_priv(dev);
4407
4408 switch (cmd) {
4409 case SIOCSHWTSTAMP:
4410 return mlx5e_hwstamp_set(priv, ifr);
4411 case SIOCGHWTSTAMP:
4412 return mlx5e_hwstamp_get(priv, ifr);
4413 default:
4414 return -EOPNOTSUPP;
4415 }
4416 }
4417
4418 #ifdef CONFIG_MLX5_ESWITCH
mlx5e_set_vf_mac(struct net_device * dev,int vf,u8 * mac)4419 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
4420 {
4421 struct mlx5e_priv *priv = netdev_priv(dev);
4422 struct mlx5_core_dev *mdev = priv->mdev;
4423
4424 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
4425 }
4426
mlx5e_set_vf_vlan(struct net_device * dev,int vf,u16 vlan,u8 qos,__be16 vlan_proto)4427 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
4428 __be16 vlan_proto)
4429 {
4430 struct mlx5e_priv *priv = netdev_priv(dev);
4431 struct mlx5_core_dev *mdev = priv->mdev;
4432
4433 if (vlan_proto != htons(ETH_P_8021Q))
4434 return -EPROTONOSUPPORT;
4435
4436 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
4437 vlan, qos);
4438 }
4439
mlx5e_set_vf_spoofchk(struct net_device * dev,int vf,bool setting)4440 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
4441 {
4442 struct mlx5e_priv *priv = netdev_priv(dev);
4443 struct mlx5_core_dev *mdev = priv->mdev;
4444
4445 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
4446 }
4447
mlx5e_set_vf_trust(struct net_device * dev,int vf,bool setting)4448 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
4449 {
4450 struct mlx5e_priv *priv = netdev_priv(dev);
4451 struct mlx5_core_dev *mdev = priv->mdev;
4452
4453 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
4454 }
4455
mlx5e_set_vf_rate(struct net_device * dev,int vf,int min_tx_rate,int max_tx_rate)4456 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
4457 int max_tx_rate)
4458 {
4459 struct mlx5e_priv *priv = netdev_priv(dev);
4460 struct mlx5_core_dev *mdev = priv->mdev;
4461
4462 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
4463 max_tx_rate, min_tx_rate);
4464 }
4465
mlx5_vport_link2ifla(u8 esw_link)4466 static int mlx5_vport_link2ifla(u8 esw_link)
4467 {
4468 switch (esw_link) {
4469 case MLX5_VPORT_ADMIN_STATE_DOWN:
4470 return IFLA_VF_LINK_STATE_DISABLE;
4471 case MLX5_VPORT_ADMIN_STATE_UP:
4472 return IFLA_VF_LINK_STATE_ENABLE;
4473 }
4474 return IFLA_VF_LINK_STATE_AUTO;
4475 }
4476
mlx5_ifla_link2vport(u8 ifla_link)4477 static int mlx5_ifla_link2vport(u8 ifla_link)
4478 {
4479 switch (ifla_link) {
4480 case IFLA_VF_LINK_STATE_DISABLE:
4481 return MLX5_VPORT_ADMIN_STATE_DOWN;
4482 case IFLA_VF_LINK_STATE_ENABLE:
4483 return MLX5_VPORT_ADMIN_STATE_UP;
4484 }
4485 return MLX5_VPORT_ADMIN_STATE_AUTO;
4486 }
4487
mlx5e_set_vf_link_state(struct net_device * dev,int vf,int link_state)4488 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
4489 int link_state)
4490 {
4491 struct mlx5e_priv *priv = netdev_priv(dev);
4492 struct mlx5_core_dev *mdev = priv->mdev;
4493
4494 if (mlx5e_is_uplink_rep(priv))
4495 return -EOPNOTSUPP;
4496
4497 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
4498 mlx5_ifla_link2vport(link_state));
4499 }
4500
mlx5e_get_vf_config(struct net_device * dev,int vf,struct ifla_vf_info * ivi)4501 int mlx5e_get_vf_config(struct net_device *dev,
4502 int vf, struct ifla_vf_info *ivi)
4503 {
4504 struct mlx5e_priv *priv = netdev_priv(dev);
4505 struct mlx5_core_dev *mdev = priv->mdev;
4506 int err;
4507
4508 if (!netif_device_present(dev))
4509 return -EOPNOTSUPP;
4510
4511 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
4512 if (err)
4513 return err;
4514 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
4515 return 0;
4516 }
4517
mlx5e_get_vf_stats(struct net_device * dev,int vf,struct ifla_vf_stats * vf_stats)4518 int mlx5e_get_vf_stats(struct net_device *dev,
4519 int vf, struct ifla_vf_stats *vf_stats)
4520 {
4521 struct mlx5e_priv *priv = netdev_priv(dev);
4522 struct mlx5_core_dev *mdev = priv->mdev;
4523
4524 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
4525 vf_stats);
4526 }
4527
4528 static bool
mlx5e_has_offload_stats(const struct net_device * dev,int attr_id)4529 mlx5e_has_offload_stats(const struct net_device *dev, int attr_id)
4530 {
4531 struct mlx5e_priv *priv = netdev_priv(dev);
4532
4533 if (!netif_device_present(dev))
4534 return false;
4535
4536 if (!mlx5e_is_uplink_rep(priv))
4537 return false;
4538
4539 return mlx5e_rep_has_offload_stats(dev, attr_id);
4540 }
4541
4542 static int
mlx5e_get_offload_stats(int attr_id,const struct net_device * dev,void * sp)4543 mlx5e_get_offload_stats(int attr_id, const struct net_device *dev,
4544 void *sp)
4545 {
4546 struct mlx5e_priv *priv = netdev_priv(dev);
4547
4548 if (!mlx5e_is_uplink_rep(priv))
4549 return -EOPNOTSUPP;
4550
4551 return mlx5e_rep_get_offload_stats(attr_id, dev, sp);
4552 }
4553 #endif
4554
mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev * mdev,u8 proto_type)4555 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type)
4556 {
4557 switch (proto_type) {
4558 case IPPROTO_GRE:
4559 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
4560 case IPPROTO_IPIP:
4561 case IPPROTO_IPV6:
4562 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
4563 MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx));
4564 default:
4565 return false;
4566 }
4567 }
4568
mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev * mdev,struct sk_buff * skb)4569 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev,
4570 struct sk_buff *skb)
4571 {
4572 switch (skb->inner_protocol) {
4573 case htons(ETH_P_IP):
4574 case htons(ETH_P_IPV6):
4575 case htons(ETH_P_TEB):
4576 return true;
4577 case htons(ETH_P_MPLS_UC):
4578 case htons(ETH_P_MPLS_MC):
4579 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre);
4580 }
4581 return false;
4582 }
4583
mlx5e_tunnel_features_check(struct mlx5e_priv * priv,struct sk_buff * skb,netdev_features_t features)4584 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
4585 struct sk_buff *skb,
4586 netdev_features_t features)
4587 {
4588 unsigned int offset = 0;
4589 struct udphdr *udph;
4590 u8 proto;
4591 u16 port;
4592
4593 switch (vlan_get_protocol(skb)) {
4594 case htons(ETH_P_IP):
4595 proto = ip_hdr(skb)->protocol;
4596 break;
4597 case htons(ETH_P_IPV6):
4598 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL);
4599 break;
4600 default:
4601 goto out;
4602 }
4603
4604 switch (proto) {
4605 case IPPROTO_GRE:
4606 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb))
4607 return features;
4608 break;
4609 case IPPROTO_IPIP:
4610 case IPPROTO_IPV6:
4611 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP))
4612 return features;
4613 break;
4614 case IPPROTO_UDP:
4615 udph = udp_hdr(skb);
4616 port = be16_to_cpu(udph->dest);
4617
4618 /* Verify if UDP port is being offloaded by HW */
4619 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port))
4620 return features;
4621
4622 #if IS_ENABLED(CONFIG_GENEVE)
4623 /* Support Geneve offload for default UDP port */
4624 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev))
4625 return features;
4626 #endif
4627 break;
4628 #ifdef CONFIG_MLX5_EN_IPSEC
4629 case IPPROTO_ESP:
4630 return mlx5e_ipsec_feature_check(skb, features);
4631 #endif
4632 }
4633
4634 out:
4635 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
4636 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4637 }
4638
mlx5e_features_check(struct sk_buff * skb,struct net_device * netdev,netdev_features_t features)4639 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
4640 struct net_device *netdev,
4641 netdev_features_t features)
4642 {
4643 struct mlx5e_priv *priv = netdev_priv(netdev);
4644
4645 features = vlan_features_check(skb, features);
4646 features = vxlan_features_check(skb, features);
4647
4648 /* Validate if the tunneled packet is being offloaded by HW */
4649 if (skb->encapsulation &&
4650 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
4651 return mlx5e_tunnel_features_check(priv, skb, features);
4652
4653 return features;
4654 }
4655
mlx5e_tx_timeout_work(struct work_struct * work)4656 static void mlx5e_tx_timeout_work(struct work_struct *work)
4657 {
4658 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
4659 tx_timeout_work);
4660 struct net_device *netdev = priv->netdev;
4661 int i;
4662
4663 rtnl_lock();
4664 mutex_lock(&priv->state_lock);
4665
4666 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4667 goto unlock;
4668
4669 for (i = 0; i < netdev->real_num_tx_queues; i++) {
4670 struct netdev_queue *dev_queue =
4671 netdev_get_tx_queue(netdev, i);
4672 struct mlx5e_txqsq *sq = priv->txq2sq[i];
4673
4674 if (!netif_xmit_stopped(dev_queue))
4675 continue;
4676
4677 if (mlx5e_reporter_tx_timeout(sq))
4678 /* break if tried to reopened channels */
4679 break;
4680 }
4681
4682 unlock:
4683 mutex_unlock(&priv->state_lock);
4684 rtnl_unlock();
4685 }
4686
mlx5e_tx_timeout(struct net_device * dev,unsigned int txqueue)4687 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue)
4688 {
4689 struct mlx5e_priv *priv = netdev_priv(dev);
4690
4691 netdev_err(dev, "TX timeout detected\n");
4692 queue_work(priv->wq, &priv->tx_timeout_work);
4693 }
4694
mlx5e_xdp_allowed(struct mlx5e_priv * priv,struct bpf_prog * prog)4695 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
4696 {
4697 struct net_device *netdev = priv->netdev;
4698 struct mlx5e_params new_params;
4699
4700 if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
4701 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n");
4702 return -EINVAL;
4703 }
4704
4705 new_params = priv->channels.params;
4706 new_params.xdp_prog = prog;
4707
4708 if (!mlx5e_params_validate_xdp(netdev, priv->mdev, &new_params))
4709 return -EINVAL;
4710
4711 return 0;
4712 }
4713
mlx5e_rq_replace_xdp_prog(struct mlx5e_rq * rq,struct bpf_prog * prog)4714 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog)
4715 {
4716 struct bpf_prog *old_prog;
4717
4718 old_prog = rcu_replace_pointer(rq->xdp_prog, prog,
4719 lockdep_is_held(&rq->priv->state_lock));
4720 if (old_prog)
4721 bpf_prog_put(old_prog);
4722 }
4723
mlx5e_xdp_set(struct net_device * netdev,struct bpf_prog * prog)4724 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
4725 {
4726 struct mlx5e_priv *priv = netdev_priv(netdev);
4727 struct mlx5e_params new_params;
4728 struct bpf_prog *old_prog;
4729 int err = 0;
4730 bool reset;
4731 int i;
4732
4733 mutex_lock(&priv->state_lock);
4734
4735 if (prog) {
4736 err = mlx5e_xdp_allowed(priv, prog);
4737 if (err)
4738 goto unlock;
4739 }
4740
4741 /* no need for full reset when exchanging programs */
4742 reset = (!priv->channels.params.xdp_prog || !prog);
4743
4744 new_params = priv->channels.params;
4745 new_params.xdp_prog = prog;
4746
4747 /* XDP affects striding RQ parameters. Block XDP if striding RQ won't be
4748 * supported with the new parameters: if PAGE_SIZE is bigger than
4749 * MLX5_MPWQE_LOG_STRIDE_SZ_MAX, striding RQ can't be used, even though
4750 * the MTU is small enough for the linear mode, because XDP uses strides
4751 * of PAGE_SIZE on regular RQs.
4752 */
4753 if (reset && MLX5E_GET_PFLAG(&new_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
4754 /* Checking for regular RQs here; XSK RQs were checked on XSK bind. */
4755 err = mlx5e_mpwrq_validate_regular(priv->mdev, &new_params);
4756 if (err)
4757 goto unlock;
4758 }
4759
4760 old_prog = priv->channels.params.xdp_prog;
4761
4762 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
4763 if (err)
4764 goto unlock;
4765
4766 if (old_prog)
4767 bpf_prog_put(old_prog);
4768
4769 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
4770 goto unlock;
4771
4772 /* exchanging programs w/o reset, we update ref counts on behalf
4773 * of the channels RQs here.
4774 */
4775 bpf_prog_add(prog, priv->channels.num);
4776 for (i = 0; i < priv->channels.num; i++) {
4777 struct mlx5e_channel *c = priv->channels.c[i];
4778
4779 mlx5e_rq_replace_xdp_prog(&c->rq, prog);
4780 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) {
4781 bpf_prog_inc(prog);
4782 mlx5e_rq_replace_xdp_prog(&c->xskrq, prog);
4783 }
4784 }
4785
4786 unlock:
4787 mutex_unlock(&priv->state_lock);
4788
4789 /* Need to fix some features. */
4790 if (!err)
4791 netdev_update_features(netdev);
4792
4793 return err;
4794 }
4795
mlx5e_xdp(struct net_device * dev,struct netdev_bpf * xdp)4796 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4797 {
4798 switch (xdp->command) {
4799 case XDP_SETUP_PROG:
4800 return mlx5e_xdp_set(dev, xdp->prog);
4801 case XDP_SETUP_XSK_POOL:
4802 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool,
4803 xdp->xsk.queue_id);
4804 default:
4805 return -EINVAL;
4806 }
4807 }
4808
4809 #ifdef CONFIG_MLX5_ESWITCH
mlx5e_bridge_getlink(struct sk_buff * skb,u32 pid,u32 seq,struct net_device * dev,u32 filter_mask,int nlflags)4810 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4811 struct net_device *dev, u32 filter_mask,
4812 int nlflags)
4813 {
4814 struct mlx5e_priv *priv = netdev_priv(dev);
4815 struct mlx5_core_dev *mdev = priv->mdev;
4816 u8 mode, setting;
4817 int err;
4818
4819 err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting);
4820 if (err)
4821 return err;
4822 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB;
4823 return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4824 mode,
4825 0, 0, nlflags, filter_mask, NULL);
4826 }
4827
mlx5e_bridge_setlink(struct net_device * dev,struct nlmsghdr * nlh,u16 flags,struct netlink_ext_ack * extack)4828 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4829 u16 flags, struct netlink_ext_ack *extack)
4830 {
4831 struct mlx5e_priv *priv = netdev_priv(dev);
4832 struct mlx5_core_dev *mdev = priv->mdev;
4833 struct nlattr *attr, *br_spec;
4834 u16 mode = BRIDGE_MODE_UNDEF;
4835 u8 setting;
4836 int rem;
4837
4838 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4839 if (!br_spec)
4840 return -EINVAL;
4841
4842 nla_for_each_nested(attr, br_spec, rem) {
4843 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4844 continue;
4845
4846 if (nla_len(attr) < sizeof(mode))
4847 return -EINVAL;
4848
4849 mode = nla_get_u16(attr);
4850 if (mode > BRIDGE_MODE_VEPA)
4851 return -EINVAL;
4852
4853 break;
4854 }
4855
4856 if (mode == BRIDGE_MODE_UNDEF)
4857 return -EINVAL;
4858
4859 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0;
4860 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting);
4861 }
4862 #endif
4863
4864 const struct net_device_ops mlx5e_netdev_ops = {
4865 .ndo_open = mlx5e_open,
4866 .ndo_stop = mlx5e_close,
4867 .ndo_start_xmit = mlx5e_xmit,
4868 .ndo_setup_tc = mlx5e_setup_tc,
4869 .ndo_select_queue = mlx5e_select_queue,
4870 .ndo_get_stats64 = mlx5e_get_stats,
4871 .ndo_set_rx_mode = mlx5e_set_rx_mode,
4872 .ndo_set_mac_address = mlx5e_set_mac,
4873 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid,
4874 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid,
4875 .ndo_set_features = mlx5e_set_features,
4876 .ndo_fix_features = mlx5e_fix_features,
4877 .ndo_change_mtu = mlx5e_change_nic_mtu,
4878 .ndo_eth_ioctl = mlx5e_ioctl,
4879 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate,
4880 .ndo_features_check = mlx5e_features_check,
4881 .ndo_tx_timeout = mlx5e_tx_timeout,
4882 .ndo_bpf = mlx5e_xdp,
4883 .ndo_xdp_xmit = mlx5e_xdp_xmit,
4884 .ndo_xsk_wakeup = mlx5e_xsk_wakeup,
4885 #ifdef CONFIG_MLX5_EN_ARFS
4886 .ndo_rx_flow_steer = mlx5e_rx_flow_steer,
4887 #endif
4888 #ifdef CONFIG_MLX5_ESWITCH
4889 .ndo_bridge_setlink = mlx5e_bridge_setlink,
4890 .ndo_bridge_getlink = mlx5e_bridge_getlink,
4891
4892 /* SRIOV E-Switch NDOs */
4893 .ndo_set_vf_mac = mlx5e_set_vf_mac,
4894 .ndo_set_vf_vlan = mlx5e_set_vf_vlan,
4895 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk,
4896 .ndo_set_vf_trust = mlx5e_set_vf_trust,
4897 .ndo_set_vf_rate = mlx5e_set_vf_rate,
4898 .ndo_get_vf_config = mlx5e_get_vf_config,
4899 .ndo_set_vf_link_state = mlx5e_set_vf_link_state,
4900 .ndo_get_vf_stats = mlx5e_get_vf_stats,
4901 .ndo_has_offload_stats = mlx5e_has_offload_stats,
4902 .ndo_get_offload_stats = mlx5e_get_offload_stats,
4903 #endif
4904 .ndo_get_devlink_port = mlx5e_get_devlink_port,
4905 };
4906
mlx5e_choose_lro_timeout(struct mlx5_core_dev * mdev,u32 wanted_timeout)4907 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
4908 {
4909 int i;
4910
4911 /* The supported periods are organized in ascending order */
4912 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
4913 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
4914 break;
4915
4916 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
4917 }
4918
mlx5e_build_nic_params(struct mlx5e_priv * priv,struct mlx5e_xsk * xsk,u16 mtu)4919 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu)
4920 {
4921 struct mlx5e_params *params = &priv->channels.params;
4922 struct mlx5_core_dev *mdev = priv->mdev;
4923 u8 rx_cq_period_mode;
4924
4925 params->sw_mtu = mtu;
4926 params->hard_mtu = MLX5E_ETH_HARD_MTU;
4927 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2,
4928 priv->max_nch);
4929 mlx5e_params_mqprio_reset(params);
4930
4931 /* SQ */
4932 params->log_sq_size = is_kdump_kernel() ?
4933 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
4934 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
4935 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
4936
4937 /* XDP SQ */
4938 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
4939
4940 /* set CQE compression */
4941 params->rx_cqe_compress_def = false;
4942 if (MLX5_CAP_GEN(mdev, cqe_compression) &&
4943 MLX5_CAP_GEN(mdev, vport_group_manager))
4944 params->rx_cqe_compress_def = slow_pci_heuristic(mdev);
4945
4946 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
4947 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false);
4948
4949 /* RQ */
4950 mlx5e_build_rq_params(mdev, params);
4951
4952 params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
4953
4954 /* CQ moderation params */
4955 rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
4956 MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
4957 MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
4958 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4959 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4960 mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode);
4961 mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
4962
4963 /* TX inline */
4964 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode);
4965
4966 params->tunneled_offload_en = mlx5_tunnel_inner_ft_supported(mdev);
4967
4968 /* AF_XDP */
4969 params->xsk = xsk;
4970
4971 /* Do not update netdev->features directly in here
4972 * on mlx5e_attach_netdev() we will call mlx5e_update_features()
4973 * To update netdev->features please modify mlx5e_fix_features()
4974 */
4975 }
4976
mlx5e_set_netdev_dev_addr(struct net_device * netdev)4977 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
4978 {
4979 struct mlx5e_priv *priv = netdev_priv(netdev);
4980 u8 addr[ETH_ALEN];
4981
4982 mlx5_query_mac_address(priv->mdev, addr);
4983 if (is_zero_ether_addr(addr) &&
4984 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
4985 eth_hw_addr_random(netdev);
4986 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
4987 return;
4988 }
4989
4990 eth_hw_addr_set(netdev, addr);
4991 }
4992
mlx5e_vxlan_set_port(struct net_device * netdev,unsigned int table,unsigned int entry,struct udp_tunnel_info * ti)4993 static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table,
4994 unsigned int entry, struct udp_tunnel_info *ti)
4995 {
4996 struct mlx5e_priv *priv = netdev_priv(netdev);
4997
4998 return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port));
4999 }
5000
mlx5e_vxlan_unset_port(struct net_device * netdev,unsigned int table,unsigned int entry,struct udp_tunnel_info * ti)5001 static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table,
5002 unsigned int entry, struct udp_tunnel_info *ti)
5003 {
5004 struct mlx5e_priv *priv = netdev_priv(netdev);
5005
5006 return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port));
5007 }
5008
mlx5e_vxlan_set_netdev_info(struct mlx5e_priv * priv)5009 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv)
5010 {
5011 if (!mlx5_vxlan_allowed(priv->mdev->vxlan))
5012 return;
5013
5014 priv->nic_info.set_port = mlx5e_vxlan_set_port;
5015 priv->nic_info.unset_port = mlx5e_vxlan_unset_port;
5016 priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
5017 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN;
5018 priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN;
5019 /* Don't count the space hard-coded to the IANA port */
5020 priv->nic_info.tables[0].n_entries =
5021 mlx5_vxlan_max_udp_ports(priv->mdev) - 1;
5022
5023 priv->netdev->udp_tunnel_nic_info = &priv->nic_info;
5024 }
5025
mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev * mdev)5026 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev)
5027 {
5028 int tt;
5029
5030 for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
5031 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt)))
5032 return true;
5033 }
5034 return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev));
5035 }
5036
mlx5e_build_nic_netdev(struct net_device * netdev)5037 static void mlx5e_build_nic_netdev(struct net_device *netdev)
5038 {
5039 struct mlx5e_priv *priv = netdev_priv(netdev);
5040 struct mlx5_core_dev *mdev = priv->mdev;
5041 bool fcs_supported;
5042 bool fcs_enabled;
5043
5044 SET_NETDEV_DEV(netdev, mdev->device);
5045
5046 netdev->netdev_ops = &mlx5e_netdev_ops;
5047
5048 mlx5e_dcbnl_build_netdev(netdev);
5049
5050 netdev->watchdog_timeo = 15 * HZ;
5051
5052 netdev->ethtool_ops = &mlx5e_ethtool_ops;
5053
5054 netdev->vlan_features |= NETIF_F_SG;
5055 netdev->vlan_features |= NETIF_F_HW_CSUM;
5056 netdev->vlan_features |= NETIF_F_GRO;
5057 netdev->vlan_features |= NETIF_F_TSO;
5058 netdev->vlan_features |= NETIF_F_TSO6;
5059 netdev->vlan_features |= NETIF_F_RXCSUM;
5060 netdev->vlan_features |= NETIF_F_RXHASH;
5061 netdev->vlan_features |= NETIF_F_GSO_PARTIAL;
5062
5063 netdev->mpls_features |= NETIF_F_SG;
5064 netdev->mpls_features |= NETIF_F_HW_CSUM;
5065 netdev->mpls_features |= NETIF_F_TSO;
5066 netdev->mpls_features |= NETIF_F_TSO6;
5067
5068 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX;
5069 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX;
5070
5071 /* Tunneled LRO is not supported in the driver, and the same RQs are
5072 * shared between inner and outer TIRs, so the driver can't disable LRO
5073 * for inner TIRs while having it enabled for outer TIRs. Due to this,
5074 * block LRO altogether if the firmware declares tunneled LRO support.
5075 */
5076 if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
5077 !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) &&
5078 !MLX5_CAP_ETH(mdev, tunnel_lro_gre) &&
5079 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT,
5080 MLX5E_MPWRQ_UMR_MODE_ALIGNED))
5081 netdev->vlan_features |= NETIF_F_LRO;
5082
5083 netdev->hw_features = netdev->vlan_features;
5084 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
5085 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
5086 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
5087 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX;
5088
5089 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) {
5090 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
5091 netdev->hw_enc_features |= NETIF_F_TSO;
5092 netdev->hw_enc_features |= NETIF_F_TSO6;
5093 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
5094 }
5095
5096 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) {
5097 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
5098 NETIF_F_GSO_UDP_TUNNEL_CSUM;
5099 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
5100 NETIF_F_GSO_UDP_TUNNEL_CSUM;
5101 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
5102 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL |
5103 NETIF_F_GSO_UDP_TUNNEL_CSUM;
5104 }
5105
5106 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) {
5107 netdev->hw_features |= NETIF_F_GSO_GRE |
5108 NETIF_F_GSO_GRE_CSUM;
5109 netdev->hw_enc_features |= NETIF_F_GSO_GRE |
5110 NETIF_F_GSO_GRE_CSUM;
5111 netdev->gso_partial_features |= NETIF_F_GSO_GRE |
5112 NETIF_F_GSO_GRE_CSUM;
5113 }
5114
5115 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) {
5116 netdev->hw_features |= NETIF_F_GSO_IPXIP4 |
5117 NETIF_F_GSO_IPXIP6;
5118 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 |
5119 NETIF_F_GSO_IPXIP6;
5120 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 |
5121 NETIF_F_GSO_IPXIP6;
5122 }
5123
5124 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4;
5125 netdev->hw_features |= NETIF_F_GSO_UDP_L4;
5126 netdev->features |= NETIF_F_GSO_UDP_L4;
5127
5128 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
5129
5130 if (fcs_supported)
5131 netdev->hw_features |= NETIF_F_RXALL;
5132
5133 if (MLX5_CAP_ETH(mdev, scatter_fcs))
5134 netdev->hw_features |= NETIF_F_RXFCS;
5135
5136 if (mlx5_qos_is_supported(mdev))
5137 netdev->hw_features |= NETIF_F_HW_TC;
5138
5139 netdev->features = netdev->hw_features;
5140
5141 /* Defaults */
5142 if (fcs_enabled)
5143 netdev->features &= ~NETIF_F_RXALL;
5144 netdev->features &= ~NETIF_F_LRO;
5145 netdev->features &= ~NETIF_F_GRO_HW;
5146 netdev->features &= ~NETIF_F_RXFCS;
5147
5148 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
5149 if (FT_CAP(flow_modify_en) &&
5150 FT_CAP(modify_root) &&
5151 FT_CAP(identified_miss_table_mode) &&
5152 FT_CAP(flow_table_modify)) {
5153 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
5154 netdev->hw_features |= NETIF_F_HW_TC;
5155 #endif
5156 #ifdef CONFIG_MLX5_EN_ARFS
5157 netdev->hw_features |= NETIF_F_NTUPLE;
5158 #endif
5159 }
5160
5161 netdev->features |= NETIF_F_HIGHDMA;
5162 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
5163
5164 netdev->priv_flags |= IFF_UNICAST_FLT;
5165
5166 netif_set_tso_max_size(netdev, GSO_MAX_SIZE);
5167 mlx5e_set_netdev_dev_addr(netdev);
5168 mlx5e_macsec_build_netdev(priv);
5169 mlx5e_ipsec_build_netdev(priv);
5170 mlx5e_ktls_build_netdev(priv);
5171 }
5172
mlx5e_create_q_counters(struct mlx5e_priv * priv)5173 void mlx5e_create_q_counters(struct mlx5e_priv *priv)
5174 {
5175 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
5176 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
5177 struct mlx5_core_dev *mdev = priv->mdev;
5178 int err;
5179
5180 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
5181 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
5182 if (!err)
5183 priv->q_counter =
5184 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5185
5186 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
5187 if (!err)
5188 priv->drop_rq_q_counter =
5189 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5190 }
5191
mlx5e_destroy_q_counters(struct mlx5e_priv * priv)5192 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
5193 {
5194 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
5195
5196 MLX5_SET(dealloc_q_counter_in, in, opcode,
5197 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
5198 if (priv->q_counter) {
5199 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5200 priv->q_counter);
5201 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
5202 }
5203
5204 if (priv->drop_rq_q_counter) {
5205 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5206 priv->drop_rq_q_counter);
5207 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
5208 }
5209 }
5210
mlx5e_nic_init(struct mlx5_core_dev * mdev,struct net_device * netdev)5211 static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
5212 struct net_device *netdev)
5213 {
5214 struct mlx5e_priv *priv = netdev_priv(netdev);
5215 struct mlx5e_flow_steering *fs;
5216 int err;
5217
5218 mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu);
5219 mlx5e_vxlan_set_netdev_info(priv);
5220
5221 mlx5e_timestamp_init(priv);
5222
5223 fs = mlx5e_fs_init(priv->profile, mdev,
5224 !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5225 if (!fs) {
5226 err = -ENOMEM;
5227 mlx5_core_err(mdev, "FS initialization failed, %d\n", err);
5228 return err;
5229 }
5230 priv->fs = fs;
5231
5232 err = mlx5e_ipsec_init(priv);
5233 if (err)
5234 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err);
5235
5236 err = mlx5e_ktls_init(priv);
5237 if (err)
5238 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
5239
5240 mlx5e_health_create_reporters(priv);
5241 return 0;
5242 }
5243
mlx5e_nic_cleanup(struct mlx5e_priv * priv)5244 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
5245 {
5246 mlx5e_health_destroy_reporters(priv);
5247 mlx5e_ktls_cleanup(priv);
5248 mlx5e_ipsec_cleanup(priv);
5249 mlx5e_fs_cleanup(priv->fs);
5250 }
5251
mlx5e_init_nic_rx(struct mlx5e_priv * priv)5252 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
5253 {
5254 struct mlx5_core_dev *mdev = priv->mdev;
5255 enum mlx5e_rx_res_features features;
5256 int err;
5257
5258 priv->rx_res = mlx5e_rx_res_alloc();
5259 if (!priv->rx_res)
5260 return -ENOMEM;
5261
5262 mlx5e_create_q_counters(priv);
5263
5264 err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
5265 if (err) {
5266 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
5267 goto err_destroy_q_counters;
5268 }
5269
5270 features = MLX5E_RX_RES_FEATURE_PTP;
5271 if (priv->channels.params.tunneled_offload_en)
5272 features |= MLX5E_RX_RES_FEATURE_INNER_FT;
5273 err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, features,
5274 priv->max_nch, priv->drop_rq.rqn,
5275 &priv->channels.params.packet_merge,
5276 priv->channels.params.num_channels);
5277 if (err)
5278 goto err_close_drop_rq;
5279
5280 err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile,
5281 priv->netdev);
5282 if (err) {
5283 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
5284 goto err_destroy_rx_res;
5285 }
5286
5287 err = mlx5e_tc_nic_init(priv);
5288 if (err)
5289 goto err_destroy_flow_steering;
5290
5291 err = mlx5e_accel_init_rx(priv);
5292 if (err)
5293 goto err_tc_nic_cleanup;
5294
5295 #ifdef CONFIG_MLX5_EN_ARFS
5296 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev);
5297 #endif
5298
5299 return 0;
5300
5301 err_tc_nic_cleanup:
5302 mlx5e_tc_nic_cleanup(priv);
5303 err_destroy_flow_steering:
5304 mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE),
5305 priv->profile);
5306 err_destroy_rx_res:
5307 mlx5e_rx_res_destroy(priv->rx_res);
5308 err_close_drop_rq:
5309 mlx5e_close_drop_rq(&priv->drop_rq);
5310 err_destroy_q_counters:
5311 mlx5e_destroy_q_counters(priv);
5312 mlx5e_rx_res_free(priv->rx_res);
5313 priv->rx_res = NULL;
5314 return err;
5315 }
5316
mlx5e_cleanup_nic_rx(struct mlx5e_priv * priv)5317 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
5318 {
5319 mlx5e_accel_cleanup_rx(priv);
5320 mlx5e_tc_nic_cleanup(priv);
5321 mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE),
5322 priv->profile);
5323 mlx5e_rx_res_destroy(priv->rx_res);
5324 mlx5e_close_drop_rq(&priv->drop_rq);
5325 mlx5e_destroy_q_counters(priv);
5326 mlx5e_rx_res_free(priv->rx_res);
5327 priv->rx_res = NULL;
5328 }
5329
mlx5e_set_mqprio_rl(struct mlx5e_priv * priv)5330 static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv)
5331 {
5332 struct mlx5e_params *params;
5333 struct mlx5e_mqprio_rl *rl;
5334
5335 params = &priv->channels.params;
5336 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL)
5337 return;
5338
5339 rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc,
5340 params->mqprio.channel.max_rate);
5341 if (IS_ERR(rl))
5342 rl = NULL;
5343 priv->mqprio_rl = rl;
5344 mlx5e_mqprio_rl_update_params(params, rl);
5345 }
5346
mlx5e_init_nic_tx(struct mlx5e_priv * priv)5347 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
5348 {
5349 int err;
5350
5351 err = mlx5e_create_tises(priv);
5352 if (err) {
5353 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
5354 return err;
5355 }
5356
5357 err = mlx5e_accel_init_tx(priv);
5358 if (err)
5359 goto err_destroy_tises;
5360
5361 mlx5e_set_mqprio_rl(priv);
5362 mlx5e_dcbnl_initialize(priv);
5363 return 0;
5364
5365 err_destroy_tises:
5366 mlx5e_destroy_tises(priv);
5367 return err;
5368 }
5369
mlx5e_nic_enable(struct mlx5e_priv * priv)5370 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
5371 {
5372 struct net_device *netdev = priv->netdev;
5373 struct mlx5_core_dev *mdev = priv->mdev;
5374 int err;
5375
5376 mlx5e_fs_init_l2_addr(priv->fs, netdev);
5377
5378 err = mlx5e_macsec_init(priv);
5379 if (err)
5380 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err);
5381
5382 /* Marking the link as currently not needed by the Driver */
5383 if (!netif_running(netdev))
5384 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN);
5385
5386 mlx5e_set_netdev_mtu_boundaries(priv);
5387 mlx5e_set_dev_port_mtu(priv);
5388
5389 mlx5_lag_add_netdev(mdev, netdev);
5390
5391 mlx5e_enable_async_events(priv);
5392 mlx5e_enable_blocking_events(priv);
5393 if (mlx5e_monitor_counter_supported(priv))
5394 mlx5e_monitor_counter_init(priv);
5395
5396 mlx5e_hv_vhca_stats_create(priv);
5397 if (netdev->reg_state != NETREG_REGISTERED)
5398 return;
5399 mlx5e_dcbnl_init_app(priv);
5400
5401 mlx5e_nic_set_rx_mode(priv);
5402
5403 rtnl_lock();
5404 if (netif_running(netdev))
5405 mlx5e_open(netdev);
5406 udp_tunnel_nic_reset_ntf(priv->netdev);
5407 netif_device_attach(netdev);
5408 rtnl_unlock();
5409 }
5410
mlx5e_nic_disable(struct mlx5e_priv * priv)5411 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
5412 {
5413 struct mlx5_core_dev *mdev = priv->mdev;
5414
5415 if (priv->netdev->reg_state == NETREG_REGISTERED)
5416 mlx5e_dcbnl_delete_app(priv);
5417
5418 rtnl_lock();
5419 if (netif_running(priv->netdev))
5420 mlx5e_close(priv->netdev);
5421 netif_device_detach(priv->netdev);
5422 rtnl_unlock();
5423
5424 mlx5e_nic_set_rx_mode(priv);
5425
5426 mlx5e_hv_vhca_stats_destroy(priv);
5427 if (mlx5e_monitor_counter_supported(priv))
5428 mlx5e_monitor_counter_cleanup(priv);
5429
5430 mlx5e_disable_blocking_events(priv);
5431 if (priv->en_trap) {
5432 mlx5e_deactivate_trap(priv);
5433 mlx5e_close_trap(priv->en_trap);
5434 priv->en_trap = NULL;
5435 }
5436 mlx5e_disable_async_events(priv);
5437 mlx5_lag_remove_netdev(mdev, priv->netdev);
5438 mlx5_vxlan_reset_to_default(mdev->vxlan);
5439 mlx5e_macsec_cleanup(priv);
5440 }
5441
mlx5e_update_nic_rx(struct mlx5e_priv * priv)5442 int mlx5e_update_nic_rx(struct mlx5e_priv *priv)
5443 {
5444 return mlx5e_refresh_tirs(priv, false, false);
5445 }
5446
5447 static const struct mlx5e_profile mlx5e_nic_profile = {
5448 .init = mlx5e_nic_init,
5449 .cleanup = mlx5e_nic_cleanup,
5450 .init_rx = mlx5e_init_nic_rx,
5451 .cleanup_rx = mlx5e_cleanup_nic_rx,
5452 .init_tx = mlx5e_init_nic_tx,
5453 .cleanup_tx = mlx5e_cleanup_nic_tx,
5454 .enable = mlx5e_nic_enable,
5455 .disable = mlx5e_nic_disable,
5456 .update_rx = mlx5e_update_nic_rx,
5457 .update_stats = mlx5e_stats_update_ndo_stats,
5458 .update_carrier = mlx5e_update_carrier,
5459 .rx_handlers = &mlx5e_rx_handlers_nic,
5460 .max_tc = MLX5E_MAX_NUM_TC,
5461 .stats_grps = mlx5e_nic_stats_grps,
5462 .stats_grps_num = mlx5e_nic_stats_grps_num,
5463 .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) |
5464 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) |
5465 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) |
5466 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) |
5467 BIT(MLX5E_PROFILE_FEATURE_FS_TC),
5468 };
5469
mlx5e_profile_max_num_channels(struct mlx5_core_dev * mdev,const struct mlx5e_profile * profile)5470 static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev,
5471 const struct mlx5e_profile *profile)
5472 {
5473 int nch;
5474
5475 nch = mlx5e_get_max_num_channels(mdev);
5476
5477 if (profile->max_nch_limit)
5478 nch = min_t(int, nch, profile->max_nch_limit(mdev));
5479 return nch;
5480 }
5481
5482 static unsigned int
mlx5e_calc_max_nch(struct mlx5_core_dev * mdev,struct net_device * netdev,const struct mlx5e_profile * profile)5483 mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev,
5484 const struct mlx5e_profile *profile)
5485
5486 {
5487 unsigned int max_nch, tmp;
5488
5489 /* core resources */
5490 max_nch = mlx5e_profile_max_num_channels(mdev, profile);
5491
5492 /* netdev rx queues */
5493 max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues);
5494
5495 /* netdev tx queues */
5496 tmp = netdev->num_tx_queues;
5497 if (mlx5_qos_is_supported(mdev))
5498 tmp -= mlx5e_qos_max_leaf_nodes(mdev);
5499 if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn))
5500 tmp -= profile->max_tc;
5501 tmp = tmp / profile->max_tc;
5502 max_nch = min_t(unsigned int, max_nch, tmp);
5503
5504 return max_nch;
5505 }
5506
mlx5e_get_pf_num_tirs(struct mlx5_core_dev * mdev)5507 int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev)
5508 {
5509 /* Indirect TIRS: 2 sets of TTCs (inner + outer steering)
5510 * and 1 set of direct TIRS
5511 */
5512 return 2 * MLX5E_NUM_INDIR_TIRS
5513 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile);
5514 }
5515
mlx5e_set_rx_mode_work(struct work_struct * work)5516 void mlx5e_set_rx_mode_work(struct work_struct *work)
5517 {
5518 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
5519 set_rx_mode_work);
5520
5521 return mlx5e_fs_set_rx_mode_work(priv->fs, priv->netdev);
5522 }
5523
5524 /* mlx5e generic netdev management API (move to en_common.c) */
mlx5e_priv_init(struct mlx5e_priv * priv,const struct mlx5e_profile * profile,struct net_device * netdev,struct mlx5_core_dev * mdev)5525 int mlx5e_priv_init(struct mlx5e_priv *priv,
5526 const struct mlx5e_profile *profile,
5527 struct net_device *netdev,
5528 struct mlx5_core_dev *mdev)
5529 {
5530 int nch, num_txqs, node;
5531 int err;
5532
5533 num_txqs = netdev->num_tx_queues;
5534 nch = mlx5e_calc_max_nch(mdev, netdev, profile);
5535 node = dev_to_node(mlx5_core_dma_dev(mdev));
5536
5537 /* priv init */
5538 priv->mdev = mdev;
5539 priv->netdev = netdev;
5540 priv->msglevel = MLX5E_MSG_LEVEL;
5541 priv->max_nch = nch;
5542 priv->max_opened_tc = 1;
5543
5544 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL))
5545 return -ENOMEM;
5546
5547 mutex_init(&priv->state_lock);
5548
5549 err = mlx5e_selq_init(&priv->selq, &priv->state_lock);
5550 if (err)
5551 goto err_free_cpumask;
5552
5553 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
5554 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
5555 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
5556 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
5557
5558 priv->wq = create_singlethread_workqueue("mlx5e");
5559 if (!priv->wq)
5560 goto err_free_selq;
5561
5562 priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node);
5563 if (!priv->txq2sq)
5564 goto err_destroy_workqueue;
5565
5566 priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node);
5567 if (!priv->tx_rates)
5568 goto err_free_txq2sq;
5569
5570 priv->channel_stats =
5571 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node);
5572 if (!priv->channel_stats)
5573 goto err_free_tx_rates;
5574
5575 return 0;
5576
5577 err_free_tx_rates:
5578 kfree(priv->tx_rates);
5579 err_free_txq2sq:
5580 kfree(priv->txq2sq);
5581 err_destroy_workqueue:
5582 destroy_workqueue(priv->wq);
5583 err_free_selq:
5584 mlx5e_selq_cleanup(&priv->selq);
5585 err_free_cpumask:
5586 free_cpumask_var(priv->scratchpad.cpumask);
5587 return -ENOMEM;
5588 }
5589
mlx5e_priv_cleanup(struct mlx5e_priv * priv)5590 void mlx5e_priv_cleanup(struct mlx5e_priv *priv)
5591 {
5592 int i;
5593
5594 /* bail if change profile failed and also rollback failed */
5595 if (!priv->mdev)
5596 return;
5597
5598 for (i = 0; i < priv->stats_nch; i++)
5599 kvfree(priv->channel_stats[i]);
5600 kfree(priv->channel_stats);
5601 kfree(priv->tx_rates);
5602 kfree(priv->txq2sq);
5603 destroy_workqueue(priv->wq);
5604 mutex_lock(&priv->state_lock);
5605 mlx5e_selq_cleanup(&priv->selq);
5606 mutex_unlock(&priv->state_lock);
5607 free_cpumask_var(priv->scratchpad.cpumask);
5608
5609 for (i = 0; i < priv->htb_max_qos_sqs; i++)
5610 kfree(priv->htb_qos_sq_stats[i]);
5611 kvfree(priv->htb_qos_sq_stats);
5612
5613 memset(priv, 0, sizeof(*priv));
5614 }
5615
mlx5e_get_max_num_txqs(struct mlx5_core_dev * mdev,const struct mlx5e_profile * profile)5616 static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev,
5617 const struct mlx5e_profile *profile)
5618 {
5619 unsigned int nch, ptp_txqs, qos_txqs;
5620
5621 nch = mlx5e_profile_max_num_channels(mdev, profile);
5622
5623 ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) &&
5624 mlx5e_profile_feature_cap(profile, PTP_TX) ?
5625 profile->max_tc : 0;
5626
5627 qos_txqs = mlx5_qos_is_supported(mdev) &&
5628 mlx5e_profile_feature_cap(profile, QOS_HTB) ?
5629 mlx5e_qos_max_leaf_nodes(mdev) : 0;
5630
5631 return nch * profile->max_tc + ptp_txqs + qos_txqs;
5632 }
5633
mlx5e_get_max_num_rxqs(struct mlx5_core_dev * mdev,const struct mlx5e_profile * profile)5634 static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev,
5635 const struct mlx5e_profile *profile)
5636 {
5637 return mlx5e_profile_max_num_channels(mdev, profile);
5638 }
5639
5640 struct net_device *
mlx5e_create_netdev(struct mlx5_core_dev * mdev,const struct mlx5e_profile * profile)5641 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile)
5642 {
5643 struct net_device *netdev;
5644 unsigned int txqs, rxqs;
5645 int err;
5646
5647 txqs = mlx5e_get_max_num_txqs(mdev, profile);
5648 rxqs = mlx5e_get_max_num_rxqs(mdev, profile);
5649
5650 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs);
5651 if (!netdev) {
5652 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
5653 return NULL;
5654 }
5655
5656 err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev);
5657 if (err) {
5658 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5659 goto err_free_netdev;
5660 }
5661
5662 netif_carrier_off(netdev);
5663 netif_tx_disable(netdev);
5664 dev_net_set(netdev, mlx5_core_net(mdev));
5665
5666 return netdev;
5667
5668 err_free_netdev:
5669 free_netdev(netdev);
5670
5671 return NULL;
5672 }
5673
mlx5e_update_features(struct net_device * netdev)5674 static void mlx5e_update_features(struct net_device *netdev)
5675 {
5676 if (netdev->reg_state != NETREG_REGISTERED)
5677 return; /* features will be updated on netdev registration */
5678
5679 rtnl_lock();
5680 netdev_update_features(netdev);
5681 rtnl_unlock();
5682 }
5683
mlx5e_reset_channels(struct net_device * netdev)5684 static void mlx5e_reset_channels(struct net_device *netdev)
5685 {
5686 netdev_reset_tc(netdev);
5687 }
5688
mlx5e_attach_netdev(struct mlx5e_priv * priv)5689 int mlx5e_attach_netdev(struct mlx5e_priv *priv)
5690 {
5691 const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED;
5692 const struct mlx5e_profile *profile = priv->profile;
5693 int max_nch;
5694 int err;
5695
5696 clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
5697 if (priv->fs)
5698 mlx5e_fs_set_state_destroy(priv->fs,
5699 !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5700
5701 /* Validate the max_wqe_size_sq capability. */
5702 if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) {
5703 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %lu\n",
5704 mlx5e_get_max_sq_wqebbs(priv->mdev), MLX5E_MAX_TX_WQEBBS);
5705 return -EIO;
5706 }
5707
5708 /* max number of channels may have changed */
5709 max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile);
5710 if (priv->channels.params.num_channels > max_nch) {
5711 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch);
5712 /* Reducing the number of channels - RXFH has to be reset, and
5713 * mlx5e_num_channels_changed below will build the RQT.
5714 */
5715 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED;
5716 priv->channels.params.num_channels = max_nch;
5717 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) {
5718 mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n");
5719 mlx5e_params_mqprio_reset(&priv->channels.params);
5720 }
5721 }
5722 if (max_nch != priv->max_nch) {
5723 mlx5_core_warn(priv->mdev,
5724 "MLX5E: Updating max number of channels from %u to %u\n",
5725 priv->max_nch, max_nch);
5726 priv->max_nch = max_nch;
5727 }
5728
5729 /* 1. Set the real number of queues in the kernel the first time.
5730 * 2. Set our default XPS cpumask.
5731 * 3. Build the RQT.
5732 *
5733 * rtnl_lock is required by netif_set_real_num_*_queues in case the
5734 * netdev has been registered by this point (if this function was called
5735 * in the reload or resume flow).
5736 */
5737 if (take_rtnl)
5738 rtnl_lock();
5739 err = mlx5e_num_channels_changed(priv);
5740 if (take_rtnl)
5741 rtnl_unlock();
5742 if (err)
5743 goto out;
5744
5745 err = profile->init_tx(priv);
5746 if (err)
5747 goto out;
5748
5749 err = profile->init_rx(priv);
5750 if (err)
5751 goto err_cleanup_tx;
5752
5753 if (profile->enable)
5754 profile->enable(priv);
5755
5756 mlx5e_update_features(priv->netdev);
5757
5758 return 0;
5759
5760 err_cleanup_tx:
5761 profile->cleanup_tx(priv);
5762
5763 out:
5764 mlx5e_reset_channels(priv->netdev);
5765 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5766 if (priv->fs)
5767 mlx5e_fs_set_state_destroy(priv->fs,
5768 !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5769 cancel_work_sync(&priv->update_stats_work);
5770 return err;
5771 }
5772
mlx5e_detach_netdev(struct mlx5e_priv * priv)5773 void mlx5e_detach_netdev(struct mlx5e_priv *priv)
5774 {
5775 const struct mlx5e_profile *profile = priv->profile;
5776
5777 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5778 if (priv->fs)
5779 mlx5e_fs_set_state_destroy(priv->fs,
5780 !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5781
5782 if (profile->disable)
5783 profile->disable(priv);
5784 flush_workqueue(priv->wq);
5785
5786 profile->cleanup_rx(priv);
5787 profile->cleanup_tx(priv);
5788 mlx5e_reset_channels(priv->netdev);
5789 cancel_work_sync(&priv->update_stats_work);
5790 }
5791
5792 static int
mlx5e_netdev_attach_profile(struct net_device * netdev,struct mlx5_core_dev * mdev,const struct mlx5e_profile * new_profile,void * new_ppriv)5793 mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev,
5794 const struct mlx5e_profile *new_profile, void *new_ppriv)
5795 {
5796 struct mlx5e_priv *priv = netdev_priv(netdev);
5797 int err;
5798
5799 err = mlx5e_priv_init(priv, new_profile, netdev, mdev);
5800 if (err) {
5801 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5802 return err;
5803 }
5804 netif_carrier_off(netdev);
5805 priv->profile = new_profile;
5806 priv->ppriv = new_ppriv;
5807 err = new_profile->init(priv->mdev, priv->netdev);
5808 if (err)
5809 goto priv_cleanup;
5810 err = mlx5e_attach_netdev(priv);
5811 if (err)
5812 goto profile_cleanup;
5813 return err;
5814
5815 profile_cleanup:
5816 new_profile->cleanup(priv);
5817 priv_cleanup:
5818 mlx5e_priv_cleanup(priv);
5819 return err;
5820 }
5821
mlx5e_netdev_change_profile(struct mlx5e_priv * priv,const struct mlx5e_profile * new_profile,void * new_ppriv)5822 int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
5823 const struct mlx5e_profile *new_profile, void *new_ppriv)
5824 {
5825 const struct mlx5e_profile *orig_profile = priv->profile;
5826 struct net_device *netdev = priv->netdev;
5827 struct mlx5_core_dev *mdev = priv->mdev;
5828 void *orig_ppriv = priv->ppriv;
5829 int err, rollback_err;
5830
5831 /* cleanup old profile */
5832 mlx5e_detach_netdev(priv);
5833 priv->profile->cleanup(priv);
5834 mlx5e_priv_cleanup(priv);
5835
5836 err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv);
5837 if (err) { /* roll back to original profile */
5838 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err);
5839 goto rollback;
5840 }
5841
5842 return 0;
5843
5844 rollback:
5845 rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv);
5846 if (rollback_err)
5847 netdev_err(netdev, "%s: failed to rollback to orig profile, %d\n",
5848 __func__, rollback_err);
5849 return err;
5850 }
5851
mlx5e_netdev_attach_nic_profile(struct mlx5e_priv * priv)5852 void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv)
5853 {
5854 mlx5e_netdev_change_profile(priv, &mlx5e_nic_profile, NULL);
5855 }
5856
mlx5e_destroy_netdev(struct mlx5e_priv * priv)5857 void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
5858 {
5859 struct net_device *netdev = priv->netdev;
5860
5861 mlx5e_priv_cleanup(priv);
5862 free_netdev(netdev);
5863 }
5864
mlx5e_resume(struct auxiliary_device * adev)5865 static int mlx5e_resume(struct auxiliary_device *adev)
5866 {
5867 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5868 struct mlx5e_priv *priv = auxiliary_get_drvdata(adev);
5869 struct net_device *netdev = priv->netdev;
5870 struct mlx5_core_dev *mdev = edev->mdev;
5871 int err;
5872
5873 if (netif_device_present(netdev))
5874 return 0;
5875
5876 err = mlx5e_create_mdev_resources(mdev);
5877 if (err)
5878 return err;
5879
5880 err = mlx5e_attach_netdev(priv);
5881 if (err) {
5882 mlx5e_destroy_mdev_resources(mdev);
5883 return err;
5884 }
5885
5886 return 0;
5887 }
5888
mlx5e_suspend(struct auxiliary_device * adev,pm_message_t state)5889 static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state)
5890 {
5891 struct mlx5e_priv *priv = auxiliary_get_drvdata(adev);
5892 struct net_device *netdev = priv->netdev;
5893 struct mlx5_core_dev *mdev = priv->mdev;
5894
5895 if (!netif_device_present(netdev))
5896 return -ENODEV;
5897
5898 mlx5e_detach_netdev(priv);
5899 mlx5e_destroy_mdev_resources(mdev);
5900 return 0;
5901 }
5902
mlx5e_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)5903 static int mlx5e_probe(struct auxiliary_device *adev,
5904 const struct auxiliary_device_id *id)
5905 {
5906 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5907 const struct mlx5e_profile *profile = &mlx5e_nic_profile;
5908 struct mlx5_core_dev *mdev = edev->mdev;
5909 struct net_device *netdev;
5910 pm_message_t state = {};
5911 struct mlx5e_priv *priv;
5912 int err;
5913
5914 netdev = mlx5e_create_netdev(mdev, profile);
5915 if (!netdev) {
5916 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
5917 return -ENOMEM;
5918 }
5919
5920 mlx5e_build_nic_netdev(netdev);
5921
5922 priv = netdev_priv(netdev);
5923 auxiliary_set_drvdata(adev, priv);
5924
5925 priv->profile = profile;
5926 priv->ppriv = NULL;
5927
5928 err = mlx5e_devlink_port_register(priv);
5929 if (err) {
5930 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err);
5931 goto err_destroy_netdev;
5932 }
5933
5934 err = profile->init(mdev, netdev);
5935 if (err) {
5936 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err);
5937 goto err_devlink_cleanup;
5938 }
5939
5940 err = mlx5e_resume(adev);
5941 if (err) {
5942 mlx5_core_err(mdev, "mlx5e_resume failed, %d\n", err);
5943 goto err_profile_cleanup;
5944 }
5945
5946 err = register_netdev(netdev);
5947 if (err) {
5948 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
5949 goto err_resume;
5950 }
5951
5952 mlx5e_devlink_port_type_eth_set(priv);
5953
5954 mlx5e_dcbnl_init_app(priv);
5955 mlx5_uplink_netdev_set(mdev, netdev);
5956 return 0;
5957
5958 err_resume:
5959 mlx5e_suspend(adev, state);
5960 err_profile_cleanup:
5961 profile->cleanup(priv);
5962 err_devlink_cleanup:
5963 mlx5e_devlink_port_unregister(priv);
5964 err_destroy_netdev:
5965 mlx5e_destroy_netdev(priv);
5966 return err;
5967 }
5968
mlx5e_remove(struct auxiliary_device * adev)5969 static void mlx5e_remove(struct auxiliary_device *adev)
5970 {
5971 struct mlx5e_priv *priv = auxiliary_get_drvdata(adev);
5972 pm_message_t state = {};
5973
5974 mlx5e_dcbnl_delete_app(priv);
5975 unregister_netdev(priv->netdev);
5976 mlx5e_suspend(adev, state);
5977 priv->profile->cleanup(priv);
5978 mlx5e_devlink_port_unregister(priv);
5979 mlx5e_destroy_netdev(priv);
5980 }
5981
5982 static const struct auxiliary_device_id mlx5e_id_table[] = {
5983 { .name = MLX5_ADEV_NAME ".eth", },
5984 {},
5985 };
5986
5987 MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table);
5988
5989 static struct auxiliary_driver mlx5e_driver = {
5990 .name = "eth",
5991 .probe = mlx5e_probe,
5992 .remove = mlx5e_remove,
5993 .suspend = mlx5e_suspend,
5994 .resume = mlx5e_resume,
5995 .id_table = mlx5e_id_table,
5996 };
5997
mlx5e_init(void)5998 int mlx5e_init(void)
5999 {
6000 int ret;
6001
6002 mlx5e_build_ptys2ethtool_map();
6003 ret = auxiliary_driver_register(&mlx5e_driver);
6004 if (ret)
6005 return ret;
6006
6007 ret = mlx5e_rep_init();
6008 if (ret)
6009 auxiliary_driver_unregister(&mlx5e_driver);
6010 return ret;
6011 }
6012
mlx5e_cleanup(void)6013 void mlx5e_cleanup(void)
6014 {
6015 mlx5e_rep_cleanup();
6016 auxiliary_driver_unregister(&mlx5e_driver);
6017 }
6018