1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
43 #include "rdma.h"
44 #include "en.h"
45 #include "fs_core.h"
46 #include "lib/devcom.h"
47 #include "lib/eq.h"
48 #include "lib/fs_chains.h"
49 #include "en_tc.h"
50 #include "en/mapping.h"
51 #include "devlink.h"
52 #include "lag/lag.h"
53
54 #define mlx5_esw_for_each_rep(esw, i, rep) \
55 xa_for_each(&((esw)->offloads.vport_reps), i, rep)
56
57 #define mlx5_esw_for_each_sf_rep(esw, i, rep) \
58 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF)
59
60 #define mlx5_esw_for_each_vf_rep(esw, index, rep) \
61 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \
62 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF)
63
64 /* There are two match-all miss flows, one for unicast dst mac and
65 * one for multicast.
66 */
67 #define MLX5_ESW_MISS_FLOWS (2)
68 #define UPLINK_REP_INDEX 0
69
70 #define MLX5_ESW_VPORT_TBL_SIZE 128
71 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
72
73 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
74
75 static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
76 .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
77 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
78 .flags = 0,
79 };
80
mlx5_eswitch_get_rep(struct mlx5_eswitch * esw,u16 vport_num)81 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
82 u16 vport_num)
83 {
84 return xa_load(&esw->offloads.vport_reps, vport_num);
85 }
86
87 static void
mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr)88 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
89 struct mlx5_flow_spec *spec,
90 struct mlx5_esw_flow_attr *attr)
91 {
92 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
93 return;
94
95 if (attr->int_port) {
96 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port);
97
98 return;
99 }
100
101 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
102 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
103 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
104 }
105
106 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
107 * are not needed as well in the following process. So clear them all for simplicity.
108 */
109 void
mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec)110 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
111 {
112 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
113 void *misc2;
114
115 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
116 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
117
118 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
119 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
120
121 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
122 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
123 }
124 }
125
126 static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct mlx5_eswitch * src_esw,u16 vport)127 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
128 struct mlx5_flow_spec *spec,
129 struct mlx5_flow_attr *attr,
130 struct mlx5_eswitch *src_esw,
131 u16 vport)
132 {
133 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
134 u32 metadata;
135 void *misc2;
136 void *misc;
137
138 /* Use metadata matching because vport is not represented by single
139 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
140 */
141 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
142 if (mlx5_esw_indir_table_decap_vport(attr))
143 vport = mlx5_esw_indir_table_decap_vport(attr);
144
145 if (!attr->chain && esw_attr && esw_attr->int_port)
146 metadata =
147 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port);
148 else
149 metadata =
150 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport);
151
152 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
153 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
154
155 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
156 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
157 mlx5_eswitch_get_vport_metadata_mask());
158
159 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
160 } else {
161 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
162 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
163
164 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
165 MLX5_SET(fte_match_set_misc, misc,
166 source_eswitch_owner_vhca_id,
167 MLX5_CAP_GEN(src_esw->dev, vhca_id));
168
169 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
170 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
171 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
172 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
173 source_eswitch_owner_vhca_id);
174
175 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
176 }
177 }
178
179 static int
esw_setup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec)180 esw_setup_decap_indir(struct mlx5_eswitch *esw,
181 struct mlx5_flow_attr *attr,
182 struct mlx5_flow_spec *spec)
183 {
184 struct mlx5_flow_table *ft;
185
186 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
187 return -EOPNOTSUPP;
188
189 ft = mlx5_esw_indir_table_get(esw, attr, spec,
190 mlx5_esw_indir_table_decap_vport(attr), true);
191 return PTR_ERR_OR_ZERO(ft);
192 }
193
194 static void
esw_cleanup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)195 esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
196 struct mlx5_flow_attr *attr)
197 {
198 if (mlx5_esw_indir_table_decap_vport(attr))
199 mlx5_esw_indir_table_put(esw, attr,
200 mlx5_esw_indir_table_decap_vport(attr),
201 true);
202 }
203
204 static int
esw_setup_sampler_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,u32 sampler_id,int i)205 esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
206 struct mlx5_flow_act *flow_act,
207 u32 sampler_id,
208 int i)
209 {
210 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
211 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
212 dest[i].sampler_id = sampler_id;
213
214 return 0;
215 }
216
217 static int
esw_setup_ft_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int i)218 esw_setup_ft_dest(struct mlx5_flow_destination *dest,
219 struct mlx5_flow_act *flow_act,
220 struct mlx5_eswitch *esw,
221 struct mlx5_flow_attr *attr,
222 struct mlx5_flow_spec *spec,
223 int i)
224 {
225 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
226 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
227 dest[i].ft = attr->dest_ft;
228
229 if (mlx5_esw_indir_table_decap_vport(attr))
230 return esw_setup_decap_indir(esw, attr, spec);
231 return 0;
232 }
233
234 static void
esw_setup_accept_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,int i)235 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
236 struct mlx5_fs_chains *chains, int i)
237 {
238 if (mlx5_chains_ignore_flow_level_supported(chains))
239 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
240 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
241 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
242 }
243
244 static void
esw_setup_slow_path_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,int i)245 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
246 struct mlx5_eswitch *esw, int i)
247 {
248 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
249 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
250 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
251 dest[i].ft = esw->fdb_table.offloads.slow_fdb;
252 }
253
254 static int
esw_setup_chain_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level,int i)255 esw_setup_chain_dest(struct mlx5_flow_destination *dest,
256 struct mlx5_flow_act *flow_act,
257 struct mlx5_fs_chains *chains,
258 u32 chain, u32 prio, u32 level,
259 int i)
260 {
261 struct mlx5_flow_table *ft;
262
263 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
264 ft = mlx5_chains_get_table(chains, chain, prio, level);
265 if (IS_ERR(ft))
266 return PTR_ERR(ft);
267
268 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
269 dest[i].ft = ft;
270 return 0;
271 }
272
esw_put_dest_tables_loop(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int from,int to)273 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
274 int from, int to)
275 {
276 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
277 struct mlx5_fs_chains *chains = esw_chains(esw);
278 int i;
279
280 for (i = from; i < to; i++)
281 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
282 mlx5_chains_put_table(chains, 0, 1, 0);
283 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
284 esw_attr->dests[i].mdev))
285 mlx5_esw_indir_table_put(esw, attr, esw_attr->dests[i].rep->vport,
286 false);
287 }
288
289 static bool
esw_is_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)290 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
291 {
292 int i;
293
294 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
295 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
296 return true;
297 return false;
298 }
299
300 static int
esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains,struct mlx5_flow_attr * attr,int * i)301 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
302 struct mlx5_flow_act *flow_act,
303 struct mlx5_eswitch *esw,
304 struct mlx5_fs_chains *chains,
305 struct mlx5_flow_attr *attr,
306 int *i)
307 {
308 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
309 int err;
310
311 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
312 return -EOPNOTSUPP;
313
314 /* flow steering cannot handle more than one dest with the same ft
315 * in a single flow
316 */
317 if (esw_attr->out_count - esw_attr->split_count > 1)
318 return -EOPNOTSUPP;
319
320 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
321 if (err)
322 return err;
323
324 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
325 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
326 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
327 }
328 (*i)++;
329
330 return 0;
331 }
332
esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)333 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
334 struct mlx5_flow_attr *attr)
335 {
336 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
337
338 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
339 }
340
341 static bool
esw_is_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)342 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
343 {
344 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
345 bool result = false;
346 int i;
347
348 /* Indirect table is supported only for flows with in_port uplink
349 * and the destination is vport on the same eswitch as the uplink,
350 * return false in case at least one of destinations doesn't meet
351 * this criteria.
352 */
353 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
354 if (esw_attr->dests[i].rep &&
355 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
356 esw_attr->dests[i].mdev)) {
357 result = true;
358 } else {
359 result = false;
360 break;
361 }
362 }
363 return result;
364 }
365
366 static int
esw_setup_indir_table(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,bool ignore_flow_lvl,int * i)367 esw_setup_indir_table(struct mlx5_flow_destination *dest,
368 struct mlx5_flow_act *flow_act,
369 struct mlx5_eswitch *esw,
370 struct mlx5_flow_attr *attr,
371 struct mlx5_flow_spec *spec,
372 bool ignore_flow_lvl,
373 int *i)
374 {
375 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
376 int j, err;
377
378 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
379 return -EOPNOTSUPP;
380
381 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
382 if (ignore_flow_lvl)
383 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
384 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
385
386 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, spec,
387 esw_attr->dests[j].rep->vport, false);
388 if (IS_ERR(dest[*i].ft)) {
389 err = PTR_ERR(dest[*i].ft);
390 goto err_indir_tbl_get;
391 }
392 }
393
394 if (mlx5_esw_indir_table_decap_vport(attr)) {
395 err = esw_setup_decap_indir(esw, attr, spec);
396 if (err)
397 goto err_indir_tbl_get;
398 }
399
400 return 0;
401
402 err_indir_tbl_get:
403 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
404 return err;
405 }
406
esw_cleanup_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)407 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
408 {
409 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
410
411 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
412 esw_cleanup_decap_indir(esw, attr);
413 }
414
415 static void
esw_cleanup_chain_dest(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)416 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
417 {
418 mlx5_chains_put_table(chains, chain, prio, level);
419 }
420
421 static void
esw_setup_vport_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)422 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
423 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
424 int attr_idx, int dest_idx, bool pkt_reformat)
425 {
426 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
427 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport;
428 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
429 dest[dest_idx].vport.vhca_id =
430 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
431 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
432 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
433 mlx5_lag_mpesw_is_activated(esw->dev))
434 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
435 }
436 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
437 if (pkt_reformat) {
438 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
439 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
440 }
441 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
442 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
443 }
444 }
445
446 static int
esw_setup_vport_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int i)447 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
448 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
449 int i)
450 {
451 int j;
452
453 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
454 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
455 return i;
456 }
457
458 static bool
esw_src_port_rewrite_supported(struct mlx5_eswitch * esw)459 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
460 {
461 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
462 mlx5_eswitch_vport_match_metadata_enabled(esw) &&
463 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
464 }
465
466 static int
esw_setup_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int * i)467 esw_setup_dests(struct mlx5_flow_destination *dest,
468 struct mlx5_flow_act *flow_act,
469 struct mlx5_eswitch *esw,
470 struct mlx5_flow_attr *attr,
471 struct mlx5_flow_spec *spec,
472 int *i)
473 {
474 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
475 struct mlx5_fs_chains *chains = esw_chains(esw);
476 int err = 0;
477
478 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
479 esw_src_port_rewrite_supported(esw))
480 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
481
482 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE &&
483 !(attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)) {
484 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
485 (*i)++;
486 } else if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
487 esw_setup_slow_path_dest(dest, flow_act, esw, *i);
488 (*i)++;
489 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
490 esw_setup_accept_dest(dest, flow_act, chains, *i);
491 (*i)++;
492 } else if (esw_is_indir_table(esw, attr)) {
493 err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i);
494 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
495 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
496 } else {
497 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
498
499 if (attr->dest_ft) {
500 err = esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i);
501 (*i)++;
502 } else if (attr->dest_chain) {
503 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
504 1, 0, *i);
505 (*i)++;
506 }
507 }
508
509 return err;
510 }
511
512 static void
esw_cleanup_dests(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)513 esw_cleanup_dests(struct mlx5_eswitch *esw,
514 struct mlx5_flow_attr *attr)
515 {
516 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
517 struct mlx5_fs_chains *chains = esw_chains(esw);
518
519 if (attr->dest_ft) {
520 esw_cleanup_decap_indir(esw, attr);
521 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
522 if (attr->dest_chain)
523 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
524 else if (esw_is_indir_table(esw, attr))
525 esw_cleanup_indir_table(esw, attr);
526 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
527 esw_cleanup_chain_src_port_rewrite(esw, attr);
528 }
529 }
530
531 static void
esw_setup_meter(struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act)532 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
533 {
534 struct mlx5e_flow_meter_handle *meter;
535
536 meter = attr->meter_attr.meter;
537 flow_act->exe_aso.type = attr->exe_aso_type;
538 flow_act->exe_aso.object_id = meter->obj_id;
539 flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
540 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
541 /* use metadata reg 5 for packet color */
542 flow_act->exe_aso.return_reg_id = 5;
543 }
544
545 struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)546 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
547 struct mlx5_flow_spec *spec,
548 struct mlx5_flow_attr *attr)
549 {
550 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
551 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
552 struct mlx5_fs_chains *chains = esw_chains(esw);
553 bool split = !!(esw_attr->split_count);
554 struct mlx5_vport_tbl_attr fwd_attr;
555 struct mlx5_flow_destination *dest;
556 struct mlx5_flow_handle *rule;
557 struct mlx5_flow_table *fdb;
558 int i = 0;
559
560 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
561 return ERR_PTR(-EOPNOTSUPP);
562
563 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
564 if (!dest)
565 return ERR_PTR(-ENOMEM);
566
567 flow_act.action = attr->action;
568 /* if per flow vlan pop/push is emulated, don't set that into the firmware */
569 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
570 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH |
571 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
572 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
573 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
574 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
575 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
576 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
577 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
578 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
579 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
580 }
581 }
582
583 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
584
585 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
586 int err;
587
588 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
589 if (err) {
590 rule = ERR_PTR(err);
591 goto err_create_goto_table;
592 }
593 }
594
595 if (esw_attr->decap_pkt_reformat)
596 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
597
598 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
599 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
600 dest[i].counter_id = mlx5_fc_id(attr->counter);
601 i++;
602 }
603
604 if (attr->outer_match_level != MLX5_MATCH_NONE)
605 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
606 if (attr->inner_match_level != MLX5_MATCH_NONE)
607 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
608
609 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
610 flow_act.modify_hdr = attr->modify_hdr;
611
612 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
613 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
614 esw_setup_meter(attr, &flow_act);
615
616 if (split) {
617 fwd_attr.chain = attr->chain;
618 fwd_attr.prio = attr->prio;
619 fwd_attr.vport = esw_attr->in_rep->vport;
620 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
621
622 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
623 } else {
624 if (attr->chain || attr->prio)
625 fdb = mlx5_chains_get_table(chains, attr->chain,
626 attr->prio, 0);
627 else
628 fdb = attr->ft;
629
630 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
631 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
632 esw_attr->in_mdev->priv.eswitch,
633 esw_attr->in_rep->vport);
634 }
635 if (IS_ERR(fdb)) {
636 rule = ERR_CAST(fdb);
637 goto err_esw_get;
638 }
639
640 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
641 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
642 &flow_act, dest, i);
643 else
644 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
645 if (IS_ERR(rule))
646 goto err_add_rule;
647 else
648 atomic64_inc(&esw->offloads.num_flows);
649
650 kfree(dest);
651 return rule;
652
653 err_add_rule:
654 if (split)
655 mlx5_esw_vporttbl_put(esw, &fwd_attr);
656 else if (attr->chain || attr->prio)
657 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
658 err_esw_get:
659 esw_cleanup_dests(esw, attr);
660 err_create_goto_table:
661 kfree(dest);
662 return rule;
663 }
664
665 struct mlx5_flow_handle *
mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)666 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
667 struct mlx5_flow_spec *spec,
668 struct mlx5_flow_attr *attr)
669 {
670 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
671 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
672 struct mlx5_fs_chains *chains = esw_chains(esw);
673 struct mlx5_vport_tbl_attr fwd_attr;
674 struct mlx5_flow_destination *dest;
675 struct mlx5_flow_table *fast_fdb;
676 struct mlx5_flow_table *fwd_fdb;
677 struct mlx5_flow_handle *rule;
678 int i, err = 0;
679
680 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
681 if (!dest)
682 return ERR_PTR(-ENOMEM);
683
684 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
685 if (IS_ERR(fast_fdb)) {
686 rule = ERR_CAST(fast_fdb);
687 goto err_get_fast;
688 }
689
690 fwd_attr.chain = attr->chain;
691 fwd_attr.prio = attr->prio;
692 fwd_attr.vport = esw_attr->in_rep->vport;
693 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
694 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
695 if (IS_ERR(fwd_fdb)) {
696 rule = ERR_CAST(fwd_fdb);
697 goto err_get_fwd;
698 }
699
700 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
701 for (i = 0; i < esw_attr->split_count; i++) {
702 if (esw_is_indir_table(esw, attr))
703 err = esw_setup_indir_table(dest, &flow_act, esw, attr, spec, false, &i);
704 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
705 err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr,
706 &i);
707 else
708 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
709
710 if (err) {
711 rule = ERR_PTR(err);
712 goto err_chain_src_rewrite;
713 }
714 }
715 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
716 dest[i].ft = fwd_fdb;
717 i++;
718
719 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
720 esw_attr->in_mdev->priv.eswitch,
721 esw_attr->in_rep->vport);
722
723 if (attr->outer_match_level != MLX5_MATCH_NONE)
724 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
725
726 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
727 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
728
729 if (IS_ERR(rule)) {
730 i = esw_attr->split_count;
731 goto err_chain_src_rewrite;
732 }
733
734 atomic64_inc(&esw->offloads.num_flows);
735
736 kfree(dest);
737 return rule;
738 err_chain_src_rewrite:
739 esw_put_dest_tables_loop(esw, attr, 0, i);
740 mlx5_esw_vporttbl_put(esw, &fwd_attr);
741 err_get_fwd:
742 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
743 err_get_fast:
744 kfree(dest);
745 return rule;
746 }
747
748 static void
__mlx5_eswitch_del_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr,bool fwd_rule)749 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
750 struct mlx5_flow_handle *rule,
751 struct mlx5_flow_attr *attr,
752 bool fwd_rule)
753 {
754 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
755 struct mlx5_fs_chains *chains = esw_chains(esw);
756 bool split = (esw_attr->split_count > 0);
757 struct mlx5_vport_tbl_attr fwd_attr;
758 int i;
759
760 mlx5_del_flow_rules(rule);
761
762 if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
763 /* unref the term table */
764 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
765 if (esw_attr->dests[i].termtbl)
766 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
767 }
768 }
769
770 atomic64_dec(&esw->offloads.num_flows);
771
772 if (fwd_rule || split) {
773 fwd_attr.chain = attr->chain;
774 fwd_attr.prio = attr->prio;
775 fwd_attr.vport = esw_attr->in_rep->vport;
776 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
777 }
778
779 if (fwd_rule) {
780 mlx5_esw_vporttbl_put(esw, &fwd_attr);
781 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
782 esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count);
783 } else {
784 if (split)
785 mlx5_esw_vporttbl_put(esw, &fwd_attr);
786 else if (attr->chain || attr->prio)
787 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
788 esw_cleanup_dests(esw, attr);
789 }
790 }
791
792 void
mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)793 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
794 struct mlx5_flow_handle *rule,
795 struct mlx5_flow_attr *attr)
796 {
797 __mlx5_eswitch_del_rule(esw, rule, attr, false);
798 }
799
800 void
mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)801 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
802 struct mlx5_flow_handle *rule,
803 struct mlx5_flow_attr *attr)
804 {
805 __mlx5_eswitch_del_rule(esw, rule, attr, true);
806 }
807
esw_set_global_vlan_pop(struct mlx5_eswitch * esw,u8 val)808 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
809 {
810 struct mlx5_eswitch_rep *rep;
811 unsigned long i;
812 int err = 0;
813
814 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
815 mlx5_esw_for_each_host_func_vport(esw, i, rep, esw->esw_funcs.num_vfs) {
816 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
817 continue;
818
819 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
820 if (err)
821 goto out;
822 }
823
824 out:
825 return err;
826 }
827
828 static struct mlx5_eswitch_rep *
esw_vlan_action_get_vport(struct mlx5_esw_flow_attr * attr,bool push,bool pop)829 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop)
830 {
831 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL;
832
833 in_rep = attr->in_rep;
834 out_rep = attr->dests[0].rep;
835
836 if (push)
837 vport = in_rep;
838 else if (pop)
839 vport = out_rep;
840 else
841 vport = in_rep;
842
843 return vport;
844 }
845
esw_add_vlan_action_check(struct mlx5_esw_flow_attr * attr,bool push,bool pop,bool fwd)846 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr,
847 bool push, bool pop, bool fwd)
848 {
849 struct mlx5_eswitch_rep *in_rep, *out_rep;
850
851 if ((push || pop) && !fwd)
852 goto out_notsupp;
853
854 in_rep = attr->in_rep;
855 out_rep = attr->dests[0].rep;
856
857 if (push && in_rep->vport == MLX5_VPORT_UPLINK)
858 goto out_notsupp;
859
860 if (pop && out_rep->vport == MLX5_VPORT_UPLINK)
861 goto out_notsupp;
862
863 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */
864 if (!push && !pop && fwd)
865 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK)
866 goto out_notsupp;
867
868 /* protects against (1) setting rules with different vlans to push and
869 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
870 */
871 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0]))
872 goto out_notsupp;
873
874 return 0;
875
876 out_notsupp:
877 return -EOPNOTSUPP;
878 }
879
mlx5_eswitch_add_vlan_action(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)880 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
881 struct mlx5_flow_attr *attr)
882 {
883 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
884 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
885 struct mlx5_eswitch_rep *vport = NULL;
886 bool push, pop, fwd;
887 int err = 0;
888
889 /* nop if we're on the vlan push/pop non emulation mode */
890 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
891 return 0;
892
893 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
894 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
895 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) &&
896 !attr->dest_chain);
897
898 mutex_lock(&esw->state_lock);
899
900 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd);
901 if (err)
902 goto unlock;
903
904 attr->flags &= ~MLX5_ATTR_FLAG_VLAN_HANDLED;
905
906 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
907
908 if (!push && !pop && fwd) {
909 /* tracks VF --> wire rules without vlan push action */
910 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {
911 vport->vlan_refcount++;
912 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED;
913 }
914
915 goto unlock;
916 }
917
918 if (!push && !pop)
919 goto unlock;
920
921 if (!(offloads->vlan_push_pop_refcount)) {
922 /* it's the 1st vlan rule, apply global vlan pop policy */
923 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP);
924 if (err)
925 goto out;
926 }
927 offloads->vlan_push_pop_refcount++;
928
929 if (push) {
930 if (vport->vlan_refcount)
931 goto skip_set_push;
932
933 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0],
934 0, SET_VLAN_INSERT | SET_VLAN_STRIP);
935 if (err)
936 goto out;
937 vport->vlan = esw_attr->vlan_vid[0];
938 skip_set_push:
939 vport->vlan_refcount++;
940 }
941 out:
942 if (!err)
943 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED;
944 unlock:
945 mutex_unlock(&esw->state_lock);
946 return err;
947 }
948
mlx5_eswitch_del_vlan_action(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)949 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
950 struct mlx5_flow_attr *attr)
951 {
952 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
953 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
954 struct mlx5_eswitch_rep *vport = NULL;
955 bool push, pop, fwd;
956 int err = 0;
957
958 /* nop if we're on the vlan push/pop non emulation mode */
959 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
960 return 0;
961
962 if (!(attr->flags & MLX5_ATTR_FLAG_VLAN_HANDLED))
963 return 0;
964
965 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
966 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
967 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
968
969 mutex_lock(&esw->state_lock);
970
971 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
972
973 if (!push && !pop && fwd) {
974 /* tracks VF --> wire rules without vlan push action */
975 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK)
976 vport->vlan_refcount--;
977
978 goto out;
979 }
980
981 if (push) {
982 vport->vlan_refcount--;
983 if (vport->vlan_refcount)
984 goto skip_unset_push;
985
986 vport->vlan = 0;
987 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport,
988 0, 0, SET_VLAN_STRIP);
989 if (err)
990 goto out;
991 }
992
993 skip_unset_push:
994 offloads->vlan_push_pop_refcount--;
995 if (offloads->vlan_push_pop_refcount)
996 goto out;
997
998 /* no more vlan rules, stop global vlan pop policy */
999 err = esw_set_global_vlan_pop(esw, 0);
1000
1001 out:
1002 mutex_unlock(&esw->state_lock);
1003 return err;
1004 }
1005
1006 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch * on_esw,struct mlx5_eswitch * from_esw,struct mlx5_eswitch_rep * rep,u32 sqn)1007 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
1008 struct mlx5_eswitch *from_esw,
1009 struct mlx5_eswitch_rep *rep,
1010 u32 sqn)
1011 {
1012 struct mlx5_flow_act flow_act = {0};
1013 struct mlx5_flow_destination dest = {};
1014 struct mlx5_flow_handle *flow_rule;
1015 struct mlx5_flow_spec *spec;
1016 void *misc;
1017
1018 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1019 if (!spec) {
1020 flow_rule = ERR_PTR(-ENOMEM);
1021 goto out;
1022 }
1023
1024 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
1025 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
1026 /* source vport is the esw manager */
1027 MLX5_SET(fte_match_set_misc, misc, source_port, from_esw->manager_vport);
1028 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
1029 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1030 MLX5_CAP_GEN(from_esw->dev, vhca_id));
1031
1032 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
1033 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
1034 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1035 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
1036 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1037 source_eswitch_owner_vhca_id);
1038
1039 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1040 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1041 dest.vport.num = rep->vport;
1042 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
1043 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1044 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1045
1046 if (rep->vport == MLX5_VPORT_UPLINK)
1047 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1048
1049 flow_rule = mlx5_add_flow_rules(on_esw->fdb_table.offloads.slow_fdb,
1050 spec, &flow_act, &dest, 1);
1051 if (IS_ERR(flow_rule))
1052 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
1053 PTR_ERR(flow_rule));
1054 out:
1055 kvfree(spec);
1056 return flow_rule;
1057 }
1058 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1059
mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle * rule)1060 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1061 {
1062 mlx5_del_flow_rules(rule);
1063 }
1064
mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle * rule)1065 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1066 {
1067 if (rule)
1068 mlx5_del_flow_rules(rule);
1069 }
1070
1071 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch * esw,u16 vport_num)1072 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1073 {
1074 struct mlx5_flow_destination dest = {};
1075 struct mlx5_flow_act flow_act = {0};
1076 struct mlx5_flow_handle *flow_rule;
1077 struct mlx5_flow_spec *spec;
1078
1079 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1080 if (!spec)
1081 return ERR_PTR(-ENOMEM);
1082
1083 MLX5_SET(fte_match_param, spec->match_criteria,
1084 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1085 MLX5_SET(fte_match_param, spec->match_criteria,
1086 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1087 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1088 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1089
1090 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1091 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1092 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1093
1094 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1095 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1096 dest.vport.num = vport_num;
1097
1098 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1099 spec, &flow_act, &dest, 1);
1100 if (IS_ERR(flow_rule))
1101 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
1102 vport_num, PTR_ERR(flow_rule));
1103
1104 kvfree(spec);
1105 return flow_rule;
1106 }
1107
mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch * esw)1108 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1109 {
1110 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1111 MLX5_FDB_TO_VPORT_REG_C_1;
1112 }
1113
esw_set_passing_vport_metadata(struct mlx5_eswitch * esw,bool enable)1114 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1115 {
1116 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1117 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1118 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1119 u8 curr, wanted;
1120 int err;
1121
1122 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1123 !mlx5_eswitch_vport_match_metadata_enabled(esw))
1124 return 0;
1125
1126 MLX5_SET(query_esw_vport_context_in, in, opcode,
1127 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1128 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1129 if (err)
1130 return err;
1131
1132 curr = MLX5_GET(query_esw_vport_context_out, out,
1133 esw_vport_context.fdb_to_vport_reg_c_id);
1134 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1135 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1136 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1137
1138 if (enable)
1139 curr |= wanted;
1140 else
1141 curr &= ~wanted;
1142
1143 MLX5_SET(modify_esw_vport_context_in, min,
1144 esw_vport_context.fdb_to_vport_reg_c_id, curr);
1145 MLX5_SET(modify_esw_vport_context_in, min,
1146 field_select.fdb_to_vport_reg_c_id, 1);
1147
1148 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
1149 if (!err) {
1150 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1151 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1152 else
1153 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1154 }
1155
1156 return err;
1157 }
1158
peer_miss_rules_setup(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev,struct mlx5_flow_spec * spec,struct mlx5_flow_destination * dest)1159 static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1160 struct mlx5_core_dev *peer_dev,
1161 struct mlx5_flow_spec *spec,
1162 struct mlx5_flow_destination *dest)
1163 {
1164 void *misc;
1165
1166 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1167 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1168 misc_parameters_2);
1169 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1170 mlx5_eswitch_get_vport_metadata_mask());
1171
1172 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1173 } else {
1174 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1175 misc_parameters);
1176
1177 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1178 MLX5_CAP_GEN(peer_dev, vhca_id));
1179
1180 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1181
1182 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1183 misc_parameters);
1184 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1185 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1186 source_eswitch_owner_vhca_id);
1187 }
1188
1189 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1190 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1191 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1192 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1193 }
1194
esw_set_peer_miss_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,struct mlx5_flow_spec * spec,u16 vport)1195 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1196 struct mlx5_eswitch *peer_esw,
1197 struct mlx5_flow_spec *spec,
1198 u16 vport)
1199 {
1200 void *misc;
1201
1202 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1203 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1204 misc_parameters_2);
1205 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1206 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1207 vport));
1208 } else {
1209 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1210 misc_parameters);
1211 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1212 }
1213 }
1214
esw_add_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1215 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1216 struct mlx5_core_dev *peer_dev)
1217 {
1218 struct mlx5_flow_destination dest = {};
1219 struct mlx5_flow_act flow_act = {0};
1220 struct mlx5_flow_handle **flows;
1221 /* total vports is the same for both e-switches */
1222 int nvports = esw->total_vports;
1223 struct mlx5_flow_handle *flow;
1224 struct mlx5_flow_spec *spec;
1225 struct mlx5_vport *vport;
1226 unsigned long i;
1227 void *misc;
1228 int err;
1229
1230 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1231 if (!spec)
1232 return -ENOMEM;
1233
1234 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
1235
1236 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL);
1237 if (!flows) {
1238 err = -ENOMEM;
1239 goto alloc_flows_err;
1240 }
1241
1242 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1243 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1244 misc_parameters);
1245
1246 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1247 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1248 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1249 spec, MLX5_VPORT_PF);
1250
1251 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1252 spec, &flow_act, &dest, 1);
1253 if (IS_ERR(flow)) {
1254 err = PTR_ERR(flow);
1255 goto add_pf_flow_err;
1256 }
1257 flows[vport->index] = flow;
1258 }
1259
1260 if (mlx5_ecpf_vport_exists(esw->dev)) {
1261 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1262 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1263 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1264 spec, &flow_act, &dest, 1);
1265 if (IS_ERR(flow)) {
1266 err = PTR_ERR(flow);
1267 goto add_ecpf_flow_err;
1268 }
1269 flows[vport->index] = flow;
1270 }
1271
1272 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1273 esw_set_peer_miss_rule_source_port(esw,
1274 peer_dev->priv.eswitch,
1275 spec, vport->vport);
1276
1277 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1278 spec, &flow_act, &dest, 1);
1279 if (IS_ERR(flow)) {
1280 err = PTR_ERR(flow);
1281 goto add_vf_flow_err;
1282 }
1283 flows[vport->index] = flow;
1284 }
1285
1286 esw->fdb_table.offloads.peer_miss_rules = flows;
1287
1288 kvfree(spec);
1289 return 0;
1290
1291 add_vf_flow_err:
1292 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1293 if (!flows[vport->index])
1294 continue;
1295 mlx5_del_flow_rules(flows[vport->index]);
1296 }
1297 if (mlx5_ecpf_vport_exists(esw->dev)) {
1298 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1299 mlx5_del_flow_rules(flows[vport->index]);
1300 }
1301 add_ecpf_flow_err:
1302 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1303 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1304 mlx5_del_flow_rules(flows[vport->index]);
1305 }
1306 add_pf_flow_err:
1307 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1308 kvfree(flows);
1309 alloc_flows_err:
1310 kvfree(spec);
1311 return err;
1312 }
1313
esw_del_fdb_peer_miss_rules(struct mlx5_eswitch * esw)1314 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw)
1315 {
1316 struct mlx5_flow_handle **flows;
1317 struct mlx5_vport *vport;
1318 unsigned long i;
1319
1320 flows = esw->fdb_table.offloads.peer_miss_rules;
1321
1322 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1323 mlx5_del_flow_rules(flows[vport->index]);
1324
1325 if (mlx5_ecpf_vport_exists(esw->dev)) {
1326 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1327 mlx5_del_flow_rules(flows[vport->index]);
1328 }
1329
1330 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1331 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1332 mlx5_del_flow_rules(flows[vport->index]);
1333 }
1334 kvfree(flows);
1335 }
1336
esw_add_fdb_miss_rule(struct mlx5_eswitch * esw)1337 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1338 {
1339 struct mlx5_flow_act flow_act = {0};
1340 struct mlx5_flow_destination dest = {};
1341 struct mlx5_flow_handle *flow_rule = NULL;
1342 struct mlx5_flow_spec *spec;
1343 void *headers_c;
1344 void *headers_v;
1345 int err = 0;
1346 u8 *dmac_c;
1347 u8 *dmac_v;
1348
1349 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1350 if (!spec) {
1351 err = -ENOMEM;
1352 goto out;
1353 }
1354
1355 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1356 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1357 outer_headers);
1358 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1359 outer_headers.dmac_47_16);
1360 dmac_c[0] = 0x01;
1361
1362 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1363 dest.vport.num = esw->manager_vport;
1364 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1365
1366 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1367 spec, &flow_act, &dest, 1);
1368 if (IS_ERR(flow_rule)) {
1369 err = PTR_ERR(flow_rule);
1370 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1371 goto out;
1372 }
1373
1374 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1375
1376 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1377 outer_headers);
1378 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1379 outer_headers.dmac_47_16);
1380 dmac_v[0] = 0x01;
1381 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1382 spec, &flow_act, &dest, 1);
1383 if (IS_ERR(flow_rule)) {
1384 err = PTR_ERR(flow_rule);
1385 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1386 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1387 goto out;
1388 }
1389
1390 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1391
1392 out:
1393 kvfree(spec);
1394 return err;
1395 }
1396
1397 struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch * esw,u32 tag)1398 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1399 {
1400 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1401 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1402 struct mlx5_flow_context *flow_context;
1403 struct mlx5_flow_handle *flow_rule;
1404 struct mlx5_flow_destination dest;
1405 struct mlx5_flow_spec *spec;
1406 void *misc;
1407
1408 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1409 return ERR_PTR(-EOPNOTSUPP);
1410
1411 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1412 if (!spec)
1413 return ERR_PTR(-ENOMEM);
1414
1415 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1416 misc_parameters_2);
1417 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1418 ESW_REG_C0_USER_DATA_METADATA_MASK);
1419 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1420 misc_parameters_2);
1421 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1422 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1423 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1424 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1425 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1426
1427 flow_context = &spec->flow_context;
1428 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1429 flow_context->flow_tag = tag;
1430 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1431 dest.ft = esw->offloads.ft_offloads;
1432
1433 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1434 kvfree(spec);
1435
1436 if (IS_ERR(flow_rule))
1437 esw_warn(esw->dev,
1438 "Failed to create restore rule for tag: %d, err(%d)\n",
1439 tag, (int)PTR_ERR(flow_rule));
1440
1441 return flow_rule;
1442 }
1443
1444 #define MAX_PF_SQ 256
1445 #define MAX_SQ_NVPORTS 32
1446
esw_set_flow_group_source_port(struct mlx5_eswitch * esw,u32 * flow_group_in)1447 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1448 u32 *flow_group_in)
1449 {
1450 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1451 flow_group_in,
1452 match_criteria);
1453
1454 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1455 MLX5_SET(create_flow_group_in, flow_group_in,
1456 match_criteria_enable,
1457 MLX5_MATCH_MISC_PARAMETERS_2);
1458
1459 MLX5_SET(fte_match_param, match_criteria,
1460 misc_parameters_2.metadata_reg_c_0,
1461 mlx5_eswitch_get_vport_metadata_mask());
1462 } else {
1463 MLX5_SET(create_flow_group_in, flow_group_in,
1464 match_criteria_enable,
1465 MLX5_MATCH_MISC_PARAMETERS);
1466
1467 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1468 misc_parameters.source_port);
1469 }
1470 }
1471
1472 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
esw_vport_tbl_put(struct mlx5_eswitch * esw)1473 static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1474 {
1475 struct mlx5_vport_tbl_attr attr;
1476 struct mlx5_vport *vport;
1477 unsigned long i;
1478
1479 attr.chain = 0;
1480 attr.prio = 1;
1481 mlx5_esw_for_each_vport(esw, i, vport) {
1482 attr.vport = vport->vport;
1483 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1484 mlx5_esw_vporttbl_put(esw, &attr);
1485 }
1486 }
1487
esw_vport_tbl_get(struct mlx5_eswitch * esw)1488 static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1489 {
1490 struct mlx5_vport_tbl_attr attr;
1491 struct mlx5_flow_table *fdb;
1492 struct mlx5_vport *vport;
1493 unsigned long i;
1494
1495 attr.chain = 0;
1496 attr.prio = 1;
1497 mlx5_esw_for_each_vport(esw, i, vport) {
1498 attr.vport = vport->vport;
1499 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1500 fdb = mlx5_esw_vporttbl_get(esw, &attr);
1501 if (IS_ERR(fdb))
1502 goto out;
1503 }
1504 return 0;
1505
1506 out:
1507 esw_vport_tbl_put(esw);
1508 return PTR_ERR(fdb);
1509 }
1510
1511 #define fdb_modify_header_fwd_to_table_supported(esw) \
1512 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
esw_init_chains_offload_flags(struct mlx5_eswitch * esw,u32 * flags)1513 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1514 {
1515 struct mlx5_core_dev *dev = esw->dev;
1516
1517 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1518 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1519
1520 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1521 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1522 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1523 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1524 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1525 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1526 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1527 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1528 /* Disabled when ttl workaround is needed, e.g
1529 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1530 */
1531 esw_warn(dev,
1532 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1533 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1534 } else {
1535 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1536 esw_info(dev, "Supported tc chains and prios offload\n");
1537 }
1538
1539 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1540 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1541 }
1542
1543 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1544 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1545 {
1546 struct mlx5_core_dev *dev = esw->dev;
1547 struct mlx5_flow_table *nf_ft, *ft;
1548 struct mlx5_chains_attr attr = {};
1549 struct mlx5_fs_chains *chains;
1550 u32 fdb_max;
1551 int err;
1552
1553 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
1554
1555 esw_init_chains_offload_flags(esw, &attr.flags);
1556 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1557 attr.max_ft_sz = fdb_max;
1558 attr.max_grp_num = esw->params.large_group_num;
1559 attr.default_ft = miss_fdb;
1560 attr.mapping = esw->offloads.reg_c0_obj_pool;
1561
1562 chains = mlx5_chains_create(dev, &attr);
1563 if (IS_ERR(chains)) {
1564 err = PTR_ERR(chains);
1565 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1566 return err;
1567 }
1568
1569 esw->fdb_table.offloads.esw_chains_priv = chains;
1570
1571 /* Create tc_end_ft which is the always created ft chain */
1572 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1573 1, 0);
1574 if (IS_ERR(nf_ft)) {
1575 err = PTR_ERR(nf_ft);
1576 goto nf_ft_err;
1577 }
1578
1579 /* Always open the root for fast path */
1580 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1581 if (IS_ERR(ft)) {
1582 err = PTR_ERR(ft);
1583 goto level_0_err;
1584 }
1585
1586 /* Open level 1 for split fdb rules now if prios isn't supported */
1587 if (!mlx5_chains_prios_supported(chains)) {
1588 err = esw_vport_tbl_get(esw);
1589 if (err)
1590 goto level_1_err;
1591 }
1592
1593 mlx5_chains_set_end_ft(chains, nf_ft);
1594
1595 return 0;
1596
1597 level_1_err:
1598 mlx5_chains_put_table(chains, 0, 1, 0);
1599 level_0_err:
1600 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1601 nf_ft_err:
1602 mlx5_chains_destroy(chains);
1603 esw->fdb_table.offloads.esw_chains_priv = NULL;
1604
1605 return err;
1606 }
1607
1608 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1609 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1610 {
1611 if (!mlx5_chains_prios_supported(chains))
1612 esw_vport_tbl_put(esw);
1613 mlx5_chains_put_table(chains, 0, 1, 0);
1614 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1615 mlx5_chains_destroy(chains);
1616 }
1617
1618 #else /* CONFIG_MLX5_CLS_ACT */
1619
1620 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1621 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1622 { return 0; }
1623
1624 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1625 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1626 {}
1627
1628 #endif
1629
1630 static int
esw_create_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1631 esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1632 struct mlx5_flow_table *fdb,
1633 u32 *flow_group_in,
1634 int *ix)
1635 {
1636 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1637 struct mlx5_flow_group *g;
1638 void *match_criteria;
1639 int count, err = 0;
1640
1641 memset(flow_group_in, 0, inlen);
1642
1643 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1644 MLX5_MATCH_MISC_PARAMETERS);
1645
1646 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1647
1648 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1649 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
1650 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1651 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1652 misc_parameters.source_eswitch_owner_vhca_id);
1653 MLX5_SET(create_flow_group_in, flow_group_in,
1654 source_eswitch_owner_vhca_id_valid, 1);
1655 }
1656
1657 /* See comment at table_size calculation */
1658 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1659 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1660 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1661 *ix += count;
1662
1663 g = mlx5_create_flow_group(fdb, flow_group_in);
1664 if (IS_ERR(g)) {
1665 err = PTR_ERR(g);
1666 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1667 goto out;
1668 }
1669 esw->fdb_table.offloads.send_to_vport_grp = g;
1670
1671 out:
1672 return err;
1673 }
1674
1675 static int
esw_create_meta_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1676 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1677 struct mlx5_flow_table *fdb,
1678 u32 *flow_group_in,
1679 int *ix)
1680 {
1681 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1682 struct mlx5_flow_group *g;
1683 void *match_criteria;
1684 int err = 0;
1685
1686 if (!esw_src_port_rewrite_supported(esw))
1687 return 0;
1688
1689 memset(flow_group_in, 0, inlen);
1690
1691 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1692 MLX5_MATCH_MISC_PARAMETERS_2);
1693
1694 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1695
1696 MLX5_SET(fte_match_param, match_criteria,
1697 misc_parameters_2.metadata_reg_c_0,
1698 mlx5_eswitch_get_vport_metadata_mask());
1699 MLX5_SET(fte_match_param, match_criteria,
1700 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1701
1702 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1703 MLX5_SET(create_flow_group_in, flow_group_in,
1704 end_flow_index, *ix + esw->total_vports - 1);
1705 *ix += esw->total_vports;
1706
1707 g = mlx5_create_flow_group(fdb, flow_group_in);
1708 if (IS_ERR(g)) {
1709 err = PTR_ERR(g);
1710 esw_warn(esw->dev,
1711 "Failed to create send-to-vport meta flow group err(%d)\n", err);
1712 goto send_vport_meta_err;
1713 }
1714 esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1715
1716 return 0;
1717
1718 send_vport_meta_err:
1719 return err;
1720 }
1721
1722 static int
esw_create_peer_esw_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1723 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1724 struct mlx5_flow_table *fdb,
1725 u32 *flow_group_in,
1726 int *ix)
1727 {
1728 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1729 struct mlx5_flow_group *g;
1730 void *match_criteria;
1731 int err = 0;
1732
1733 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1734 return 0;
1735
1736 memset(flow_group_in, 0, inlen);
1737
1738 esw_set_flow_group_source_port(esw, flow_group_in);
1739
1740 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1741 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1742 flow_group_in,
1743 match_criteria);
1744
1745 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1746 misc_parameters.source_eswitch_owner_vhca_id);
1747
1748 MLX5_SET(create_flow_group_in, flow_group_in,
1749 source_eswitch_owner_vhca_id_valid, 1);
1750 }
1751
1752 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1753 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1754 *ix + esw->total_vports - 1);
1755 *ix += esw->total_vports;
1756
1757 g = mlx5_create_flow_group(fdb, flow_group_in);
1758 if (IS_ERR(g)) {
1759 err = PTR_ERR(g);
1760 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1761 goto out;
1762 }
1763 esw->fdb_table.offloads.peer_miss_grp = g;
1764
1765 out:
1766 return err;
1767 }
1768
1769 static int
esw_create_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1770 esw_create_miss_group(struct mlx5_eswitch *esw,
1771 struct mlx5_flow_table *fdb,
1772 u32 *flow_group_in,
1773 int *ix)
1774 {
1775 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1776 struct mlx5_flow_group *g;
1777 void *match_criteria;
1778 int err = 0;
1779 u8 *dmac;
1780
1781 memset(flow_group_in, 0, inlen);
1782
1783 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1784 MLX5_MATCH_OUTER_HEADERS);
1785 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1786 match_criteria);
1787 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1788 outer_headers.dmac_47_16);
1789 dmac[0] = 0x01;
1790
1791 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1792 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1793 *ix + MLX5_ESW_MISS_FLOWS);
1794
1795 g = mlx5_create_flow_group(fdb, flow_group_in);
1796 if (IS_ERR(g)) {
1797 err = PTR_ERR(g);
1798 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1799 goto miss_err;
1800 }
1801 esw->fdb_table.offloads.miss_grp = g;
1802
1803 err = esw_add_fdb_miss_rule(esw);
1804 if (err)
1805 goto miss_rule_err;
1806
1807 return 0;
1808
1809 miss_rule_err:
1810 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1811 miss_err:
1812 return err;
1813 }
1814
esw_create_offloads_fdb_tables(struct mlx5_eswitch * esw)1815 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1816 {
1817 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1818 struct mlx5_flow_table_attr ft_attr = {};
1819 struct mlx5_core_dev *dev = esw->dev;
1820 struct mlx5_flow_namespace *root_ns;
1821 struct mlx5_flow_table *fdb = NULL;
1822 int table_size, ix = 0, err = 0;
1823 u32 flags = 0, *flow_group_in;
1824
1825 esw_debug(esw->dev, "Create offloads FDB Tables\n");
1826
1827 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1828 if (!flow_group_in)
1829 return -ENOMEM;
1830
1831 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1832 if (!root_ns) {
1833 esw_warn(dev, "Failed to get FDB flow namespace\n");
1834 err = -EOPNOTSUPP;
1835 goto ns_err;
1836 }
1837 esw->fdb_table.offloads.ns = root_ns;
1838 err = mlx5_flow_namespace_set_mode(root_ns,
1839 esw->dev->priv.steering->mode);
1840 if (err) {
1841 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1842 goto ns_err;
1843 }
1844
1845 /* To be strictly correct:
1846 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1847 * should be:
1848 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1849 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1850 * but as the peer device might not be in switchdev mode it's not
1851 * possible. We use the fact that by default FW sets max vfs and max sfs
1852 * to the same value on both devices. If it needs to be changed in the future note
1853 * the peer miss group should also be created based on the number of
1854 * total vports of the peer (currently is also uses esw->total_vports).
1855 */
1856 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1857 esw->total_vports * 2 + MLX5_ESW_MISS_FLOWS;
1858
1859 /* create the slow path fdb with encap set, so further table instances
1860 * can be created at run time while VFs are probed if the FW allows that.
1861 */
1862 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1863 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1864 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1865
1866 ft_attr.flags = flags;
1867 ft_attr.max_fte = table_size;
1868 ft_attr.prio = FDB_SLOW_PATH;
1869
1870 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1871 if (IS_ERR(fdb)) {
1872 err = PTR_ERR(fdb);
1873 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1874 goto slow_fdb_err;
1875 }
1876 esw->fdb_table.offloads.slow_fdb = fdb;
1877
1878 /* Create empty TC-miss managed table. This allows plugging in following
1879 * priorities without directly exposing their level 0 table to
1880 * eswitch_offloads and passing it as miss_fdb to following call to
1881 * esw_chains_create().
1882 */
1883 memset(&ft_attr, 0, sizeof(ft_attr));
1884 ft_attr.prio = FDB_TC_MISS;
1885 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1886 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1887 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1888 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1889 goto tc_miss_table_err;
1890 }
1891
1892 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
1893 if (err) {
1894 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1895 goto fdb_chains_err;
1896 }
1897
1898 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1899 if (err)
1900 goto send_vport_err;
1901
1902 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1903 if (err)
1904 goto send_vport_meta_err;
1905
1906 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix);
1907 if (err)
1908 goto peer_miss_err;
1909
1910 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix);
1911 if (err)
1912 goto miss_err;
1913
1914 kvfree(flow_group_in);
1915 return 0;
1916
1917 miss_err:
1918 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1919 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1920 peer_miss_err:
1921 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1922 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1923 send_vport_meta_err:
1924 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1925 send_vport_err:
1926 esw_chains_destroy(esw, esw_chains(esw));
1927 fdb_chains_err:
1928 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1929 tc_miss_table_err:
1930 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1931 slow_fdb_err:
1932 /* Holds true only as long as DMFS is the default */
1933 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
1934 ns_err:
1935 kvfree(flow_group_in);
1936 return err;
1937 }
1938
esw_destroy_offloads_fdb_tables(struct mlx5_eswitch * esw)1939 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1940 {
1941 if (!esw->fdb_table.offloads.slow_fdb)
1942 return;
1943
1944 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1945 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1946 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1947 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1948 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1949 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1950 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1951 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1952 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1953
1954 esw_chains_destroy(esw, esw_chains(esw));
1955
1956 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1957 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1958 /* Holds true only as long as DMFS is the default */
1959 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1960 MLX5_FLOW_STEERING_MODE_DMFS);
1961 atomic64_set(&esw->user_count, 0);
1962 }
1963
esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch * esw)1964 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
1965 {
1966 int nvports;
1967
1968 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1969 if (mlx5e_tc_int_port_supported(esw))
1970 nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1971
1972 return nvports;
1973 }
1974
esw_create_offloads_table(struct mlx5_eswitch * esw)1975 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1976 {
1977 struct mlx5_flow_table_attr ft_attr = {};
1978 struct mlx5_core_dev *dev = esw->dev;
1979 struct mlx5_flow_table *ft_offloads;
1980 struct mlx5_flow_namespace *ns;
1981 int err = 0;
1982
1983 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1984 if (!ns) {
1985 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1986 return -EOPNOTSUPP;
1987 }
1988
1989 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
1990 MLX5_ESW_FT_OFFLOADS_DROP_RULE;
1991 ft_attr.prio = 1;
1992
1993 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
1994 if (IS_ERR(ft_offloads)) {
1995 err = PTR_ERR(ft_offloads);
1996 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
1997 return err;
1998 }
1999
2000 esw->offloads.ft_offloads = ft_offloads;
2001 return 0;
2002 }
2003
esw_destroy_offloads_table(struct mlx5_eswitch * esw)2004 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2005 {
2006 struct mlx5_esw_offload *offloads = &esw->offloads;
2007
2008 mlx5_destroy_flow_table(offloads->ft_offloads);
2009 }
2010
esw_create_vport_rx_group(struct mlx5_eswitch * esw)2011 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2012 {
2013 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2014 struct mlx5_flow_group *g;
2015 u32 *flow_group_in;
2016 int nvports;
2017 int err = 0;
2018
2019 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2020 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2021 if (!flow_group_in)
2022 return -ENOMEM;
2023
2024 /* create vport rx group */
2025 esw_set_flow_group_source_port(esw, flow_group_in);
2026
2027 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2028 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2029
2030 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2031
2032 if (IS_ERR(g)) {
2033 err = PTR_ERR(g);
2034 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
2035 goto out;
2036 }
2037
2038 esw->offloads.vport_rx_group = g;
2039 out:
2040 kvfree(flow_group_in);
2041 return err;
2042 }
2043
esw_destroy_vport_rx_group(struct mlx5_eswitch * esw)2044 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2045 {
2046 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
2047 }
2048
esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch * esw)2049 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2050 {
2051 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2052 * for the drop rule, which is placed at the end of the table.
2053 * So return the total of vport and int_port as rule index.
2054 */
2055 return esw_get_nr_ft_offloads_steering_src_ports(esw);
2056 }
2057
esw_create_vport_rx_drop_group(struct mlx5_eswitch * esw)2058 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2059 {
2060 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2061 struct mlx5_flow_group *g;
2062 u32 *flow_group_in;
2063 int flow_index;
2064 int err = 0;
2065
2066 flow_index = esw_create_vport_rx_drop_rule_index(esw);
2067
2068 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2069 if (!flow_group_in)
2070 return -ENOMEM;
2071
2072 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2073 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2074
2075 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2076
2077 if (IS_ERR(g)) {
2078 err = PTR_ERR(g);
2079 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
2080 goto out;
2081 }
2082
2083 esw->offloads.vport_rx_drop_group = g;
2084 out:
2085 kvfree(flow_group_in);
2086 return err;
2087 }
2088
esw_destroy_vport_rx_drop_group(struct mlx5_eswitch * esw)2089 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2090 {
2091 if (esw->offloads.vport_rx_drop_group)
2092 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group);
2093 }
2094
2095 struct mlx5_flow_handle *
mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_destination * dest)2096 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2097 struct mlx5_flow_destination *dest)
2098 {
2099 struct mlx5_flow_act flow_act = {0};
2100 struct mlx5_flow_handle *flow_rule;
2101 struct mlx5_flow_spec *spec;
2102 void *misc;
2103
2104 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2105 if (!spec) {
2106 flow_rule = ERR_PTR(-ENOMEM);
2107 goto out;
2108 }
2109
2110 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2111 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2112 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2113 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2114
2115 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2116 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2117 mlx5_eswitch_get_vport_metadata_mask());
2118
2119 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2120 } else {
2121 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2122 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2123
2124 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2125 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2126
2127 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2128 }
2129
2130 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2131 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
2132 &flow_act, dest, 1);
2133 if (IS_ERR(flow_rule)) {
2134 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
2135 goto out;
2136 }
2137
2138 out:
2139 kvfree(spec);
2140 return flow_rule;
2141 }
2142
esw_create_vport_rx_drop_rule(struct mlx5_eswitch * esw)2143 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2144 {
2145 struct mlx5_flow_act flow_act = {};
2146 struct mlx5_flow_handle *flow_rule;
2147
2148 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2149 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL,
2150 &flow_act, NULL, 0);
2151 if (IS_ERR(flow_rule)) {
2152 esw_warn(esw->dev,
2153 "fs offloads: Failed to add vport rx drop rule err %ld\n",
2154 PTR_ERR(flow_rule));
2155 return PTR_ERR(flow_rule);
2156 }
2157
2158 esw->offloads.vport_rx_drop_rule = flow_rule;
2159
2160 return 0;
2161 }
2162
esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch * esw)2163 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2164 {
2165 if (esw->offloads.vport_rx_drop_rule)
2166 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule);
2167 }
2168
mlx5_eswitch_inline_mode_get(struct mlx5_eswitch * esw,u8 * mode)2169 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2170 {
2171 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2172 struct mlx5_core_dev *dev = esw->dev;
2173 struct mlx5_vport *vport;
2174 unsigned long i;
2175
2176 if (!MLX5_CAP_GEN(dev, vport_group_manager))
2177 return -EOPNOTSUPP;
2178
2179 if (!mlx5_esw_is_fdb_created(esw))
2180 return -EOPNOTSUPP;
2181
2182 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2183 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2184 mlx5_mode = MLX5_INLINE_MODE_NONE;
2185 goto out;
2186 case MLX5_CAP_INLINE_MODE_L2:
2187 mlx5_mode = MLX5_INLINE_MODE_L2;
2188 goto out;
2189 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2190 goto query_vports;
2191 }
2192
2193 query_vports:
2194 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
2195 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2196 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
2197 if (prev_mlx5_mode != mlx5_mode)
2198 return -EINVAL;
2199 prev_mlx5_mode = mlx5_mode;
2200 }
2201
2202 out:
2203 *mode = mlx5_mode;
2204 return 0;
2205 }
2206
esw_destroy_restore_table(struct mlx5_eswitch * esw)2207 static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2208 {
2209 struct mlx5_esw_offload *offloads = &esw->offloads;
2210
2211 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2212 return;
2213
2214 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
2215 mlx5_destroy_flow_group(offloads->restore_group);
2216 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2217 }
2218
esw_create_restore_table(struct mlx5_eswitch * esw)2219 static int esw_create_restore_table(struct mlx5_eswitch *esw)
2220 {
2221 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2222 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2223 struct mlx5_flow_table_attr ft_attr = {};
2224 struct mlx5_core_dev *dev = esw->dev;
2225 struct mlx5_flow_namespace *ns;
2226 struct mlx5_modify_hdr *mod_hdr;
2227 void *match_criteria, *misc;
2228 struct mlx5_flow_table *ft;
2229 struct mlx5_flow_group *g;
2230 u32 *flow_group_in;
2231 int err = 0;
2232
2233 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2234 return 0;
2235
2236 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2237 if (!ns) {
2238 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2239 return -EOPNOTSUPP;
2240 }
2241
2242 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2243 if (!flow_group_in) {
2244 err = -ENOMEM;
2245 goto out_free;
2246 }
2247
2248 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2249 ft = mlx5_create_flow_table(ns, &ft_attr);
2250 if (IS_ERR(ft)) {
2251 err = PTR_ERR(ft);
2252 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2253 err);
2254 goto out_free;
2255 }
2256
2257 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2258 match_criteria);
2259 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2260 misc_parameters_2);
2261
2262 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2263 ESW_REG_C0_USER_DATA_METADATA_MASK);
2264 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2265 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2266 ft_attr.max_fte - 1);
2267 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2268 MLX5_MATCH_MISC_PARAMETERS_2);
2269 g = mlx5_create_flow_group(ft, flow_group_in);
2270 if (IS_ERR(g)) {
2271 err = PTR_ERR(g);
2272 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2273 err);
2274 goto err_group;
2275 }
2276
2277 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2278 MLX5_SET(copy_action_in, modact, src_field,
2279 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2280 MLX5_SET(copy_action_in, modact, dst_field,
2281 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2282 mod_hdr = mlx5_modify_header_alloc(esw->dev,
2283 MLX5_FLOW_NAMESPACE_KERNEL, 1,
2284 modact);
2285 if (IS_ERR(mod_hdr)) {
2286 err = PTR_ERR(mod_hdr);
2287 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2288 err);
2289 goto err_mod_hdr;
2290 }
2291
2292 esw->offloads.ft_offloads_restore = ft;
2293 esw->offloads.restore_group = g;
2294 esw->offloads.restore_copy_hdr_id = mod_hdr;
2295
2296 kvfree(flow_group_in);
2297
2298 return 0;
2299
2300 err_mod_hdr:
2301 mlx5_destroy_flow_group(g);
2302 err_group:
2303 mlx5_destroy_flow_table(ft);
2304 out_free:
2305 kvfree(flow_group_in);
2306
2307 return err;
2308 }
2309
esw_offloads_start(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)2310 static int esw_offloads_start(struct mlx5_eswitch *esw,
2311 struct netlink_ext_ack *extack)
2312 {
2313 int err;
2314
2315 esw->mode = MLX5_ESWITCH_OFFLOADS;
2316 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs);
2317 if (err) {
2318 NL_SET_ERR_MSG_MOD(extack,
2319 "Failed setting eswitch to offloads");
2320 esw->mode = MLX5_ESWITCH_LEGACY;
2321 mlx5_rescan_drivers(esw->dev);
2322 }
2323 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2324 if (mlx5_eswitch_inline_mode_get(esw,
2325 &esw->offloads.inline_mode)) {
2326 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2327 NL_SET_ERR_MSG_MOD(extack,
2328 "Inline mode is different between vports");
2329 }
2330 }
2331 return err;
2332 }
2333
mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,xa_mark_t mark)2334 static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch *esw,
2335 struct mlx5_eswitch_rep *rep,
2336 xa_mark_t mark)
2337 {
2338 bool mark_set;
2339
2340 /* Copy the mark from vport to its rep */
2341 mark_set = xa_get_mark(&esw->vports, rep->vport, mark);
2342 if (mark_set)
2343 xa_set_mark(&esw->offloads.vport_reps, rep->vport, mark);
2344 }
2345
mlx5_esw_offloads_rep_init(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2346 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
2347 {
2348 struct mlx5_eswitch_rep *rep;
2349 int rep_type;
2350 int err;
2351
2352 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2353 if (!rep)
2354 return -ENOMEM;
2355
2356 rep->vport = vport->vport;
2357 rep->vport_index = vport->index;
2358 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2359 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
2360
2361 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2362 if (err)
2363 goto insert_err;
2364
2365 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_HOST_FN);
2366 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_VF);
2367 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_SF);
2368 return 0;
2369
2370 insert_err:
2371 kfree(rep);
2372 return err;
2373 }
2374
mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep)2375 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2376 struct mlx5_eswitch_rep *rep)
2377 {
2378 xa_erase(&esw->offloads.vport_reps, rep->vport);
2379 kfree(rep);
2380 }
2381
esw_offloads_cleanup_reps(struct mlx5_eswitch * esw)2382 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2383 {
2384 struct mlx5_eswitch_rep *rep;
2385 unsigned long i;
2386
2387 mlx5_esw_for_each_rep(esw, i, rep)
2388 mlx5_esw_offloads_rep_cleanup(esw, rep);
2389 xa_destroy(&esw->offloads.vport_reps);
2390 }
2391
esw_offloads_init_reps(struct mlx5_eswitch * esw)2392 int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2393 {
2394 struct mlx5_vport *vport;
2395 unsigned long i;
2396 int err;
2397
2398 xa_init(&esw->offloads.vport_reps);
2399
2400 mlx5_esw_for_each_vport(esw, i, vport) {
2401 err = mlx5_esw_offloads_rep_init(esw, vport);
2402 if (err)
2403 goto err;
2404 }
2405 return 0;
2406
2407 err:
2408 esw_offloads_cleanup_reps(esw);
2409 return err;
2410 }
2411
__esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2412 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2413 struct mlx5_eswitch_rep *rep, u8 rep_type)
2414 {
2415 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2416 REP_LOADED, REP_REGISTERED) == REP_LOADED)
2417 esw->offloads.rep_ops[rep_type]->unload(rep);
2418 }
2419
__unload_reps_sf_vport(struct mlx5_eswitch * esw,u8 rep_type)2420 static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type)
2421 {
2422 struct mlx5_eswitch_rep *rep;
2423 unsigned long i;
2424
2425 mlx5_esw_for_each_sf_rep(esw, i, rep)
2426 __esw_offloads_unload_rep(esw, rep, rep_type);
2427 }
2428
__unload_reps_all_vport(struct mlx5_eswitch * esw,u8 rep_type)2429 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2430 {
2431 struct mlx5_eswitch_rep *rep;
2432 unsigned long i;
2433
2434 __unload_reps_sf_vport(esw, rep_type);
2435
2436 mlx5_esw_for_each_vf_rep(esw, i, rep)
2437 __esw_offloads_unload_rep(esw, rep, rep_type);
2438
2439 if (mlx5_ecpf_vport_exists(esw->dev)) {
2440 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF);
2441 __esw_offloads_unload_rep(esw, rep, rep_type);
2442 }
2443
2444 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
2445 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF);
2446 __esw_offloads_unload_rep(esw, rep, rep_type);
2447 }
2448
2449 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
2450 __esw_offloads_unload_rep(esw, rep, rep_type);
2451 }
2452
mlx5_esw_offloads_rep_load(struct mlx5_eswitch * esw,u16 vport_num)2453 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2454 {
2455 struct mlx5_eswitch_rep *rep;
2456 int rep_type;
2457 int err;
2458
2459 rep = mlx5_eswitch_get_rep(esw, vport_num);
2460 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2461 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2462 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) {
2463 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2464 if (err)
2465 goto err_reps;
2466 }
2467
2468 return 0;
2469
2470 err_reps:
2471 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2472 for (--rep_type; rep_type >= 0; rep_type--)
2473 __esw_offloads_unload_rep(esw, rep, rep_type);
2474 return err;
2475 }
2476
mlx5_esw_offloads_rep_unload(struct mlx5_eswitch * esw,u16 vport_num)2477 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2478 {
2479 struct mlx5_eswitch_rep *rep;
2480 int rep_type;
2481
2482 rep = mlx5_eswitch_get_rep(esw, vport_num);
2483 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2484 __esw_offloads_unload_rep(esw, rep, rep_type);
2485 }
2486
esw_offloads_load_rep(struct mlx5_eswitch * esw,u16 vport_num)2487 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num)
2488 {
2489 int err;
2490
2491 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2492 return 0;
2493
2494 if (vport_num != MLX5_VPORT_UPLINK) {
2495 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num);
2496 if (err)
2497 return err;
2498 }
2499
2500 err = mlx5_esw_offloads_rep_load(esw, vport_num);
2501 if (err)
2502 goto load_err;
2503 return err;
2504
2505 load_err:
2506 if (vport_num != MLX5_VPORT_UPLINK)
2507 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
2508 return err;
2509 }
2510
esw_offloads_unload_rep(struct mlx5_eswitch * esw,u16 vport_num)2511 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num)
2512 {
2513 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2514 return;
2515
2516 mlx5_esw_offloads_rep_unload(esw, vport_num);
2517
2518 if (vport_num != MLX5_VPORT_UPLINK)
2519 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
2520 }
2521
esw_set_slave_root_fdb(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2522 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2523 struct mlx5_core_dev *slave)
2524 {
2525 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2526 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2527 struct mlx5_flow_root_namespace *root;
2528 struct mlx5_flow_namespace *ns;
2529 int err;
2530
2531 MLX5_SET(set_flow_table_root_in, in, opcode,
2532 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2533 MLX5_SET(set_flow_table_root_in, in, table_type,
2534 FS_FT_FDB);
2535
2536 if (master) {
2537 ns = mlx5_get_flow_namespace(master,
2538 MLX5_FLOW_NAMESPACE_FDB);
2539 root = find_root(&ns->node);
2540 mutex_lock(&root->chain_lock);
2541 MLX5_SET(set_flow_table_root_in, in,
2542 table_eswitch_owner_vhca_id_valid, 1);
2543 MLX5_SET(set_flow_table_root_in, in,
2544 table_eswitch_owner_vhca_id,
2545 MLX5_CAP_GEN(master, vhca_id));
2546 MLX5_SET(set_flow_table_root_in, in, table_id,
2547 root->root_ft->id);
2548 } else {
2549 ns = mlx5_get_flow_namespace(slave,
2550 MLX5_FLOW_NAMESPACE_FDB);
2551 root = find_root(&ns->node);
2552 mutex_lock(&root->chain_lock);
2553 MLX5_SET(set_flow_table_root_in, in, table_id,
2554 root->root_ft->id);
2555 }
2556
2557 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2558 mutex_unlock(&root->chain_lock);
2559
2560 return err;
2561 }
2562
__esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,struct mlx5_vport * vport,struct mlx5_flow_table * acl)2563 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2564 struct mlx5_core_dev *slave,
2565 struct mlx5_vport *vport,
2566 struct mlx5_flow_table *acl)
2567 {
2568 struct mlx5_flow_handle *flow_rule = NULL;
2569 struct mlx5_flow_destination dest = {};
2570 struct mlx5_flow_act flow_act = {};
2571 struct mlx5_flow_spec *spec;
2572 int err = 0;
2573 void *misc;
2574
2575 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2576 if (!spec)
2577 return -ENOMEM;
2578
2579 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2580 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2581 misc_parameters);
2582 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2583 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
2584 MLX5_CAP_GEN(slave, vhca_id));
2585
2586 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2587 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2588 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2589 source_eswitch_owner_vhca_id);
2590
2591 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2592 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2593 dest.vport.num = slave->priv.eswitch->manager_vport;
2594 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2595 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2596
2597 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2598 &dest, 1);
2599 if (IS_ERR(flow_rule))
2600 err = PTR_ERR(flow_rule);
2601 else
2602 vport->egress.offloads.bounce_rule = flow_rule;
2603
2604 kvfree(spec);
2605 return err;
2606 }
2607
esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2608 static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2609 struct mlx5_core_dev *slave)
2610 {
2611 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2612 struct mlx5_eswitch *esw = master->priv.eswitch;
2613 struct mlx5_flow_table_attr ft_attr = {
2614 .max_fte = 1, .prio = 0, .level = 0,
2615 .flags = MLX5_FLOW_TABLE_OTHER_VPORT,
2616 };
2617 struct mlx5_flow_namespace *egress_ns;
2618 struct mlx5_flow_table *acl;
2619 struct mlx5_flow_group *g;
2620 struct mlx5_vport *vport;
2621 void *match_criteria;
2622 u32 *flow_group_in;
2623 int err;
2624
2625 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
2626 if (IS_ERR(vport))
2627 return PTR_ERR(vport);
2628
2629 egress_ns = mlx5_get_flow_vport_acl_namespace(master,
2630 MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2631 vport->index);
2632 if (!egress_ns)
2633 return -EINVAL;
2634
2635 if (vport->egress.acl)
2636 return -EINVAL;
2637
2638 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2639 if (!flow_group_in)
2640 return -ENOMEM;
2641
2642 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2643 if (IS_ERR(acl)) {
2644 err = PTR_ERR(acl);
2645 goto out;
2646 }
2647
2648 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2649 match_criteria);
2650 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2651 misc_parameters.source_port);
2652 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2653 misc_parameters.source_eswitch_owner_vhca_id);
2654 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2655 MLX5_MATCH_MISC_PARAMETERS);
2656
2657 MLX5_SET(create_flow_group_in, flow_group_in,
2658 source_eswitch_owner_vhca_id_valid, 1);
2659 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2660 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0);
2661
2662 g = mlx5_create_flow_group(acl, flow_group_in);
2663 if (IS_ERR(g)) {
2664 err = PTR_ERR(g);
2665 goto err_group;
2666 }
2667
2668 err = __esw_set_master_egress_rule(master, slave, vport, acl);
2669 if (err)
2670 goto err_rule;
2671
2672 vport->egress.acl = acl;
2673 vport->egress.offloads.bounce_grp = g;
2674
2675 kvfree(flow_group_in);
2676
2677 return 0;
2678
2679 err_rule:
2680 mlx5_destroy_flow_group(g);
2681 err_group:
2682 mlx5_destroy_flow_table(acl);
2683 out:
2684 kvfree(flow_group_in);
2685 return err;
2686 }
2687
esw_unset_master_egress_rule(struct mlx5_core_dev * dev)2688 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev)
2689 {
2690 struct mlx5_vport *vport;
2691
2692 vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
2693 dev->priv.eswitch->manager_vport);
2694
2695 esw_acl_egress_ofld_cleanup(vport);
2696 }
2697
mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2698 int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch *master_esw,
2699 struct mlx5_eswitch *slave_esw)
2700 {
2701 int err;
2702
2703 err = esw_set_slave_root_fdb(master_esw->dev,
2704 slave_esw->dev);
2705 if (err)
2706 return err;
2707
2708 err = esw_set_master_egress_rule(master_esw->dev,
2709 slave_esw->dev);
2710 if (err)
2711 goto err_acl;
2712
2713 return err;
2714
2715 err_acl:
2716 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2717
2718 return err;
2719 }
2720
mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2721 void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw,
2722 struct mlx5_eswitch *slave_esw)
2723 {
2724 esw_unset_master_egress_rule(master_esw->dev);
2725 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2726 }
2727
2728 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
2729 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2730
mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch * esw)2731 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw)
2732 {
2733 const struct mlx5_eswitch_rep_ops *ops;
2734 struct mlx5_eswitch_rep *rep;
2735 unsigned long i;
2736 u8 rep_type;
2737
2738 mlx5_esw_for_each_rep(esw, i, rep) {
2739 rep_type = NUM_REP_TYPES;
2740 while (rep_type--) {
2741 ops = esw->offloads.rep_ops[rep_type];
2742 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2743 ops->event)
2744 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, NULL);
2745 }
2746 }
2747 }
2748
mlx5_esw_offloads_unpair(struct mlx5_eswitch * esw)2749 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw)
2750 {
2751 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2752 mlx5e_tc_clean_fdb_peer_flows(esw);
2753 #endif
2754 mlx5_esw_offloads_rep_event_unpair(esw);
2755 esw_del_fdb_peer_miss_rules(esw);
2756 }
2757
mlx5_esw_offloads_pair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2758 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2759 struct mlx5_eswitch *peer_esw)
2760 {
2761 const struct mlx5_eswitch_rep_ops *ops;
2762 struct mlx5_eswitch_rep *rep;
2763 unsigned long i;
2764 u8 rep_type;
2765 int err;
2766
2767 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2768 if (err)
2769 return err;
2770
2771 mlx5_esw_for_each_rep(esw, i, rep) {
2772 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2773 ops = esw->offloads.rep_ops[rep_type];
2774 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2775 ops->event) {
2776 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2777 if (err)
2778 goto err_out;
2779 }
2780 }
2781 }
2782
2783 return 0;
2784
2785 err_out:
2786 mlx5_esw_offloads_unpair(esw);
2787 return err;
2788 }
2789
mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,bool pair)2790 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2791 struct mlx5_eswitch *peer_esw,
2792 bool pair)
2793 {
2794 struct mlx5_flow_root_namespace *peer_ns;
2795 struct mlx5_flow_root_namespace *ns;
2796 int err;
2797
2798 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2799 ns = esw->dev->priv.steering->fdb_root_ns;
2800
2801 if (pair) {
2802 err = mlx5_flow_namespace_set_peer(ns, peer_ns);
2803 if (err)
2804 return err;
2805
2806 err = mlx5_flow_namespace_set_peer(peer_ns, ns);
2807 if (err) {
2808 mlx5_flow_namespace_set_peer(ns, NULL);
2809 return err;
2810 }
2811 } else {
2812 mlx5_flow_namespace_set_peer(ns, NULL);
2813 mlx5_flow_namespace_set_peer(peer_ns, NULL);
2814 }
2815
2816 return 0;
2817 }
2818
mlx5_esw_offloads_devcom_event(int event,void * my_data,void * event_data)2819 static int mlx5_esw_offloads_devcom_event(int event,
2820 void *my_data,
2821 void *event_data)
2822 {
2823 struct mlx5_eswitch *esw = my_data;
2824 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2825 struct mlx5_eswitch *peer_esw = event_data;
2826 int err;
2827
2828 switch (event) {
2829 case ESW_OFFLOADS_DEVCOM_PAIR:
2830 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2831 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2832 break;
2833
2834 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
2835 if (err)
2836 goto err_out;
2837 err = mlx5_esw_offloads_pair(esw, peer_esw);
2838 if (err)
2839 goto err_peer;
2840
2841 err = mlx5_esw_offloads_pair(peer_esw, esw);
2842 if (err)
2843 goto err_pair;
2844
2845 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true);
2846 break;
2847
2848 case ESW_OFFLOADS_DEVCOM_UNPAIR:
2849 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS))
2850 break;
2851
2852 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false);
2853 mlx5_esw_offloads_unpair(peer_esw);
2854 mlx5_esw_offloads_unpair(esw);
2855 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
2856 break;
2857 }
2858
2859 return 0;
2860
2861 err_pair:
2862 mlx5_esw_offloads_unpair(esw);
2863 err_peer:
2864 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
2865 err_out:
2866 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
2867 event, err);
2868 return err;
2869 }
2870
esw_offloads_devcom_init(struct mlx5_eswitch * esw)2871 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw)
2872 {
2873 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2874
2875 INIT_LIST_HEAD(&esw->offloads.peer_flows);
2876 mutex_init(&esw->offloads.peer_mutex);
2877
2878 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2879 return;
2880
2881 if (!mlx5_is_lag_supported(esw->dev))
2882 return;
2883
2884 mlx5_devcom_register_component(devcom,
2885 MLX5_DEVCOM_ESW_OFFLOADS,
2886 mlx5_esw_offloads_devcom_event,
2887 esw);
2888
2889 mlx5_devcom_send_event(devcom,
2890 MLX5_DEVCOM_ESW_OFFLOADS,
2891 ESW_OFFLOADS_DEVCOM_PAIR, esw);
2892 }
2893
esw_offloads_devcom_cleanup(struct mlx5_eswitch * esw)2894 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
2895 {
2896 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2897
2898 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2899 return;
2900
2901 if (!mlx5_is_lag_supported(esw->dev))
2902 return;
2903
2904 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS,
2905 ESW_OFFLOADS_DEVCOM_UNPAIR, esw);
2906
2907 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
2908 }
2909
mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch * esw)2910 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
2911 {
2912 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
2913 return false;
2914
2915 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
2916 MLX5_FDB_TO_VPORT_REG_C_0))
2917 return false;
2918
2919 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
2920 return false;
2921
2922 return true;
2923 }
2924
2925 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
2926
2927 /* Share the same metadata for uplink's. This is fine because:
2928 * (a) In shared FDB mode (LAG) both uplink's are treated the
2929 * same and tagged with the same metadata.
2930 * (b) In non shared FDB mode, packets from physical port0
2931 * cannot hit eswitch of PF1 and vice versa.
2932 */
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch * esw)2933 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
2934 {
2935 return MLX5_ESW_METADATA_RSVD_UPLINK;
2936 }
2937
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch * esw)2938 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
2939 {
2940 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
2941 /* Reserve 0xf for internal port offload */
2942 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
2943 u32 pf_num;
2944 int id;
2945
2946 /* Only 4 bits of pf_num */
2947 pf_num = mlx5_get_dev_index(esw->dev);
2948 if (pf_num > max_pf_num)
2949 return 0;
2950
2951 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
2952 /* Use only non-zero vport_id (2-4095) for all PF's */
2953 id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
2954 MLX5_ESW_METADATA_RSVD_UPLINK + 1,
2955 vport_end_ida, GFP_KERNEL);
2956 if (id < 0)
2957 return 0;
2958 id = (pf_num << ESW_VPORT_BITS) | id;
2959 return id;
2960 }
2961
mlx5_esw_match_metadata_free(struct mlx5_eswitch * esw,u32 metadata)2962 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
2963 {
2964 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
2965
2966 /* Metadata contains only 12 bits of actual ida id */
2967 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
2968 }
2969
esw_offloads_vport_metadata_setup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2970 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
2971 struct mlx5_vport *vport)
2972 {
2973 if (vport->vport == MLX5_VPORT_UPLINK)
2974 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
2975 else
2976 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
2977
2978 vport->metadata = vport->default_metadata;
2979 return vport->metadata ? 0 : -ENOSPC;
2980 }
2981
esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2982 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
2983 struct mlx5_vport *vport)
2984 {
2985 if (!vport->default_metadata)
2986 return;
2987
2988 if (vport->vport == MLX5_VPORT_UPLINK)
2989 return;
2990
2991 WARN_ON(vport->metadata != vport->default_metadata);
2992 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
2993 }
2994
esw_offloads_metadata_uninit(struct mlx5_eswitch * esw)2995 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
2996 {
2997 struct mlx5_vport *vport;
2998 unsigned long i;
2999
3000 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3001 return;
3002
3003 mlx5_esw_for_each_vport(esw, i, vport)
3004 esw_offloads_vport_metadata_cleanup(esw, vport);
3005 }
3006
esw_offloads_metadata_init(struct mlx5_eswitch * esw)3007 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3008 {
3009 struct mlx5_vport *vport;
3010 unsigned long i;
3011 int err;
3012
3013 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3014 return 0;
3015
3016 mlx5_esw_for_each_vport(esw, i, vport) {
3017 err = esw_offloads_vport_metadata_setup(esw, vport);
3018 if (err)
3019 goto metadata_err;
3020 }
3021
3022 return 0;
3023
3024 metadata_err:
3025 esw_offloads_metadata_uninit(esw);
3026 return err;
3027 }
3028
mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch * esw,bool enable)3029 int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable)
3030 {
3031 int err = 0;
3032
3033 down_write(&esw->mode_lock);
3034 if (mlx5_esw_is_fdb_created(esw)) {
3035 err = -EBUSY;
3036 goto done;
3037 }
3038 if (!mlx5_esw_vport_match_metadata_supported(esw)) {
3039 err = -EOPNOTSUPP;
3040 goto done;
3041 }
3042 if (enable)
3043 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
3044 else
3045 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
3046 done:
3047 up_write(&esw->mode_lock);
3048 return err;
3049 }
3050
3051 int
esw_vport_create_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3052 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3053 struct mlx5_vport *vport)
3054 {
3055 int err;
3056
3057 err = esw_acl_ingress_ofld_setup(esw, vport);
3058 if (err)
3059 return err;
3060
3061 err = esw_acl_egress_ofld_setup(esw, vport);
3062 if (err)
3063 goto egress_err;
3064
3065 return 0;
3066
3067 egress_err:
3068 esw_acl_ingress_ofld_cleanup(esw, vport);
3069 return err;
3070 }
3071
3072 void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3073 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3074 struct mlx5_vport *vport)
3075 {
3076 esw_acl_egress_ofld_cleanup(vport);
3077 esw_acl_ingress_ofld_cleanup(esw, vport);
3078 }
3079
esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch * esw)3080 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
3081 {
3082 struct mlx5_vport *vport;
3083
3084 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3085 if (IS_ERR(vport))
3086 return PTR_ERR(vport);
3087
3088 return esw_vport_create_offloads_acl_tables(esw, vport);
3089 }
3090
esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch * esw)3091 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
3092 {
3093 struct mlx5_vport *vport;
3094
3095 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3096 if (IS_ERR(vport))
3097 return;
3098
3099 esw_vport_destroy_offloads_acl_tables(esw, vport);
3100 }
3101
mlx5_eswitch_reload_reps(struct mlx5_eswitch * esw)3102 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
3103 {
3104 struct mlx5_eswitch_rep *rep;
3105 unsigned long i;
3106 int ret;
3107
3108 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3109 return 0;
3110
3111 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3112 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
3113 return 0;
3114
3115 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
3116 if (ret)
3117 return ret;
3118
3119 mlx5_esw_for_each_rep(esw, i, rep) {
3120 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
3121 mlx5_esw_offloads_rep_load(esw, rep->vport);
3122 }
3123
3124 return 0;
3125 }
3126
esw_offloads_steering_init(struct mlx5_eswitch * esw)3127 static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3128 {
3129 struct mlx5_esw_indir_table *indir;
3130 int err;
3131
3132 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3133 mutex_init(&esw->fdb_table.offloads.vports.lock);
3134 hash_init(esw->fdb_table.offloads.vports.table);
3135 atomic64_set(&esw->user_count, 0);
3136
3137 indir = mlx5_esw_indir_table_init();
3138 if (IS_ERR(indir)) {
3139 err = PTR_ERR(indir);
3140 goto create_indir_err;
3141 }
3142 esw->fdb_table.offloads.indir = indir;
3143
3144 err = esw_create_uplink_offloads_acl_tables(esw);
3145 if (err)
3146 goto create_acl_err;
3147
3148 err = esw_create_offloads_table(esw);
3149 if (err)
3150 goto create_offloads_err;
3151
3152 err = esw_create_restore_table(esw);
3153 if (err)
3154 goto create_restore_err;
3155
3156 err = esw_create_offloads_fdb_tables(esw);
3157 if (err)
3158 goto create_fdb_err;
3159
3160 err = esw_create_vport_rx_group(esw);
3161 if (err)
3162 goto create_fg_err;
3163
3164 err = esw_create_vport_rx_drop_group(esw);
3165 if (err)
3166 goto create_rx_drop_fg_err;
3167
3168 err = esw_create_vport_rx_drop_rule(esw);
3169 if (err)
3170 goto create_rx_drop_rule_err;
3171
3172 return 0;
3173
3174 create_rx_drop_rule_err:
3175 esw_destroy_vport_rx_drop_group(esw);
3176 create_rx_drop_fg_err:
3177 esw_destroy_vport_rx_group(esw);
3178 create_fg_err:
3179 esw_destroy_offloads_fdb_tables(esw);
3180 create_fdb_err:
3181 esw_destroy_restore_table(esw);
3182 create_restore_err:
3183 esw_destroy_offloads_table(esw);
3184 create_offloads_err:
3185 esw_destroy_uplink_offloads_acl_tables(esw);
3186 create_acl_err:
3187 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3188 create_indir_err:
3189 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3190 return err;
3191 }
3192
esw_offloads_steering_cleanup(struct mlx5_eswitch * esw)3193 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3194 {
3195 esw_destroy_vport_rx_drop_rule(esw);
3196 esw_destroy_vport_rx_drop_group(esw);
3197 esw_destroy_vport_rx_group(esw);
3198 esw_destroy_offloads_fdb_tables(esw);
3199 esw_destroy_restore_table(esw);
3200 esw_destroy_offloads_table(esw);
3201 esw_destroy_uplink_offloads_acl_tables(esw);
3202 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3203 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3204 }
3205
3206 static void
esw_vfs_changed_event_handler(struct mlx5_eswitch * esw,const u32 * out)3207 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3208 {
3209 struct devlink *devlink;
3210 bool host_pf_disabled;
3211 u16 new_num_vfs;
3212
3213 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3214 host_params_context.host_num_of_vfs);
3215 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3216 host_params_context.host_pf_disabled);
3217
3218 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3219 return;
3220
3221 devlink = priv_to_devlink(esw->dev);
3222 devl_lock(devlink);
3223 /* Number of VFs can only change from "0 to x" or "x to 0". */
3224 if (esw->esw_funcs.num_vfs > 0) {
3225 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
3226 } else {
3227 int err;
3228
3229 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3230 MLX5_VPORT_UC_ADDR_CHANGE);
3231 if (err) {
3232 devl_unlock(devlink);
3233 return;
3234 }
3235 }
3236 esw->esw_funcs.num_vfs = new_num_vfs;
3237 devl_unlock(devlink);
3238 }
3239
esw_functions_changed_event_handler(struct work_struct * work)3240 static void esw_functions_changed_event_handler(struct work_struct *work)
3241 {
3242 struct mlx5_host_work *host_work;
3243 struct mlx5_eswitch *esw;
3244 const u32 *out;
3245
3246 host_work = container_of(work, struct mlx5_host_work, work);
3247 esw = host_work->esw;
3248
3249 out = mlx5_esw_query_functions(esw->dev);
3250 if (IS_ERR(out))
3251 goto out;
3252
3253 esw_vfs_changed_event_handler(esw, out);
3254 kvfree(out);
3255 out:
3256 kfree(host_work);
3257 }
3258
mlx5_esw_funcs_changed_handler(struct notifier_block * nb,unsigned long type,void * data)3259 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3260 {
3261 struct mlx5_esw_functions *esw_funcs;
3262 struct mlx5_host_work *host_work;
3263 struct mlx5_eswitch *esw;
3264
3265 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3266 if (!host_work)
3267 return NOTIFY_DONE;
3268
3269 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3270 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3271
3272 host_work->esw = esw;
3273
3274 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3275 queue_work(esw->work_queue, &host_work->work);
3276
3277 return NOTIFY_OK;
3278 }
3279
mlx5_esw_host_number_init(struct mlx5_eswitch * esw)3280 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3281 {
3282 const u32 *query_host_out;
3283
3284 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3285 return 0;
3286
3287 query_host_out = mlx5_esw_query_functions(esw->dev);
3288 if (IS_ERR(query_host_out))
3289 return PTR_ERR(query_host_out);
3290
3291 /* Mark non local controller with non zero controller number. */
3292 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3293 host_params_context.host_number);
3294 kvfree(query_host_out);
3295 return 0;
3296 }
3297
mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch * esw,u32 controller)3298 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3299 {
3300 /* Local controller is always valid */
3301 if (controller == 0)
3302 return true;
3303
3304 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3305 return false;
3306
3307 /* External host number starts with zero in device */
3308 return (controller == esw->offloads.host_number + 1);
3309 }
3310
esw_offloads_enable(struct mlx5_eswitch * esw)3311 int esw_offloads_enable(struct mlx5_eswitch *esw)
3312 {
3313 struct mapping_ctx *reg_c0_obj_pool;
3314 struct mlx5_vport *vport;
3315 unsigned long i;
3316 u64 mapping_id;
3317 int err;
3318
3319 mutex_init(&esw->offloads.termtbl_mutex);
3320 mlx5_rdma_enable_roce(esw->dev);
3321
3322 err = mlx5_esw_host_number_init(esw);
3323 if (err)
3324 goto err_metadata;
3325
3326 err = esw_offloads_metadata_init(esw);
3327 if (err)
3328 goto err_metadata;
3329
3330 err = esw_set_passing_vport_metadata(esw, true);
3331 if (err)
3332 goto err_vport_metadata;
3333
3334 mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
3335
3336 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN,
3337 sizeof(struct mlx5_mapped_obj),
3338 ESW_REG_C0_USER_DATA_METADATA_MASK,
3339 true);
3340
3341 if (IS_ERR(reg_c0_obj_pool)) {
3342 err = PTR_ERR(reg_c0_obj_pool);
3343 goto err_pool;
3344 }
3345 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3346
3347 err = esw_offloads_steering_init(esw);
3348 if (err)
3349 goto err_steering_init;
3350
3351 /* Representor will control the vport link state */
3352 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3353 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3354
3355 /* Uplink vport rep must load first. */
3356 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK);
3357 if (err)
3358 goto err_uplink;
3359
3360 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
3361 if (err)
3362 goto err_vports;
3363
3364 esw_offloads_devcom_init(esw);
3365
3366 return 0;
3367
3368 err_vports:
3369 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
3370 err_uplink:
3371 esw_offloads_steering_cleanup(esw);
3372 err_steering_init:
3373 mapping_destroy(reg_c0_obj_pool);
3374 err_pool:
3375 esw_set_passing_vport_metadata(esw, false);
3376 err_vport_metadata:
3377 esw_offloads_metadata_uninit(esw);
3378 err_metadata:
3379 mlx5_rdma_disable_roce(esw->dev);
3380 mutex_destroy(&esw->offloads.termtbl_mutex);
3381 return err;
3382 }
3383
esw_offloads_stop(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)3384 static int esw_offloads_stop(struct mlx5_eswitch *esw,
3385 struct netlink_ext_ack *extack)
3386 {
3387 int err;
3388
3389 esw->mode = MLX5_ESWITCH_LEGACY;
3390
3391 /* If changing from switchdev to legacy mode without sriov enabled,
3392 * no need to create legacy fdb.
3393 */
3394 if (!mlx5_sriov_is_enabled(esw->dev))
3395 return 0;
3396
3397 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3398 if (err)
3399 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3400
3401 return err;
3402 }
3403
esw_offloads_disable(struct mlx5_eswitch * esw)3404 void esw_offloads_disable(struct mlx5_eswitch *esw)
3405 {
3406 esw_offloads_devcom_cleanup(esw);
3407 mlx5_eswitch_disable_pf_vf_vports(esw);
3408 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
3409 esw_set_passing_vport_metadata(esw, false);
3410 esw_offloads_steering_cleanup(esw);
3411 mapping_destroy(esw->offloads.reg_c0_obj_pool);
3412 esw_offloads_metadata_uninit(esw);
3413 mlx5_rdma_disable_roce(esw->dev);
3414 mutex_destroy(&esw->offloads.termtbl_mutex);
3415 }
3416
esw_mode_from_devlink(u16 mode,u16 * mlx5_mode)3417 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3418 {
3419 switch (mode) {
3420 case DEVLINK_ESWITCH_MODE_LEGACY:
3421 *mlx5_mode = MLX5_ESWITCH_LEGACY;
3422 break;
3423 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3424 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3425 break;
3426 default:
3427 return -EINVAL;
3428 }
3429
3430 return 0;
3431 }
3432
esw_mode_to_devlink(u16 mlx5_mode,u16 * mode)3433 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3434 {
3435 switch (mlx5_mode) {
3436 case MLX5_ESWITCH_LEGACY:
3437 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
3438 break;
3439 case MLX5_ESWITCH_OFFLOADS:
3440 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3441 break;
3442 default:
3443 return -EINVAL;
3444 }
3445
3446 return 0;
3447 }
3448
esw_inline_mode_from_devlink(u8 mode,u8 * mlx5_mode)3449 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3450 {
3451 switch (mode) {
3452 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3453 *mlx5_mode = MLX5_INLINE_MODE_NONE;
3454 break;
3455 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3456 *mlx5_mode = MLX5_INLINE_MODE_L2;
3457 break;
3458 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3459 *mlx5_mode = MLX5_INLINE_MODE_IP;
3460 break;
3461 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3462 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3463 break;
3464 default:
3465 return -EINVAL;
3466 }
3467
3468 return 0;
3469 }
3470
esw_inline_mode_to_devlink(u8 mlx5_mode,u8 * mode)3471 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3472 {
3473 switch (mlx5_mode) {
3474 case MLX5_INLINE_MODE_NONE:
3475 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3476 break;
3477 case MLX5_INLINE_MODE_L2:
3478 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3479 break;
3480 case MLX5_INLINE_MODE_IP:
3481 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3482 break;
3483 case MLX5_INLINE_MODE_TCP_UDP:
3484 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3485 break;
3486 default:
3487 return -EINVAL;
3488 }
3489
3490 return 0;
3491 }
3492
mlx5_devlink_eswitch_mode_set(struct devlink * devlink,u16 mode,struct netlink_ext_ack * extack)3493 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3494 struct netlink_ext_ack *extack)
3495 {
3496 u16 cur_mlx5_mode, mlx5_mode = 0;
3497 struct mlx5_eswitch *esw;
3498 int err = 0;
3499
3500 esw = mlx5_devlink_eswitch_get(devlink);
3501 if (IS_ERR(esw))
3502 return PTR_ERR(esw);
3503
3504 if (esw_mode_from_devlink(mode, &mlx5_mode))
3505 return -EINVAL;
3506
3507 mlx5_lag_disable_change(esw->dev);
3508 err = mlx5_esw_try_lock(esw);
3509 if (err < 0) {
3510 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
3511 goto enable_lag;
3512 }
3513 cur_mlx5_mode = err;
3514 err = 0;
3515
3516 if (cur_mlx5_mode == mlx5_mode)
3517 goto unlock;
3518
3519 mlx5_eswitch_disable_locked(esw);
3520 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3521 if (mlx5_devlink_trap_get_num_active(esw->dev)) {
3522 NL_SET_ERR_MSG_MOD(extack,
3523 "Can't change mode while devlink traps are active");
3524 err = -EOPNOTSUPP;
3525 goto unlock;
3526 }
3527 err = esw_offloads_start(esw, extack);
3528 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
3529 err = esw_offloads_stop(esw, extack);
3530 mlx5_rescan_drivers(esw->dev);
3531 } else {
3532 err = -EINVAL;
3533 }
3534
3535 unlock:
3536 mlx5_esw_unlock(esw);
3537 enable_lag:
3538 mlx5_lag_enable_change(esw->dev);
3539 return err;
3540 }
3541
mlx5_devlink_eswitch_mode_get(struct devlink * devlink,u16 * mode)3542 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3543 {
3544 struct mlx5_eswitch *esw;
3545 int err;
3546
3547 esw = mlx5_devlink_eswitch_get(devlink);
3548 if (IS_ERR(esw))
3549 return PTR_ERR(esw);
3550
3551 down_write(&esw->mode_lock);
3552 err = esw_mode_to_devlink(esw->mode, mode);
3553 up_write(&esw->mode_lock);
3554 return err;
3555 }
3556
mlx5_esw_vports_inline_set(struct mlx5_eswitch * esw,u8 mlx5_mode,struct netlink_ext_ack * extack)3557 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3558 struct netlink_ext_ack *extack)
3559 {
3560 struct mlx5_core_dev *dev = esw->dev;
3561 struct mlx5_vport *vport;
3562 u16 err_vport_num = 0;
3563 unsigned long i;
3564 int err = 0;
3565
3566 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3567 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3568 if (err) {
3569 err_vport_num = vport->vport;
3570 NL_SET_ERR_MSG_MOD(extack,
3571 "Failed to set min inline on vport");
3572 goto revert_inline_mode;
3573 }
3574 }
3575 return 0;
3576
3577 revert_inline_mode:
3578 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3579 if (vport->vport == err_vport_num)
3580 break;
3581 mlx5_modify_nic_vport_min_inline(dev,
3582 vport->vport,
3583 esw->offloads.inline_mode);
3584 }
3585 return err;
3586 }
3587
mlx5_devlink_eswitch_inline_mode_set(struct devlink * devlink,u8 mode,struct netlink_ext_ack * extack)3588 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3589 struct netlink_ext_ack *extack)
3590 {
3591 struct mlx5_core_dev *dev = devlink_priv(devlink);
3592 struct mlx5_eswitch *esw;
3593 u8 mlx5_mode;
3594 int err;
3595
3596 esw = mlx5_devlink_eswitch_get(devlink);
3597 if (IS_ERR(esw))
3598 return PTR_ERR(esw);
3599
3600 down_write(&esw->mode_lock);
3601
3602 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3603 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3604 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3605 err = 0;
3606 goto out;
3607 }
3608
3609 fallthrough;
3610 case MLX5_CAP_INLINE_MODE_L2:
3611 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
3612 err = -EOPNOTSUPP;
3613 goto out;
3614 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3615 break;
3616 }
3617
3618 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3619 NL_SET_ERR_MSG_MOD(extack,
3620 "Can't set inline mode when flows are configured");
3621 err = -EOPNOTSUPP;
3622 goto out;
3623 }
3624
3625 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
3626 if (err)
3627 goto out;
3628
3629 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3630 if (err)
3631 goto out;
3632
3633 esw->offloads.inline_mode = mlx5_mode;
3634 up_write(&esw->mode_lock);
3635 return 0;
3636
3637 out:
3638 up_write(&esw->mode_lock);
3639 return err;
3640 }
3641
mlx5_devlink_eswitch_inline_mode_get(struct devlink * devlink,u8 * mode)3642 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3643 {
3644 struct mlx5_eswitch *esw;
3645 int err;
3646
3647 esw = mlx5_devlink_eswitch_get(devlink);
3648 if (IS_ERR(esw))
3649 return PTR_ERR(esw);
3650
3651 down_write(&esw->mode_lock);
3652 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3653 up_write(&esw->mode_lock);
3654 return err;
3655 }
3656
mlx5_devlink_eswitch_encap_mode_set(struct devlink * devlink,enum devlink_eswitch_encap_mode encap,struct netlink_ext_ack * extack)3657 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3658 enum devlink_eswitch_encap_mode encap,
3659 struct netlink_ext_ack *extack)
3660 {
3661 struct mlx5_core_dev *dev = devlink_priv(devlink);
3662 struct mlx5_eswitch *esw;
3663 int err = 0;
3664
3665 esw = mlx5_devlink_eswitch_get(devlink);
3666 if (IS_ERR(esw))
3667 return PTR_ERR(esw);
3668
3669 down_write(&esw->mode_lock);
3670
3671 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
3672 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
3673 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3674 err = -EOPNOTSUPP;
3675 goto unlock;
3676 }
3677
3678 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3679 err = -EOPNOTSUPP;
3680 goto unlock;
3681 }
3682
3683 if (esw->mode == MLX5_ESWITCH_LEGACY) {
3684 esw->offloads.encap = encap;
3685 goto unlock;
3686 }
3687
3688 if (esw->offloads.encap == encap)
3689 goto unlock;
3690
3691 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3692 NL_SET_ERR_MSG_MOD(extack,
3693 "Can't set encapsulation when flows are configured");
3694 err = -EOPNOTSUPP;
3695 goto unlock;
3696 }
3697
3698 esw_destroy_offloads_fdb_tables(esw);
3699
3700 esw->offloads.encap = encap;
3701
3702 err = esw_create_offloads_fdb_tables(esw);
3703
3704 if (err) {
3705 NL_SET_ERR_MSG_MOD(extack,
3706 "Failed re-creating fast FDB table");
3707 esw->offloads.encap = !encap;
3708 (void)esw_create_offloads_fdb_tables(esw);
3709 }
3710
3711 unlock:
3712 up_write(&esw->mode_lock);
3713 return err;
3714 }
3715
mlx5_devlink_eswitch_encap_mode_get(struct devlink * devlink,enum devlink_eswitch_encap_mode * encap)3716 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
3717 enum devlink_eswitch_encap_mode *encap)
3718 {
3719 struct mlx5_eswitch *esw;
3720
3721 esw = mlx5_devlink_eswitch_get(devlink);
3722 if (IS_ERR(esw))
3723 return PTR_ERR(esw);
3724
3725 down_write(&esw->mode_lock);
3726 *encap = esw->offloads.encap;
3727 up_write(&esw->mode_lock);
3728 return 0;
3729 }
3730
3731 static bool
mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch * esw,u16 vport_num)3732 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
3733 {
3734 /* Currently, only ECPF based device has representor for host PF. */
3735 if (vport_num == MLX5_VPORT_PF &&
3736 !mlx5_core_is_ecpf_esw_manager(esw->dev))
3737 return false;
3738
3739 if (vport_num == MLX5_VPORT_ECPF &&
3740 !mlx5_ecpf_vport_exists(esw->dev))
3741 return false;
3742
3743 return true;
3744 }
3745
mlx5_eswitch_register_vport_reps(struct mlx5_eswitch * esw,const struct mlx5_eswitch_rep_ops * ops,u8 rep_type)3746 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
3747 const struct mlx5_eswitch_rep_ops *ops,
3748 u8 rep_type)
3749 {
3750 struct mlx5_eswitch_rep_data *rep_data;
3751 struct mlx5_eswitch_rep *rep;
3752 unsigned long i;
3753
3754 esw->offloads.rep_ops[rep_type] = ops;
3755 mlx5_esw_for_each_rep(esw, i, rep) {
3756 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
3757 rep->esw = esw;
3758 rep_data = &rep->rep_data[rep_type];
3759 atomic_set(&rep_data->state, REP_REGISTERED);
3760 }
3761 }
3762 }
3763 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
3764
mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch * esw,u8 rep_type)3765 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
3766 {
3767 struct mlx5_eswitch_rep *rep;
3768 unsigned long i;
3769
3770 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
3771 __unload_reps_all_vport(esw, rep_type);
3772
3773 mlx5_esw_for_each_rep(esw, i, rep)
3774 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
3775 }
3776 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
3777
mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch * esw,u8 rep_type)3778 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
3779 {
3780 struct mlx5_eswitch_rep *rep;
3781
3782 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3783 return rep->rep_data[rep_type].priv;
3784 }
3785
mlx5_eswitch_get_proto_dev(struct mlx5_eswitch * esw,u16 vport,u8 rep_type)3786 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
3787 u16 vport,
3788 u8 rep_type)
3789 {
3790 struct mlx5_eswitch_rep *rep;
3791
3792 rep = mlx5_eswitch_get_rep(esw, vport);
3793
3794 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
3795 esw->offloads.rep_ops[rep_type]->get_proto_dev)
3796 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
3797 return NULL;
3798 }
3799 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
3800
mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch * esw,u8 rep_type)3801 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
3802 {
3803 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
3804 }
3805 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
3806
mlx5_eswitch_vport_rep(struct mlx5_eswitch * esw,u16 vport)3807 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
3808 u16 vport)
3809 {
3810 return mlx5_eswitch_get_rep(esw, vport);
3811 }
3812 EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
3813
mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch * esw)3814 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
3815 {
3816 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
3817 }
3818 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
3819
mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch * esw)3820 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
3821 {
3822 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
3823 }
3824 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
3825
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch * esw,u16 vport_num)3826 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
3827 u16 vport_num)
3828 {
3829 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3830
3831 if (WARN_ON_ONCE(IS_ERR(vport)))
3832 return 0;
3833
3834 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
3835 }
3836 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
3837
mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch * esw,struct devlink_port * dl_port,u16 vport_num,u32 controller,u32 sfnum)3838 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port,
3839 u16 vport_num, u32 controller, u32 sfnum)
3840 {
3841 int err;
3842
3843 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE);
3844 if (err)
3845 return err;
3846
3847 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, controller, sfnum);
3848 if (err)
3849 goto devlink_err;
3850
3851 mlx5_esw_vport_debugfs_create(esw, vport_num, true, sfnum);
3852 err = mlx5_esw_offloads_rep_load(esw, vport_num);
3853 if (err)
3854 goto rep_err;
3855 return 0;
3856
3857 rep_err:
3858 mlx5_esw_vport_debugfs_destroy(esw, vport_num);
3859 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3860 devlink_err:
3861 mlx5_esw_vport_disable(esw, vport_num);
3862 return err;
3863 }
3864
mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch * esw,u16 vport_num)3865 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num)
3866 {
3867 mlx5_esw_offloads_rep_unload(esw, vport_num);
3868 mlx5_esw_vport_debugfs_destroy(esw, vport_num);
3869 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3870 mlx5_esw_vport_disable(esw, vport_num);
3871 }
3872
mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch * esw,u16 vport_num,u16 * vhca_id)3873 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
3874 {
3875 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
3876 void *query_ctx;
3877 void *hca_caps;
3878 int err;
3879
3880 *vhca_id = 0;
3881 if (mlx5_esw_is_manager_vport(esw, vport_num) ||
3882 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
3883 return -EPERM;
3884
3885 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
3886 if (!query_ctx)
3887 return -ENOMEM;
3888
3889 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx);
3890 if (err)
3891 goto out_free;
3892
3893 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
3894 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
3895
3896 out_free:
3897 kfree(query_ctx);
3898 return err;
3899 }
3900
mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch * esw,u16 vport_num)3901 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
3902 {
3903 u16 *old_entry, *vhca_map_entry, vhca_id;
3904 int err;
3905
3906 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3907 if (err) {
3908 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
3909 vport_num, err);
3910 return err;
3911 }
3912
3913 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
3914 if (!vhca_map_entry)
3915 return -ENOMEM;
3916
3917 *vhca_map_entry = vport_num;
3918 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
3919 if (xa_is_err(old_entry)) {
3920 kfree(vhca_map_entry);
3921 return xa_err(old_entry);
3922 }
3923 kfree(old_entry);
3924 return 0;
3925 }
3926
mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch * esw,u16 vport_num)3927 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
3928 {
3929 u16 *vhca_map_entry, vhca_id;
3930 int err;
3931
3932 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3933 if (err)
3934 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
3935 vport_num, err);
3936
3937 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
3938 kfree(vhca_map_entry);
3939 }
3940
mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch * esw,u16 vhca_id,u16 * vport_num)3941 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
3942 {
3943 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
3944
3945 if (!res)
3946 return -ENOENT;
3947
3948 *vport_num = *res;
3949 return 0;
3950 }
3951
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch * esw,u16 vport_num)3952 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
3953 u16 vport_num)
3954 {
3955 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3956
3957 if (WARN_ON_ONCE(IS_ERR(vport)))
3958 return 0;
3959
3960 return vport->metadata;
3961 }
3962 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
3963
3964 static bool
is_port_function_supported(struct mlx5_eswitch * esw,u16 vport_num)3965 is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num)
3966 {
3967 return vport_num == MLX5_VPORT_PF ||
3968 mlx5_eswitch_is_vf_vport(esw, vport_num) ||
3969 mlx5_esw_is_sf_vport(esw, vport_num);
3970 }
3971
mlx5_devlink_port_function_hw_addr_get(struct devlink_port * port,u8 * hw_addr,int * hw_addr_len,struct netlink_ext_ack * extack)3972 int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port,
3973 u8 *hw_addr, int *hw_addr_len,
3974 struct netlink_ext_ack *extack)
3975 {
3976 struct mlx5_eswitch *esw;
3977 struct mlx5_vport *vport;
3978 u16 vport_num;
3979
3980 esw = mlx5_devlink_eswitch_get(port->devlink);
3981 if (IS_ERR(esw))
3982 return PTR_ERR(esw);
3983
3984 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
3985 if (!is_port_function_supported(esw, vport_num))
3986 return -EOPNOTSUPP;
3987
3988 vport = mlx5_eswitch_get_vport(esw, vport_num);
3989 if (IS_ERR(vport)) {
3990 NL_SET_ERR_MSG_MOD(extack, "Invalid port");
3991 return PTR_ERR(vport);
3992 }
3993
3994 mutex_lock(&esw->state_lock);
3995 ether_addr_copy(hw_addr, vport->info.mac);
3996 *hw_addr_len = ETH_ALEN;
3997 mutex_unlock(&esw->state_lock);
3998 return 0;
3999 }
4000
mlx5_devlink_port_function_hw_addr_set(struct devlink_port * port,const u8 * hw_addr,int hw_addr_len,struct netlink_ext_ack * extack)4001 int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port,
4002 const u8 *hw_addr, int hw_addr_len,
4003 struct netlink_ext_ack *extack)
4004 {
4005 struct mlx5_eswitch *esw;
4006 u16 vport_num;
4007
4008 esw = mlx5_devlink_eswitch_get(port->devlink);
4009 if (IS_ERR(esw)) {
4010 NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr");
4011 return PTR_ERR(esw);
4012 }
4013
4014 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4015 if (!is_port_function_supported(esw, vport_num)) {
4016 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr");
4017 return -EINVAL;
4018 }
4019
4020 return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr);
4021 }
4022