1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
43 #include "rdma.h"
44 #include "en.h"
45 #include "fs_core.h"
46 #include "lib/devcom.h"
47 #include "lib/eq.h"
48 #include "lib/fs_chains.h"
49 #include "en_tc.h"
50 #include "en/mapping.h"
51 #include "devlink.h"
52 #include "lag/lag.h"
53
54 #define mlx5_esw_for_each_rep(esw, i, rep) \
55 xa_for_each(&((esw)->offloads.vport_reps), i, rep)
56
57 #define mlx5_esw_for_each_sf_rep(esw, i, rep) \
58 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF)
59
60 #define mlx5_esw_for_each_vf_rep(esw, index, rep) \
61 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \
62 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF)
63
64 /* There are two match-all miss flows, one for unicast dst mac and
65 * one for multicast.
66 */
67 #define MLX5_ESW_MISS_FLOWS (2)
68 #define UPLINK_REP_INDEX 0
69
70 #define MLX5_ESW_VPORT_TBL_SIZE 128
71 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
72
73 static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
74 .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
75 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
76 .flags = 0,
77 };
78
mlx5_eswitch_get_rep(struct mlx5_eswitch * esw,u16 vport_num)79 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
80 u16 vport_num)
81 {
82 return xa_load(&esw->offloads.vport_reps, vport_num);
83 }
84
85 static void
mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr)86 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
87 struct mlx5_flow_spec *spec,
88 struct mlx5_esw_flow_attr *attr)
89 {
90 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
91 return;
92
93 if (attr->int_port) {
94 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port);
95
96 return;
97 }
98
99 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
100 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
101 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
102 }
103
104 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
105 * are not needed as well in the following process. So clear them all for simplicity.
106 */
107 void
mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec)108 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
109 {
110 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
111 void *misc2;
112
113 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
114 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
115
116 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
117 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
118
119 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
120 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
121 }
122 }
123
124 static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct mlx5_eswitch * src_esw,u16 vport)125 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
126 struct mlx5_flow_spec *spec,
127 struct mlx5_flow_attr *attr,
128 struct mlx5_eswitch *src_esw,
129 u16 vport)
130 {
131 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
132 u32 metadata;
133 void *misc2;
134 void *misc;
135
136 /* Use metadata matching because vport is not represented by single
137 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
138 */
139 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
140 if (mlx5_esw_indir_table_decap_vport(attr))
141 vport = mlx5_esw_indir_table_decap_vport(attr);
142
143 if (attr && !attr->chain && esw_attr->int_port)
144 metadata =
145 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port);
146 else
147 metadata =
148 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport);
149
150 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
151 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
152
153 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
154 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
155 mlx5_eswitch_get_vport_metadata_mask());
156
157 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
158 } else {
159 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
160 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
161
162 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
163 MLX5_SET(fte_match_set_misc, misc,
164 source_eswitch_owner_vhca_id,
165 MLX5_CAP_GEN(src_esw->dev, vhca_id));
166
167 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
168 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
169 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
170 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
171 source_eswitch_owner_vhca_id);
172
173 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
174 }
175 }
176
177 static int
esw_setup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec)178 esw_setup_decap_indir(struct mlx5_eswitch *esw,
179 struct mlx5_flow_attr *attr,
180 struct mlx5_flow_spec *spec)
181 {
182 struct mlx5_flow_table *ft;
183
184 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
185 return -EOPNOTSUPP;
186
187 ft = mlx5_esw_indir_table_get(esw, attr, spec,
188 mlx5_esw_indir_table_decap_vport(attr), true);
189 return PTR_ERR_OR_ZERO(ft);
190 }
191
192 static void
esw_cleanup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)193 esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
194 struct mlx5_flow_attr *attr)
195 {
196 if (mlx5_esw_indir_table_decap_vport(attr))
197 mlx5_esw_indir_table_put(esw, attr,
198 mlx5_esw_indir_table_decap_vport(attr),
199 true);
200 }
201
202 static int
esw_setup_sampler_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,u32 sampler_id,int i)203 esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
204 struct mlx5_flow_act *flow_act,
205 u32 sampler_id,
206 int i)
207 {
208 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
209 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
210 dest[i].sampler_id = sampler_id;
211
212 return 0;
213 }
214
215 static int
esw_setup_ft_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int i)216 esw_setup_ft_dest(struct mlx5_flow_destination *dest,
217 struct mlx5_flow_act *flow_act,
218 struct mlx5_eswitch *esw,
219 struct mlx5_flow_attr *attr,
220 struct mlx5_flow_spec *spec,
221 int i)
222 {
223 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
224 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
225 dest[i].ft = attr->dest_ft;
226
227 if (mlx5_esw_indir_table_decap_vport(attr))
228 return esw_setup_decap_indir(esw, attr, spec);
229 return 0;
230 }
231
232 static void
esw_setup_accept_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,int i)233 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
234 struct mlx5_fs_chains *chains, int i)
235 {
236 if (mlx5_chains_ignore_flow_level_supported(chains))
237 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
238 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
239 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
240 }
241
242 static void
esw_setup_slow_path_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,int i)243 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
244 struct mlx5_eswitch *esw, int i)
245 {
246 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
247 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
248 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
249 dest[i].ft = esw->fdb_table.offloads.slow_fdb;
250 }
251
252 static int
esw_setup_chain_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level,int i)253 esw_setup_chain_dest(struct mlx5_flow_destination *dest,
254 struct mlx5_flow_act *flow_act,
255 struct mlx5_fs_chains *chains,
256 u32 chain, u32 prio, u32 level,
257 int i)
258 {
259 struct mlx5_flow_table *ft;
260
261 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
262 ft = mlx5_chains_get_table(chains, chain, prio, level);
263 if (IS_ERR(ft))
264 return PTR_ERR(ft);
265
266 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
267 dest[i].ft = ft;
268 return 0;
269 }
270
esw_put_dest_tables_loop(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int from,int to)271 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
272 int from, int to)
273 {
274 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
275 struct mlx5_fs_chains *chains = esw_chains(esw);
276 int i;
277
278 for (i = from; i < to; i++)
279 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
280 mlx5_chains_put_table(chains, 0, 1, 0);
281 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
282 esw_attr->dests[i].mdev))
283 mlx5_esw_indir_table_put(esw, attr, esw_attr->dests[i].rep->vport,
284 false);
285 }
286
287 static bool
esw_is_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)288 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
289 {
290 int i;
291
292 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
293 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
294 return true;
295 return false;
296 }
297
298 static int
esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains,struct mlx5_flow_attr * attr,int * i)299 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
300 struct mlx5_flow_act *flow_act,
301 struct mlx5_eswitch *esw,
302 struct mlx5_fs_chains *chains,
303 struct mlx5_flow_attr *attr,
304 int *i)
305 {
306 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
307 int err;
308
309 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
310 return -EOPNOTSUPP;
311
312 /* flow steering cannot handle more than one dest with the same ft
313 * in a single flow
314 */
315 if (esw_attr->out_count - esw_attr->split_count > 1)
316 return -EOPNOTSUPP;
317
318 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
319 if (err)
320 return err;
321
322 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
323 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
324 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
325 }
326 (*i)++;
327
328 return 0;
329 }
330
esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)331 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
332 struct mlx5_flow_attr *attr)
333 {
334 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
335
336 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
337 }
338
339 static bool
esw_is_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)340 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
341 {
342 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
343 bool result = false;
344 int i;
345
346 /* Indirect table is supported only for flows with in_port uplink
347 * and the destination is vport on the same eswitch as the uplink,
348 * return false in case at least one of destinations doesn't meet
349 * this criteria.
350 */
351 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
352 if (esw_attr->dests[i].rep &&
353 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
354 esw_attr->dests[i].mdev)) {
355 result = true;
356 } else {
357 result = false;
358 break;
359 }
360 }
361 return result;
362 }
363
364 static int
esw_setup_indir_table(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,bool ignore_flow_lvl,int * i)365 esw_setup_indir_table(struct mlx5_flow_destination *dest,
366 struct mlx5_flow_act *flow_act,
367 struct mlx5_eswitch *esw,
368 struct mlx5_flow_attr *attr,
369 struct mlx5_flow_spec *spec,
370 bool ignore_flow_lvl,
371 int *i)
372 {
373 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
374 int j, err;
375
376 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
377 return -EOPNOTSUPP;
378
379 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
380 if (ignore_flow_lvl)
381 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
382 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
383
384 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, spec,
385 esw_attr->dests[j].rep->vport, false);
386 if (IS_ERR(dest[*i].ft)) {
387 err = PTR_ERR(dest[*i].ft);
388 goto err_indir_tbl_get;
389 }
390 }
391
392 if (mlx5_esw_indir_table_decap_vport(attr)) {
393 err = esw_setup_decap_indir(esw, attr, spec);
394 if (err)
395 goto err_indir_tbl_get;
396 }
397
398 return 0;
399
400 err_indir_tbl_get:
401 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
402 return err;
403 }
404
esw_cleanup_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)405 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
406 {
407 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
408
409 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
410 esw_cleanup_decap_indir(esw, attr);
411 }
412
413 static void
esw_cleanup_chain_dest(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)414 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
415 {
416 mlx5_chains_put_table(chains, chain, prio, level);
417 }
418
419 static void
esw_setup_vport_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)420 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
421 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
422 int attr_idx, int dest_idx, bool pkt_reformat)
423 {
424 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
425 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport;
426 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
427 dest[dest_idx].vport.vhca_id =
428 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
429 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
430 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
431 mlx5_lag_mpesw_is_activated(esw->dev))
432 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
433 }
434 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP) {
435 if (pkt_reformat) {
436 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
437 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
438 }
439 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
440 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
441 }
442 }
443
444 static int
esw_setup_vport_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int i)445 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
446 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
447 int i)
448 {
449 int j;
450
451 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
452 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
453 return i;
454 }
455
456 static bool
esw_src_port_rewrite_supported(struct mlx5_eswitch * esw)457 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
458 {
459 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
460 mlx5_eswitch_vport_match_metadata_enabled(esw) &&
461 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
462 }
463
464 static int
esw_setup_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int * i)465 esw_setup_dests(struct mlx5_flow_destination *dest,
466 struct mlx5_flow_act *flow_act,
467 struct mlx5_eswitch *esw,
468 struct mlx5_flow_attr *attr,
469 struct mlx5_flow_spec *spec,
470 int *i)
471 {
472 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
473 struct mlx5_fs_chains *chains = esw_chains(esw);
474 int err = 0;
475
476 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
477 esw_src_port_rewrite_supported(esw))
478 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
479
480 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE &&
481 !(attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)) {
482 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
483 (*i)++;
484 } else if (attr->dest_ft) {
485 esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i);
486 (*i)++;
487 } else if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
488 esw_setup_slow_path_dest(dest, flow_act, esw, *i);
489 (*i)++;
490 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
491 esw_setup_accept_dest(dest, flow_act, chains, *i);
492 (*i)++;
493 } else if (attr->dest_chain) {
494 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
495 1, 0, *i);
496 (*i)++;
497 } else if (esw_is_indir_table(esw, attr)) {
498 err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i);
499 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
500 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
501 } else {
502 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
503 }
504
505 return err;
506 }
507
508 static void
esw_cleanup_dests(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)509 esw_cleanup_dests(struct mlx5_eswitch *esw,
510 struct mlx5_flow_attr *attr)
511 {
512 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
513 struct mlx5_fs_chains *chains = esw_chains(esw);
514
515 if (attr->dest_ft) {
516 esw_cleanup_decap_indir(esw, attr);
517 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
518 if (attr->dest_chain)
519 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
520 else if (esw_is_indir_table(esw, attr))
521 esw_cleanup_indir_table(esw, attr);
522 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
523 esw_cleanup_chain_src_port_rewrite(esw, attr);
524 }
525 }
526
527 struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)528 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
529 struct mlx5_flow_spec *spec,
530 struct mlx5_flow_attr *attr)
531 {
532 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
533 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
534 struct mlx5_fs_chains *chains = esw_chains(esw);
535 bool split = !!(esw_attr->split_count);
536 struct mlx5_vport_tbl_attr fwd_attr;
537 struct mlx5_flow_destination *dest;
538 struct mlx5_flow_handle *rule;
539 struct mlx5_flow_table *fdb;
540 int i = 0;
541
542 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
543 return ERR_PTR(-EOPNOTSUPP);
544
545 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
546 if (!dest)
547 return ERR_PTR(-ENOMEM);
548
549 flow_act.action = attr->action;
550 /* if per flow vlan pop/push is emulated, don't set that into the firmware */
551 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
552 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH |
553 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
554 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
555 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
556 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
557 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
558 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
559 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
560 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
561 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
562 }
563 }
564
565 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
566
567 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
568 int err;
569
570 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
571 if (err) {
572 rule = ERR_PTR(err);
573 goto err_create_goto_table;
574 }
575 }
576
577 if (esw_attr->decap_pkt_reformat)
578 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
579
580 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
581 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
582 dest[i].counter_id = mlx5_fc_id(attr->counter);
583 i++;
584 }
585
586 if (attr->outer_match_level != MLX5_MATCH_NONE)
587 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
588 if (attr->inner_match_level != MLX5_MATCH_NONE)
589 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
590
591 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
592 flow_act.modify_hdr = attr->modify_hdr;
593
594 if (split) {
595 fwd_attr.chain = attr->chain;
596 fwd_attr.prio = attr->prio;
597 fwd_attr.vport = esw_attr->in_rep->vport;
598 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
599
600 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
601 } else {
602 if (attr->chain || attr->prio)
603 fdb = mlx5_chains_get_table(chains, attr->chain,
604 attr->prio, 0);
605 else
606 fdb = attr->ft;
607
608 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
609 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
610 esw_attr->in_mdev->priv.eswitch,
611 esw_attr->in_rep->vport);
612 }
613 if (IS_ERR(fdb)) {
614 rule = ERR_CAST(fdb);
615 goto err_esw_get;
616 }
617
618 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
619 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
620 &flow_act, dest, i);
621 else
622 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
623 if (IS_ERR(rule))
624 goto err_add_rule;
625 else
626 atomic64_inc(&esw->offloads.num_flows);
627
628 kfree(dest);
629 return rule;
630
631 err_add_rule:
632 if (split)
633 mlx5_esw_vporttbl_put(esw, &fwd_attr);
634 else if (attr->chain || attr->prio)
635 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
636 err_esw_get:
637 esw_cleanup_dests(esw, attr);
638 err_create_goto_table:
639 kfree(dest);
640 return rule;
641 }
642
643 struct mlx5_flow_handle *
mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)644 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
645 struct mlx5_flow_spec *spec,
646 struct mlx5_flow_attr *attr)
647 {
648 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
649 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
650 struct mlx5_fs_chains *chains = esw_chains(esw);
651 struct mlx5_vport_tbl_attr fwd_attr;
652 struct mlx5_flow_destination *dest;
653 struct mlx5_flow_table *fast_fdb;
654 struct mlx5_flow_table *fwd_fdb;
655 struct mlx5_flow_handle *rule;
656 int i, err = 0;
657
658 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
659 if (!dest)
660 return ERR_PTR(-ENOMEM);
661
662 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
663 if (IS_ERR(fast_fdb)) {
664 rule = ERR_CAST(fast_fdb);
665 goto err_get_fast;
666 }
667
668 fwd_attr.chain = attr->chain;
669 fwd_attr.prio = attr->prio;
670 fwd_attr.vport = esw_attr->in_rep->vport;
671 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
672 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
673 if (IS_ERR(fwd_fdb)) {
674 rule = ERR_CAST(fwd_fdb);
675 goto err_get_fwd;
676 }
677
678 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
679 for (i = 0; i < esw_attr->split_count; i++) {
680 if (esw_is_indir_table(esw, attr))
681 err = esw_setup_indir_table(dest, &flow_act, esw, attr, spec, false, &i);
682 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
683 err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr,
684 &i);
685 else
686 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
687
688 if (err) {
689 rule = ERR_PTR(err);
690 goto err_chain_src_rewrite;
691 }
692 }
693 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
694 dest[i].ft = fwd_fdb;
695 i++;
696
697 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
698 esw_attr->in_mdev->priv.eswitch,
699 esw_attr->in_rep->vport);
700
701 if (attr->outer_match_level != MLX5_MATCH_NONE)
702 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
703
704 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
705 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
706
707 if (IS_ERR(rule)) {
708 i = esw_attr->split_count;
709 goto err_chain_src_rewrite;
710 }
711
712 atomic64_inc(&esw->offloads.num_flows);
713
714 kfree(dest);
715 return rule;
716 err_chain_src_rewrite:
717 esw_put_dest_tables_loop(esw, attr, 0, i);
718 mlx5_esw_vporttbl_put(esw, &fwd_attr);
719 err_get_fwd:
720 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
721 err_get_fast:
722 kfree(dest);
723 return rule;
724 }
725
726 static void
__mlx5_eswitch_del_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr,bool fwd_rule)727 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
728 struct mlx5_flow_handle *rule,
729 struct mlx5_flow_attr *attr,
730 bool fwd_rule)
731 {
732 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
733 struct mlx5_fs_chains *chains = esw_chains(esw);
734 bool split = (esw_attr->split_count > 0);
735 struct mlx5_vport_tbl_attr fwd_attr;
736 int i;
737
738 mlx5_del_flow_rules(rule);
739
740 if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
741 /* unref the term table */
742 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
743 if (esw_attr->dests[i].termtbl)
744 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
745 }
746 }
747
748 atomic64_dec(&esw->offloads.num_flows);
749
750 if (fwd_rule || split) {
751 fwd_attr.chain = attr->chain;
752 fwd_attr.prio = attr->prio;
753 fwd_attr.vport = esw_attr->in_rep->vport;
754 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
755 }
756
757 if (fwd_rule) {
758 mlx5_esw_vporttbl_put(esw, &fwd_attr);
759 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
760 esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count);
761 } else {
762 if (split)
763 mlx5_esw_vporttbl_put(esw, &fwd_attr);
764 else if (attr->chain || attr->prio)
765 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
766 esw_cleanup_dests(esw, attr);
767 }
768 }
769
770 void
mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)771 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
772 struct mlx5_flow_handle *rule,
773 struct mlx5_flow_attr *attr)
774 {
775 __mlx5_eswitch_del_rule(esw, rule, attr, false);
776 }
777
778 void
mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)779 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
780 struct mlx5_flow_handle *rule,
781 struct mlx5_flow_attr *attr)
782 {
783 __mlx5_eswitch_del_rule(esw, rule, attr, true);
784 }
785
esw_set_global_vlan_pop(struct mlx5_eswitch * esw,u8 val)786 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
787 {
788 struct mlx5_eswitch_rep *rep;
789 unsigned long i;
790 int err = 0;
791
792 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
793 mlx5_esw_for_each_host_func_vport(esw, i, rep, esw->esw_funcs.num_vfs) {
794 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
795 continue;
796
797 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
798 if (err)
799 goto out;
800 }
801
802 out:
803 return err;
804 }
805
806 static struct mlx5_eswitch_rep *
esw_vlan_action_get_vport(struct mlx5_esw_flow_attr * attr,bool push,bool pop)807 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop)
808 {
809 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL;
810
811 in_rep = attr->in_rep;
812 out_rep = attr->dests[0].rep;
813
814 if (push)
815 vport = in_rep;
816 else if (pop)
817 vport = out_rep;
818 else
819 vport = in_rep;
820
821 return vport;
822 }
823
esw_add_vlan_action_check(struct mlx5_esw_flow_attr * attr,bool push,bool pop,bool fwd)824 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr,
825 bool push, bool pop, bool fwd)
826 {
827 struct mlx5_eswitch_rep *in_rep, *out_rep;
828
829 if ((push || pop) && !fwd)
830 goto out_notsupp;
831
832 in_rep = attr->in_rep;
833 out_rep = attr->dests[0].rep;
834
835 if (push && in_rep->vport == MLX5_VPORT_UPLINK)
836 goto out_notsupp;
837
838 if (pop && out_rep->vport == MLX5_VPORT_UPLINK)
839 goto out_notsupp;
840
841 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */
842 if (!push && !pop && fwd)
843 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK)
844 goto out_notsupp;
845
846 /* protects against (1) setting rules with different vlans to push and
847 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
848 */
849 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0]))
850 goto out_notsupp;
851
852 return 0;
853
854 out_notsupp:
855 return -EOPNOTSUPP;
856 }
857
mlx5_eswitch_add_vlan_action(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)858 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
859 struct mlx5_flow_attr *attr)
860 {
861 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
862 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
863 struct mlx5_eswitch_rep *vport = NULL;
864 bool push, pop, fwd;
865 int err = 0;
866
867 /* nop if we're on the vlan push/pop non emulation mode */
868 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
869 return 0;
870
871 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
872 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
873 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) &&
874 !attr->dest_chain);
875
876 mutex_lock(&esw->state_lock);
877
878 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd);
879 if (err)
880 goto unlock;
881
882 attr->flags &= ~MLX5_ATTR_FLAG_VLAN_HANDLED;
883
884 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
885
886 if (!push && !pop && fwd) {
887 /* tracks VF --> wire rules without vlan push action */
888 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {
889 vport->vlan_refcount++;
890 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED;
891 }
892
893 goto unlock;
894 }
895
896 if (!push && !pop)
897 goto unlock;
898
899 if (!(offloads->vlan_push_pop_refcount)) {
900 /* it's the 1st vlan rule, apply global vlan pop policy */
901 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP);
902 if (err)
903 goto out;
904 }
905 offloads->vlan_push_pop_refcount++;
906
907 if (push) {
908 if (vport->vlan_refcount)
909 goto skip_set_push;
910
911 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0],
912 0, SET_VLAN_INSERT | SET_VLAN_STRIP);
913 if (err)
914 goto out;
915 vport->vlan = esw_attr->vlan_vid[0];
916 skip_set_push:
917 vport->vlan_refcount++;
918 }
919 out:
920 if (!err)
921 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED;
922 unlock:
923 mutex_unlock(&esw->state_lock);
924 return err;
925 }
926
mlx5_eswitch_del_vlan_action(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)927 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
928 struct mlx5_flow_attr *attr)
929 {
930 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
931 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
932 struct mlx5_eswitch_rep *vport = NULL;
933 bool push, pop, fwd;
934 int err = 0;
935
936 /* nop if we're on the vlan push/pop non emulation mode */
937 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
938 return 0;
939
940 if (!(attr->flags & MLX5_ATTR_FLAG_VLAN_HANDLED))
941 return 0;
942
943 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
944 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
945 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
946
947 mutex_lock(&esw->state_lock);
948
949 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
950
951 if (!push && !pop && fwd) {
952 /* tracks VF --> wire rules without vlan push action */
953 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK)
954 vport->vlan_refcount--;
955
956 goto out;
957 }
958
959 if (push) {
960 vport->vlan_refcount--;
961 if (vport->vlan_refcount)
962 goto skip_unset_push;
963
964 vport->vlan = 0;
965 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport,
966 0, 0, SET_VLAN_STRIP);
967 if (err)
968 goto out;
969 }
970
971 skip_unset_push:
972 offloads->vlan_push_pop_refcount--;
973 if (offloads->vlan_push_pop_refcount)
974 goto out;
975
976 /* no more vlan rules, stop global vlan pop policy */
977 err = esw_set_global_vlan_pop(esw, 0);
978
979 out:
980 mutex_unlock(&esw->state_lock);
981 return err;
982 }
983
984 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch * on_esw,struct mlx5_eswitch * from_esw,struct mlx5_eswitch_rep * rep,u32 sqn)985 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
986 struct mlx5_eswitch *from_esw,
987 struct mlx5_eswitch_rep *rep,
988 u32 sqn)
989 {
990 struct mlx5_flow_act flow_act = {0};
991 struct mlx5_flow_destination dest = {};
992 struct mlx5_flow_handle *flow_rule;
993 struct mlx5_flow_spec *spec;
994 void *misc;
995
996 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
997 if (!spec) {
998 flow_rule = ERR_PTR(-ENOMEM);
999 goto out;
1000 }
1001
1002 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
1003 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
1004 /* source vport is the esw manager */
1005 MLX5_SET(fte_match_set_misc, misc, source_port, from_esw->manager_vport);
1006 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
1007 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1008 MLX5_CAP_GEN(from_esw->dev, vhca_id));
1009
1010 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
1011 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
1012 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1013 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
1014 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1015 source_eswitch_owner_vhca_id);
1016
1017 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1018 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1019 dest.vport.num = rep->vport;
1020 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
1021 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1022 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1023
1024 if (rep->vport == MLX5_VPORT_UPLINK)
1025 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1026
1027 flow_rule = mlx5_add_flow_rules(on_esw->fdb_table.offloads.slow_fdb,
1028 spec, &flow_act, &dest, 1);
1029 if (IS_ERR(flow_rule))
1030 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
1031 PTR_ERR(flow_rule));
1032 out:
1033 kvfree(spec);
1034 return flow_rule;
1035 }
1036 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1037
mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle * rule)1038 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1039 {
1040 mlx5_del_flow_rules(rule);
1041 }
1042
mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch * esw)1043 static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
1044 {
1045 struct mlx5_flow_handle **flows = esw->fdb_table.offloads.send_to_vport_meta_rules;
1046 int i = 0, num_vfs = esw->esw_funcs.num_vfs;
1047
1048 if (!num_vfs || !flows)
1049 return;
1050
1051 for (i = 0; i < num_vfs; i++)
1052 mlx5_del_flow_rules(flows[i]);
1053
1054 kvfree(flows);
1055 }
1056
1057 static int
mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch * esw)1058 mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
1059 {
1060 struct mlx5_flow_destination dest = {};
1061 struct mlx5_flow_act flow_act = {0};
1062 int num_vfs, rule_idx = 0, err = 0;
1063 struct mlx5_flow_handle *flow_rule;
1064 struct mlx5_flow_handle **flows;
1065 struct mlx5_flow_spec *spec;
1066 struct mlx5_vport *vport;
1067 unsigned long i;
1068 u16 vport_num;
1069
1070 num_vfs = esw->esw_funcs.num_vfs;
1071 flows = kvcalloc(num_vfs, sizeof(*flows), GFP_KERNEL);
1072 if (!flows)
1073 return -ENOMEM;
1074
1075 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1076 if (!spec) {
1077 err = -ENOMEM;
1078 goto alloc_err;
1079 }
1080
1081 MLX5_SET(fte_match_param, spec->match_criteria,
1082 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1083 MLX5_SET(fte_match_param, spec->match_criteria,
1084 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1085 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1086 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1087
1088 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1089 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1090 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1091
1092 mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) {
1093 vport_num = vport->vport;
1094 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1095 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1096 dest.vport.num = vport_num;
1097
1098 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1099 spec, &flow_act, &dest, 1);
1100 if (IS_ERR(flow_rule)) {
1101 err = PTR_ERR(flow_rule);
1102 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n",
1103 rule_idx, PTR_ERR(flow_rule));
1104 goto rule_err;
1105 }
1106 flows[rule_idx++] = flow_rule;
1107 }
1108
1109 esw->fdb_table.offloads.send_to_vport_meta_rules = flows;
1110 kvfree(spec);
1111 return 0;
1112
1113 rule_err:
1114 while (--rule_idx >= 0)
1115 mlx5_del_flow_rules(flows[rule_idx]);
1116 kvfree(spec);
1117 alloc_err:
1118 kvfree(flows);
1119 return err;
1120 }
1121
mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch * esw)1122 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1123 {
1124 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1125 MLX5_FDB_TO_VPORT_REG_C_1;
1126 }
1127
esw_set_passing_vport_metadata(struct mlx5_eswitch * esw,bool enable)1128 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1129 {
1130 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1131 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1132 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1133 u8 curr, wanted;
1134 int err;
1135
1136 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1137 !mlx5_eswitch_vport_match_metadata_enabled(esw))
1138 return 0;
1139
1140 MLX5_SET(query_esw_vport_context_in, in, opcode,
1141 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1142 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1143 if (err)
1144 return err;
1145
1146 curr = MLX5_GET(query_esw_vport_context_out, out,
1147 esw_vport_context.fdb_to_vport_reg_c_id);
1148 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1149 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1150 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1151
1152 if (enable)
1153 curr |= wanted;
1154 else
1155 curr &= ~wanted;
1156
1157 MLX5_SET(modify_esw_vport_context_in, min,
1158 esw_vport_context.fdb_to_vport_reg_c_id, curr);
1159 MLX5_SET(modify_esw_vport_context_in, min,
1160 field_select.fdb_to_vport_reg_c_id, 1);
1161
1162 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
1163 if (!err) {
1164 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1165 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1166 else
1167 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1168 }
1169
1170 return err;
1171 }
1172
peer_miss_rules_setup(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev,struct mlx5_flow_spec * spec,struct mlx5_flow_destination * dest)1173 static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1174 struct mlx5_core_dev *peer_dev,
1175 struct mlx5_flow_spec *spec,
1176 struct mlx5_flow_destination *dest)
1177 {
1178 void *misc;
1179
1180 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1181 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1182 misc_parameters_2);
1183 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1184 mlx5_eswitch_get_vport_metadata_mask());
1185
1186 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1187 } else {
1188 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1189 misc_parameters);
1190
1191 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1192 MLX5_CAP_GEN(peer_dev, vhca_id));
1193
1194 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1195
1196 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1197 misc_parameters);
1198 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1199 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1200 source_eswitch_owner_vhca_id);
1201 }
1202
1203 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1204 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1205 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1206 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1207 }
1208
esw_set_peer_miss_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,struct mlx5_flow_spec * spec,u16 vport)1209 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1210 struct mlx5_eswitch *peer_esw,
1211 struct mlx5_flow_spec *spec,
1212 u16 vport)
1213 {
1214 void *misc;
1215
1216 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1217 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1218 misc_parameters_2);
1219 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1220 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1221 vport));
1222 } else {
1223 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1224 misc_parameters);
1225 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1226 }
1227 }
1228
esw_add_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1229 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1230 struct mlx5_core_dev *peer_dev)
1231 {
1232 struct mlx5_flow_destination dest = {};
1233 struct mlx5_flow_act flow_act = {0};
1234 struct mlx5_flow_handle **flows;
1235 /* total vports is the same for both e-switches */
1236 int nvports = esw->total_vports;
1237 struct mlx5_flow_handle *flow;
1238 struct mlx5_flow_spec *spec;
1239 struct mlx5_vport *vport;
1240 unsigned long i;
1241 void *misc;
1242 int err;
1243
1244 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1245 if (!spec)
1246 return -ENOMEM;
1247
1248 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
1249
1250 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL);
1251 if (!flows) {
1252 err = -ENOMEM;
1253 goto alloc_flows_err;
1254 }
1255
1256 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1257 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1258 misc_parameters);
1259
1260 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1261 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1262 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1263 spec, MLX5_VPORT_PF);
1264
1265 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1266 spec, &flow_act, &dest, 1);
1267 if (IS_ERR(flow)) {
1268 err = PTR_ERR(flow);
1269 goto add_pf_flow_err;
1270 }
1271 flows[vport->index] = flow;
1272 }
1273
1274 if (mlx5_ecpf_vport_exists(esw->dev)) {
1275 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1276 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1277 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1278 spec, &flow_act, &dest, 1);
1279 if (IS_ERR(flow)) {
1280 err = PTR_ERR(flow);
1281 goto add_ecpf_flow_err;
1282 }
1283 flows[vport->index] = flow;
1284 }
1285
1286 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1287 esw_set_peer_miss_rule_source_port(esw,
1288 peer_dev->priv.eswitch,
1289 spec, vport->vport);
1290
1291 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1292 spec, &flow_act, &dest, 1);
1293 if (IS_ERR(flow)) {
1294 err = PTR_ERR(flow);
1295 goto add_vf_flow_err;
1296 }
1297 flows[vport->index] = flow;
1298 }
1299
1300 esw->fdb_table.offloads.peer_miss_rules = flows;
1301
1302 kvfree(spec);
1303 return 0;
1304
1305 add_vf_flow_err:
1306 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1307 if (!flows[vport->index])
1308 continue;
1309 mlx5_del_flow_rules(flows[vport->index]);
1310 }
1311 if (mlx5_ecpf_vport_exists(esw->dev)) {
1312 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1313 mlx5_del_flow_rules(flows[vport->index]);
1314 }
1315 add_ecpf_flow_err:
1316 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1317 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1318 mlx5_del_flow_rules(flows[vport->index]);
1319 }
1320 add_pf_flow_err:
1321 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1322 kvfree(flows);
1323 alloc_flows_err:
1324 kvfree(spec);
1325 return err;
1326 }
1327
esw_del_fdb_peer_miss_rules(struct mlx5_eswitch * esw)1328 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw)
1329 {
1330 struct mlx5_flow_handle **flows;
1331 struct mlx5_vport *vport;
1332 unsigned long i;
1333
1334 flows = esw->fdb_table.offloads.peer_miss_rules;
1335
1336 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1337 mlx5_del_flow_rules(flows[vport->index]);
1338
1339 if (mlx5_ecpf_vport_exists(esw->dev)) {
1340 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1341 mlx5_del_flow_rules(flows[vport->index]);
1342 }
1343
1344 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1345 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1346 mlx5_del_flow_rules(flows[vport->index]);
1347 }
1348 kvfree(flows);
1349 }
1350
esw_add_fdb_miss_rule(struct mlx5_eswitch * esw)1351 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1352 {
1353 struct mlx5_flow_act flow_act = {0};
1354 struct mlx5_flow_destination dest = {};
1355 struct mlx5_flow_handle *flow_rule = NULL;
1356 struct mlx5_flow_spec *spec;
1357 void *headers_c;
1358 void *headers_v;
1359 int err = 0;
1360 u8 *dmac_c;
1361 u8 *dmac_v;
1362
1363 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1364 if (!spec) {
1365 err = -ENOMEM;
1366 goto out;
1367 }
1368
1369 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1370 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1371 outer_headers);
1372 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1373 outer_headers.dmac_47_16);
1374 dmac_c[0] = 0x01;
1375
1376 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1377 dest.vport.num = esw->manager_vport;
1378 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1379
1380 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1381 spec, &flow_act, &dest, 1);
1382 if (IS_ERR(flow_rule)) {
1383 err = PTR_ERR(flow_rule);
1384 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1385 goto out;
1386 }
1387
1388 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1389
1390 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1391 outer_headers);
1392 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1393 outer_headers.dmac_47_16);
1394 dmac_v[0] = 0x01;
1395 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1396 spec, &flow_act, &dest, 1);
1397 if (IS_ERR(flow_rule)) {
1398 err = PTR_ERR(flow_rule);
1399 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1400 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1401 goto out;
1402 }
1403
1404 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1405
1406 out:
1407 kvfree(spec);
1408 return err;
1409 }
1410
1411 struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch * esw,u32 tag)1412 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1413 {
1414 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1415 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1416 struct mlx5_flow_context *flow_context;
1417 struct mlx5_flow_handle *flow_rule;
1418 struct mlx5_flow_destination dest;
1419 struct mlx5_flow_spec *spec;
1420 void *misc;
1421
1422 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1423 return ERR_PTR(-EOPNOTSUPP);
1424
1425 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1426 if (!spec)
1427 return ERR_PTR(-ENOMEM);
1428
1429 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1430 misc_parameters_2);
1431 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1432 ESW_REG_C0_USER_DATA_METADATA_MASK);
1433 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1434 misc_parameters_2);
1435 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1436 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1437 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1438 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1439 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1440
1441 flow_context = &spec->flow_context;
1442 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1443 flow_context->flow_tag = tag;
1444 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1445 dest.ft = esw->offloads.ft_offloads;
1446
1447 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1448 kvfree(spec);
1449
1450 if (IS_ERR(flow_rule))
1451 esw_warn(esw->dev,
1452 "Failed to create restore rule for tag: %d, err(%d)\n",
1453 tag, (int)PTR_ERR(flow_rule));
1454
1455 return flow_rule;
1456 }
1457
1458 #define MAX_PF_SQ 256
1459 #define MAX_SQ_NVPORTS 32
1460
esw_set_flow_group_source_port(struct mlx5_eswitch * esw,u32 * flow_group_in)1461 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1462 u32 *flow_group_in)
1463 {
1464 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1465 flow_group_in,
1466 match_criteria);
1467
1468 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1469 MLX5_SET(create_flow_group_in, flow_group_in,
1470 match_criteria_enable,
1471 MLX5_MATCH_MISC_PARAMETERS_2);
1472
1473 MLX5_SET(fte_match_param, match_criteria,
1474 misc_parameters_2.metadata_reg_c_0,
1475 mlx5_eswitch_get_vport_metadata_mask());
1476 } else {
1477 MLX5_SET(create_flow_group_in, flow_group_in,
1478 match_criteria_enable,
1479 MLX5_MATCH_MISC_PARAMETERS);
1480
1481 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1482 misc_parameters.source_port);
1483 }
1484 }
1485
1486 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
esw_vport_tbl_put(struct mlx5_eswitch * esw)1487 static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1488 {
1489 struct mlx5_vport_tbl_attr attr;
1490 struct mlx5_vport *vport;
1491 unsigned long i;
1492
1493 attr.chain = 0;
1494 attr.prio = 1;
1495 mlx5_esw_for_each_vport(esw, i, vport) {
1496 attr.vport = vport->vport;
1497 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1498 mlx5_esw_vporttbl_put(esw, &attr);
1499 }
1500 }
1501
esw_vport_tbl_get(struct mlx5_eswitch * esw)1502 static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1503 {
1504 struct mlx5_vport_tbl_attr attr;
1505 struct mlx5_flow_table *fdb;
1506 struct mlx5_vport *vport;
1507 unsigned long i;
1508
1509 attr.chain = 0;
1510 attr.prio = 1;
1511 mlx5_esw_for_each_vport(esw, i, vport) {
1512 attr.vport = vport->vport;
1513 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1514 fdb = mlx5_esw_vporttbl_get(esw, &attr);
1515 if (IS_ERR(fdb))
1516 goto out;
1517 }
1518 return 0;
1519
1520 out:
1521 esw_vport_tbl_put(esw);
1522 return PTR_ERR(fdb);
1523 }
1524
1525 #define fdb_modify_header_fwd_to_table_supported(esw) \
1526 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
esw_init_chains_offload_flags(struct mlx5_eswitch * esw,u32 * flags)1527 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1528 {
1529 struct mlx5_core_dev *dev = esw->dev;
1530
1531 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1532 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1533
1534 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1535 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1536 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1537 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1538 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1539 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1540 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1541 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1542 /* Disabled when ttl workaround is needed, e.g
1543 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1544 */
1545 esw_warn(dev,
1546 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1547 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1548 } else {
1549 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1550 esw_info(dev, "Supported tc chains and prios offload\n");
1551 }
1552
1553 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1554 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1555 }
1556
1557 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1558 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1559 {
1560 struct mlx5_core_dev *dev = esw->dev;
1561 struct mlx5_flow_table *nf_ft, *ft;
1562 struct mlx5_chains_attr attr = {};
1563 struct mlx5_fs_chains *chains;
1564 u32 fdb_max;
1565 int err;
1566
1567 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
1568
1569 esw_init_chains_offload_flags(esw, &attr.flags);
1570 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1571 attr.max_ft_sz = fdb_max;
1572 attr.max_grp_num = esw->params.large_group_num;
1573 attr.default_ft = miss_fdb;
1574 attr.mapping = esw->offloads.reg_c0_obj_pool;
1575
1576 chains = mlx5_chains_create(dev, &attr);
1577 if (IS_ERR(chains)) {
1578 err = PTR_ERR(chains);
1579 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1580 return err;
1581 }
1582
1583 esw->fdb_table.offloads.esw_chains_priv = chains;
1584
1585 /* Create tc_end_ft which is the always created ft chain */
1586 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1587 1, 0);
1588 if (IS_ERR(nf_ft)) {
1589 err = PTR_ERR(nf_ft);
1590 goto nf_ft_err;
1591 }
1592
1593 /* Always open the root for fast path */
1594 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1595 if (IS_ERR(ft)) {
1596 err = PTR_ERR(ft);
1597 goto level_0_err;
1598 }
1599
1600 /* Open level 1 for split fdb rules now if prios isn't supported */
1601 if (!mlx5_chains_prios_supported(chains)) {
1602 err = esw_vport_tbl_get(esw);
1603 if (err)
1604 goto level_1_err;
1605 }
1606
1607 mlx5_chains_set_end_ft(chains, nf_ft);
1608
1609 return 0;
1610
1611 level_1_err:
1612 mlx5_chains_put_table(chains, 0, 1, 0);
1613 level_0_err:
1614 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1615 nf_ft_err:
1616 mlx5_chains_destroy(chains);
1617 esw->fdb_table.offloads.esw_chains_priv = NULL;
1618
1619 return err;
1620 }
1621
1622 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1623 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1624 {
1625 if (!mlx5_chains_prios_supported(chains))
1626 esw_vport_tbl_put(esw);
1627 mlx5_chains_put_table(chains, 0, 1, 0);
1628 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1629 mlx5_chains_destroy(chains);
1630 }
1631
1632 #else /* CONFIG_MLX5_CLS_ACT */
1633
1634 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1635 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1636 { return 0; }
1637
1638 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1639 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1640 {}
1641
1642 #endif
1643
esw_create_offloads_fdb_tables(struct mlx5_eswitch * esw)1644 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1645 {
1646 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1647 struct mlx5_flow_table_attr ft_attr = {};
1648 int num_vfs, table_size, ix, err = 0;
1649 struct mlx5_core_dev *dev = esw->dev;
1650 struct mlx5_flow_namespace *root_ns;
1651 struct mlx5_flow_table *fdb = NULL;
1652 u32 flags = 0, *flow_group_in;
1653 struct mlx5_flow_group *g;
1654 void *match_criteria;
1655 u8 *dmac;
1656
1657 esw_debug(esw->dev, "Create offloads FDB Tables\n");
1658
1659 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1660 if (!flow_group_in)
1661 return -ENOMEM;
1662
1663 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1664 if (!root_ns) {
1665 esw_warn(dev, "Failed to get FDB flow namespace\n");
1666 err = -EOPNOTSUPP;
1667 goto ns_err;
1668 }
1669 esw->fdb_table.offloads.ns = root_ns;
1670 err = mlx5_flow_namespace_set_mode(root_ns,
1671 esw->dev->priv.steering->mode);
1672 if (err) {
1673 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1674 goto ns_err;
1675 }
1676
1677 /* To be strictly correct:
1678 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1679 * should be:
1680 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1681 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1682 * but as the peer device might not be in switchdev mode it's not
1683 * possible. We use the fact that by default FW sets max vfs and max sfs
1684 * to the same value on both devices. If it needs to be changed in the future note
1685 * the peer miss group should also be created based on the number of
1686 * total vports of the peer (currently is also uses esw->total_vports).
1687 */
1688 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1689 MLX5_ESW_MISS_FLOWS + esw->total_vports + esw->esw_funcs.num_vfs;
1690
1691 /* create the slow path fdb with encap set, so further table instances
1692 * can be created at run time while VFs are probed if the FW allows that.
1693 */
1694 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1695 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1696 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1697
1698 ft_attr.flags = flags;
1699 ft_attr.max_fte = table_size;
1700 ft_attr.prio = FDB_SLOW_PATH;
1701
1702 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1703 if (IS_ERR(fdb)) {
1704 err = PTR_ERR(fdb);
1705 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1706 goto slow_fdb_err;
1707 }
1708 esw->fdb_table.offloads.slow_fdb = fdb;
1709
1710 /* Create empty TC-miss managed table. This allows plugging in following
1711 * priorities without directly exposing their level 0 table to
1712 * eswitch_offloads and passing it as miss_fdb to following call to
1713 * esw_chains_create().
1714 */
1715 memset(&ft_attr, 0, sizeof(ft_attr));
1716 ft_attr.prio = FDB_TC_MISS;
1717 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1718 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1719 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1720 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1721 goto tc_miss_table_err;
1722 }
1723
1724 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
1725 if (err) {
1726 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1727 goto fdb_chains_err;
1728 }
1729
1730 /* create send-to-vport group */
1731 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1732 MLX5_MATCH_MISC_PARAMETERS);
1733
1734 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1735
1736 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1737 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
1738 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1739 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1740 misc_parameters.source_eswitch_owner_vhca_id);
1741 MLX5_SET(create_flow_group_in, flow_group_in,
1742 source_eswitch_owner_vhca_id_valid, 1);
1743 }
1744
1745 /* See comment above table_size calculation */
1746 ix = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1747 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1748 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1);
1749
1750 g = mlx5_create_flow_group(fdb, flow_group_in);
1751 if (IS_ERR(g)) {
1752 err = PTR_ERR(g);
1753 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1754 goto send_vport_err;
1755 }
1756 esw->fdb_table.offloads.send_to_vport_grp = g;
1757
1758 if (esw_src_port_rewrite_supported(esw)) {
1759 /* meta send to vport */
1760 memset(flow_group_in, 0, inlen);
1761 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1762 MLX5_MATCH_MISC_PARAMETERS_2);
1763
1764 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1765
1766 MLX5_SET(fte_match_param, match_criteria,
1767 misc_parameters_2.metadata_reg_c_0,
1768 mlx5_eswitch_get_vport_metadata_mask());
1769 MLX5_SET(fte_match_param, match_criteria,
1770 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1771
1772 num_vfs = esw->esw_funcs.num_vfs;
1773 if (num_vfs) {
1774 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
1775 MLX5_SET(create_flow_group_in, flow_group_in,
1776 end_flow_index, ix + num_vfs - 1);
1777 ix += num_vfs;
1778
1779 g = mlx5_create_flow_group(fdb, flow_group_in);
1780 if (IS_ERR(g)) {
1781 err = PTR_ERR(g);
1782 esw_warn(dev, "Failed to create send-to-vport meta flow group err(%d)\n",
1783 err);
1784 goto send_vport_meta_err;
1785 }
1786 esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1787
1788 err = mlx5_eswitch_add_send_to_vport_meta_rules(esw);
1789 if (err)
1790 goto meta_rule_err;
1791 }
1792 }
1793
1794 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1795 /* create peer esw miss group */
1796 memset(flow_group_in, 0, inlen);
1797
1798 esw_set_flow_group_source_port(esw, flow_group_in);
1799
1800 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1801 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1802 flow_group_in,
1803 match_criteria);
1804
1805 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1806 misc_parameters.source_eswitch_owner_vhca_id);
1807
1808 MLX5_SET(create_flow_group_in, flow_group_in,
1809 source_eswitch_owner_vhca_id_valid, 1);
1810 }
1811
1812 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
1813 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1814 ix + esw->total_vports - 1);
1815 ix += esw->total_vports;
1816
1817 g = mlx5_create_flow_group(fdb, flow_group_in);
1818 if (IS_ERR(g)) {
1819 err = PTR_ERR(g);
1820 esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err);
1821 goto peer_miss_err;
1822 }
1823 esw->fdb_table.offloads.peer_miss_grp = g;
1824 }
1825
1826 /* create miss group */
1827 memset(flow_group_in, 0, inlen);
1828 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1829 MLX5_MATCH_OUTER_HEADERS);
1830 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1831 match_criteria);
1832 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1833 outer_headers.dmac_47_16);
1834 dmac[0] = 0x01;
1835
1836 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
1837 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1838 ix + MLX5_ESW_MISS_FLOWS);
1839
1840 g = mlx5_create_flow_group(fdb, flow_group_in);
1841 if (IS_ERR(g)) {
1842 err = PTR_ERR(g);
1843 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err);
1844 goto miss_err;
1845 }
1846 esw->fdb_table.offloads.miss_grp = g;
1847
1848 err = esw_add_fdb_miss_rule(esw);
1849 if (err)
1850 goto miss_rule_err;
1851
1852 kvfree(flow_group_in);
1853 return 0;
1854
1855 miss_rule_err:
1856 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1857 miss_err:
1858 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1859 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1860 peer_miss_err:
1861 mlx5_eswitch_del_send_to_vport_meta_rules(esw);
1862 meta_rule_err:
1863 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1864 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1865 send_vport_meta_err:
1866 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1867 send_vport_err:
1868 esw_chains_destroy(esw, esw_chains(esw));
1869 fdb_chains_err:
1870 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1871 tc_miss_table_err:
1872 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1873 slow_fdb_err:
1874 /* Holds true only as long as DMFS is the default */
1875 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
1876 ns_err:
1877 kvfree(flow_group_in);
1878 return err;
1879 }
1880
esw_destroy_offloads_fdb_tables(struct mlx5_eswitch * esw)1881 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1882 {
1883 if (!esw->fdb_table.offloads.slow_fdb)
1884 return;
1885
1886 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1887 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1888 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1889 mlx5_eswitch_del_send_to_vport_meta_rules(esw);
1890 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1891 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1892 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1893 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1894 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1895 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1896
1897 esw_chains_destroy(esw, esw_chains(esw));
1898
1899 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1900 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1901 /* Holds true only as long as DMFS is the default */
1902 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1903 MLX5_FLOW_STEERING_MODE_DMFS);
1904 atomic64_set(&esw->user_count, 0);
1905 }
1906
esw_get_offloads_ft_size(struct mlx5_eswitch * esw)1907 static int esw_get_offloads_ft_size(struct mlx5_eswitch *esw)
1908 {
1909 int nvports;
1910
1911 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1912 if (mlx5e_tc_int_port_supported(esw))
1913 nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1914
1915 return nvports;
1916 }
1917
esw_create_offloads_table(struct mlx5_eswitch * esw)1918 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1919 {
1920 struct mlx5_flow_table_attr ft_attr = {};
1921 struct mlx5_core_dev *dev = esw->dev;
1922 struct mlx5_flow_table *ft_offloads;
1923 struct mlx5_flow_namespace *ns;
1924 int err = 0;
1925
1926 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1927 if (!ns) {
1928 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1929 return -EOPNOTSUPP;
1930 }
1931
1932 ft_attr.max_fte = esw_get_offloads_ft_size(esw);
1933 ft_attr.prio = 1;
1934
1935 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
1936 if (IS_ERR(ft_offloads)) {
1937 err = PTR_ERR(ft_offloads);
1938 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
1939 return err;
1940 }
1941
1942 esw->offloads.ft_offloads = ft_offloads;
1943 return 0;
1944 }
1945
esw_destroy_offloads_table(struct mlx5_eswitch * esw)1946 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
1947 {
1948 struct mlx5_esw_offload *offloads = &esw->offloads;
1949
1950 mlx5_destroy_flow_table(offloads->ft_offloads);
1951 }
1952
esw_create_vport_rx_group(struct mlx5_eswitch * esw)1953 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
1954 {
1955 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1956 struct mlx5_flow_group *g;
1957 u32 *flow_group_in;
1958 int nvports;
1959 int err = 0;
1960
1961 nvports = esw_get_offloads_ft_size(esw);
1962 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1963 if (!flow_group_in)
1964 return -ENOMEM;
1965
1966 /* create vport rx group */
1967 esw_set_flow_group_source_port(esw, flow_group_in);
1968
1969 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1970 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
1971
1972 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
1973
1974 if (IS_ERR(g)) {
1975 err = PTR_ERR(g);
1976 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
1977 goto out;
1978 }
1979
1980 esw->offloads.vport_rx_group = g;
1981 out:
1982 kvfree(flow_group_in);
1983 return err;
1984 }
1985
esw_destroy_vport_rx_group(struct mlx5_eswitch * esw)1986 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
1987 {
1988 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
1989 }
1990
1991 struct mlx5_flow_handle *
mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_destination * dest)1992 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
1993 struct mlx5_flow_destination *dest)
1994 {
1995 struct mlx5_flow_act flow_act = {0};
1996 struct mlx5_flow_handle *flow_rule;
1997 struct mlx5_flow_spec *spec;
1998 void *misc;
1999
2000 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2001 if (!spec) {
2002 flow_rule = ERR_PTR(-ENOMEM);
2003 goto out;
2004 }
2005
2006 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2007 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2008 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2009 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2010
2011 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2012 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2013 mlx5_eswitch_get_vport_metadata_mask());
2014
2015 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2016 } else {
2017 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2018 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2019
2020 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2021 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2022
2023 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2024 }
2025
2026 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2027 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
2028 &flow_act, dest, 1);
2029 if (IS_ERR(flow_rule)) {
2030 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
2031 goto out;
2032 }
2033
2034 out:
2035 kvfree(spec);
2036 return flow_rule;
2037 }
2038
mlx5_eswitch_inline_mode_get(struct mlx5_eswitch * esw,u8 * mode)2039 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2040 {
2041 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2042 struct mlx5_core_dev *dev = esw->dev;
2043 struct mlx5_vport *vport;
2044 unsigned long i;
2045
2046 if (!MLX5_CAP_GEN(dev, vport_group_manager))
2047 return -EOPNOTSUPP;
2048
2049 if (esw->mode == MLX5_ESWITCH_NONE)
2050 return -EOPNOTSUPP;
2051
2052 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2053 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2054 mlx5_mode = MLX5_INLINE_MODE_NONE;
2055 goto out;
2056 case MLX5_CAP_INLINE_MODE_L2:
2057 mlx5_mode = MLX5_INLINE_MODE_L2;
2058 goto out;
2059 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2060 goto query_vports;
2061 }
2062
2063 query_vports:
2064 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
2065 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2066 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
2067 if (prev_mlx5_mode != mlx5_mode)
2068 return -EINVAL;
2069 prev_mlx5_mode = mlx5_mode;
2070 }
2071
2072 out:
2073 *mode = mlx5_mode;
2074 return 0;
2075 }
2076
esw_destroy_restore_table(struct mlx5_eswitch * esw)2077 static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2078 {
2079 struct mlx5_esw_offload *offloads = &esw->offloads;
2080
2081 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2082 return;
2083
2084 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
2085 mlx5_destroy_flow_group(offloads->restore_group);
2086 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2087 }
2088
esw_create_restore_table(struct mlx5_eswitch * esw)2089 static int esw_create_restore_table(struct mlx5_eswitch *esw)
2090 {
2091 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2092 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2093 struct mlx5_flow_table_attr ft_attr = {};
2094 struct mlx5_core_dev *dev = esw->dev;
2095 struct mlx5_flow_namespace *ns;
2096 struct mlx5_modify_hdr *mod_hdr;
2097 void *match_criteria, *misc;
2098 struct mlx5_flow_table *ft;
2099 struct mlx5_flow_group *g;
2100 u32 *flow_group_in;
2101 int err = 0;
2102
2103 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2104 return 0;
2105
2106 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2107 if (!ns) {
2108 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2109 return -EOPNOTSUPP;
2110 }
2111
2112 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2113 if (!flow_group_in) {
2114 err = -ENOMEM;
2115 goto out_free;
2116 }
2117
2118 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2119 ft = mlx5_create_flow_table(ns, &ft_attr);
2120 if (IS_ERR(ft)) {
2121 err = PTR_ERR(ft);
2122 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2123 err);
2124 goto out_free;
2125 }
2126
2127 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2128 match_criteria);
2129 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2130 misc_parameters_2);
2131
2132 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2133 ESW_REG_C0_USER_DATA_METADATA_MASK);
2134 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2135 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2136 ft_attr.max_fte - 1);
2137 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2138 MLX5_MATCH_MISC_PARAMETERS_2);
2139 g = mlx5_create_flow_group(ft, flow_group_in);
2140 if (IS_ERR(g)) {
2141 err = PTR_ERR(g);
2142 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2143 err);
2144 goto err_group;
2145 }
2146
2147 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2148 MLX5_SET(copy_action_in, modact, src_field,
2149 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2150 MLX5_SET(copy_action_in, modact, dst_field,
2151 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2152 mod_hdr = mlx5_modify_header_alloc(esw->dev,
2153 MLX5_FLOW_NAMESPACE_KERNEL, 1,
2154 modact);
2155 if (IS_ERR(mod_hdr)) {
2156 err = PTR_ERR(mod_hdr);
2157 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2158 err);
2159 goto err_mod_hdr;
2160 }
2161
2162 esw->offloads.ft_offloads_restore = ft;
2163 esw->offloads.restore_group = g;
2164 esw->offloads.restore_copy_hdr_id = mod_hdr;
2165
2166 kvfree(flow_group_in);
2167
2168 return 0;
2169
2170 err_mod_hdr:
2171 mlx5_destroy_flow_group(g);
2172 err_group:
2173 mlx5_destroy_flow_table(ft);
2174 out_free:
2175 kvfree(flow_group_in);
2176
2177 return err;
2178 }
2179
esw_offloads_start(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)2180 static int esw_offloads_start(struct mlx5_eswitch *esw,
2181 struct netlink_ext_ack *extack)
2182 {
2183 int err, err1;
2184
2185 mlx5_eswitch_disable_locked(esw, false);
2186 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS,
2187 esw->dev->priv.sriov.num_vfs);
2188 if (err) {
2189 NL_SET_ERR_MSG_MOD(extack,
2190 "Failed setting eswitch to offloads");
2191 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY,
2192 MLX5_ESWITCH_IGNORE_NUM_VFS);
2193 if (err1) {
2194 NL_SET_ERR_MSG_MOD(extack,
2195 "Failed setting eswitch back to legacy");
2196 }
2197 }
2198 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2199 if (mlx5_eswitch_inline_mode_get(esw,
2200 &esw->offloads.inline_mode)) {
2201 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2202 NL_SET_ERR_MSG_MOD(extack,
2203 "Inline mode is different between vports");
2204 }
2205 }
2206 return err;
2207 }
2208
mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,xa_mark_t mark)2209 static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch *esw,
2210 struct mlx5_eswitch_rep *rep,
2211 xa_mark_t mark)
2212 {
2213 bool mark_set;
2214
2215 /* Copy the mark from vport to its rep */
2216 mark_set = xa_get_mark(&esw->vports, rep->vport, mark);
2217 if (mark_set)
2218 xa_set_mark(&esw->offloads.vport_reps, rep->vport, mark);
2219 }
2220
mlx5_esw_offloads_rep_init(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2221 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
2222 {
2223 struct mlx5_eswitch_rep *rep;
2224 int rep_type;
2225 int err;
2226
2227 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2228 if (!rep)
2229 return -ENOMEM;
2230
2231 rep->vport = vport->vport;
2232 rep->vport_index = vport->index;
2233 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2234 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
2235
2236 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2237 if (err)
2238 goto insert_err;
2239
2240 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_HOST_FN);
2241 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_VF);
2242 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_SF);
2243 return 0;
2244
2245 insert_err:
2246 kfree(rep);
2247 return err;
2248 }
2249
mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep)2250 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2251 struct mlx5_eswitch_rep *rep)
2252 {
2253 xa_erase(&esw->offloads.vport_reps, rep->vport);
2254 kfree(rep);
2255 }
2256
esw_offloads_cleanup_reps(struct mlx5_eswitch * esw)2257 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2258 {
2259 struct mlx5_eswitch_rep *rep;
2260 unsigned long i;
2261
2262 mlx5_esw_for_each_rep(esw, i, rep)
2263 mlx5_esw_offloads_rep_cleanup(esw, rep);
2264 xa_destroy(&esw->offloads.vport_reps);
2265 }
2266
esw_offloads_init_reps(struct mlx5_eswitch * esw)2267 int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2268 {
2269 struct mlx5_vport *vport;
2270 unsigned long i;
2271 int err;
2272
2273 xa_init(&esw->offloads.vport_reps);
2274
2275 mlx5_esw_for_each_vport(esw, i, vport) {
2276 err = mlx5_esw_offloads_rep_init(esw, vport);
2277 if (err)
2278 goto err;
2279 }
2280 return 0;
2281
2282 err:
2283 esw_offloads_cleanup_reps(esw);
2284 return err;
2285 }
2286
__esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2287 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2288 struct mlx5_eswitch_rep *rep, u8 rep_type)
2289 {
2290 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2291 REP_LOADED, REP_REGISTERED) == REP_LOADED)
2292 esw->offloads.rep_ops[rep_type]->unload(rep);
2293 }
2294
__unload_reps_sf_vport(struct mlx5_eswitch * esw,u8 rep_type)2295 static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type)
2296 {
2297 struct mlx5_eswitch_rep *rep;
2298 unsigned long i;
2299
2300 mlx5_esw_for_each_sf_rep(esw, i, rep)
2301 __esw_offloads_unload_rep(esw, rep, rep_type);
2302 }
2303
__unload_reps_all_vport(struct mlx5_eswitch * esw,u8 rep_type)2304 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2305 {
2306 struct mlx5_eswitch_rep *rep;
2307 unsigned long i;
2308
2309 __unload_reps_sf_vport(esw, rep_type);
2310
2311 mlx5_esw_for_each_vf_rep(esw, i, rep)
2312 __esw_offloads_unload_rep(esw, rep, rep_type);
2313
2314 if (mlx5_ecpf_vport_exists(esw->dev)) {
2315 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF);
2316 __esw_offloads_unload_rep(esw, rep, rep_type);
2317 }
2318
2319 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
2320 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF);
2321 __esw_offloads_unload_rep(esw, rep, rep_type);
2322 }
2323
2324 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
2325 __esw_offloads_unload_rep(esw, rep, rep_type);
2326 }
2327
mlx5_esw_offloads_rep_load(struct mlx5_eswitch * esw,u16 vport_num)2328 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2329 {
2330 struct mlx5_eswitch_rep *rep;
2331 int rep_type;
2332 int err;
2333
2334 rep = mlx5_eswitch_get_rep(esw, vport_num);
2335 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2336 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2337 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) {
2338 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2339 if (err)
2340 goto err_reps;
2341 }
2342
2343 return 0;
2344
2345 err_reps:
2346 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2347 for (--rep_type; rep_type >= 0; rep_type--)
2348 __esw_offloads_unload_rep(esw, rep, rep_type);
2349 return err;
2350 }
2351
mlx5_esw_offloads_rep_unload(struct mlx5_eswitch * esw,u16 vport_num)2352 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2353 {
2354 struct mlx5_eswitch_rep *rep;
2355 int rep_type;
2356
2357 rep = mlx5_eswitch_get_rep(esw, vport_num);
2358 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2359 __esw_offloads_unload_rep(esw, rep, rep_type);
2360 }
2361
esw_offloads_load_rep(struct mlx5_eswitch * esw,u16 vport_num)2362 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num)
2363 {
2364 int err;
2365
2366 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2367 return 0;
2368
2369 if (vport_num != MLX5_VPORT_UPLINK) {
2370 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num);
2371 if (err)
2372 return err;
2373 }
2374
2375 err = mlx5_esw_offloads_rep_load(esw, vport_num);
2376 if (err)
2377 goto load_err;
2378 return err;
2379
2380 load_err:
2381 if (vport_num != MLX5_VPORT_UPLINK)
2382 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
2383 return err;
2384 }
2385
esw_offloads_unload_rep(struct mlx5_eswitch * esw,u16 vport_num)2386 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num)
2387 {
2388 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2389 return;
2390
2391 mlx5_esw_offloads_rep_unload(esw, vport_num);
2392
2393 if (vport_num != MLX5_VPORT_UPLINK)
2394 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
2395 }
2396
esw_set_slave_root_fdb(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2397 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2398 struct mlx5_core_dev *slave)
2399 {
2400 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2401 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2402 struct mlx5_flow_root_namespace *root;
2403 struct mlx5_flow_namespace *ns;
2404 int err;
2405
2406 MLX5_SET(set_flow_table_root_in, in, opcode,
2407 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2408 MLX5_SET(set_flow_table_root_in, in, table_type,
2409 FS_FT_FDB);
2410
2411 if (master) {
2412 ns = mlx5_get_flow_namespace(master,
2413 MLX5_FLOW_NAMESPACE_FDB);
2414 root = find_root(&ns->node);
2415 mutex_lock(&root->chain_lock);
2416 MLX5_SET(set_flow_table_root_in, in,
2417 table_eswitch_owner_vhca_id_valid, 1);
2418 MLX5_SET(set_flow_table_root_in, in,
2419 table_eswitch_owner_vhca_id,
2420 MLX5_CAP_GEN(master, vhca_id));
2421 MLX5_SET(set_flow_table_root_in, in, table_id,
2422 root->root_ft->id);
2423 } else {
2424 ns = mlx5_get_flow_namespace(slave,
2425 MLX5_FLOW_NAMESPACE_FDB);
2426 root = find_root(&ns->node);
2427 mutex_lock(&root->chain_lock);
2428 MLX5_SET(set_flow_table_root_in, in, table_id,
2429 root->root_ft->id);
2430 }
2431
2432 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2433 mutex_unlock(&root->chain_lock);
2434
2435 return err;
2436 }
2437
__esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,struct mlx5_vport * vport,struct mlx5_flow_table * acl)2438 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2439 struct mlx5_core_dev *slave,
2440 struct mlx5_vport *vport,
2441 struct mlx5_flow_table *acl)
2442 {
2443 struct mlx5_flow_handle *flow_rule = NULL;
2444 struct mlx5_flow_destination dest = {};
2445 struct mlx5_flow_act flow_act = {};
2446 struct mlx5_flow_spec *spec;
2447 int err = 0;
2448 void *misc;
2449
2450 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2451 if (!spec)
2452 return -ENOMEM;
2453
2454 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2455 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2456 misc_parameters);
2457 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2458 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
2459 MLX5_CAP_GEN(slave, vhca_id));
2460
2461 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2462 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2463 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2464 source_eswitch_owner_vhca_id);
2465
2466 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2467 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2468 dest.vport.num = slave->priv.eswitch->manager_vport;
2469 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2470 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2471
2472 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2473 &dest, 1);
2474 if (IS_ERR(flow_rule))
2475 err = PTR_ERR(flow_rule);
2476 else
2477 vport->egress.offloads.bounce_rule = flow_rule;
2478
2479 kvfree(spec);
2480 return err;
2481 }
2482
esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2483 static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2484 struct mlx5_core_dev *slave)
2485 {
2486 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2487 struct mlx5_eswitch *esw = master->priv.eswitch;
2488 struct mlx5_flow_table_attr ft_attr = {
2489 .max_fte = 1, .prio = 0, .level = 0,
2490 .flags = MLX5_FLOW_TABLE_OTHER_VPORT,
2491 };
2492 struct mlx5_flow_namespace *egress_ns;
2493 struct mlx5_flow_table *acl;
2494 struct mlx5_flow_group *g;
2495 struct mlx5_vport *vport;
2496 void *match_criteria;
2497 u32 *flow_group_in;
2498 int err;
2499
2500 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
2501 if (IS_ERR(vport))
2502 return PTR_ERR(vport);
2503
2504 egress_ns = mlx5_get_flow_vport_acl_namespace(master,
2505 MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2506 vport->index);
2507 if (!egress_ns)
2508 return -EINVAL;
2509
2510 if (vport->egress.acl)
2511 return -EINVAL;
2512
2513 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2514 if (!flow_group_in)
2515 return -ENOMEM;
2516
2517 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2518 if (IS_ERR(acl)) {
2519 err = PTR_ERR(acl);
2520 goto out;
2521 }
2522
2523 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2524 match_criteria);
2525 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2526 misc_parameters.source_port);
2527 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2528 misc_parameters.source_eswitch_owner_vhca_id);
2529 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2530 MLX5_MATCH_MISC_PARAMETERS);
2531
2532 MLX5_SET(create_flow_group_in, flow_group_in,
2533 source_eswitch_owner_vhca_id_valid, 1);
2534 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2535 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0);
2536
2537 g = mlx5_create_flow_group(acl, flow_group_in);
2538 if (IS_ERR(g)) {
2539 err = PTR_ERR(g);
2540 goto err_group;
2541 }
2542
2543 err = __esw_set_master_egress_rule(master, slave, vport, acl);
2544 if (err)
2545 goto err_rule;
2546
2547 vport->egress.acl = acl;
2548 vport->egress.offloads.bounce_grp = g;
2549
2550 kvfree(flow_group_in);
2551
2552 return 0;
2553
2554 err_rule:
2555 mlx5_destroy_flow_group(g);
2556 err_group:
2557 mlx5_destroy_flow_table(acl);
2558 out:
2559 kvfree(flow_group_in);
2560 return err;
2561 }
2562
esw_unset_master_egress_rule(struct mlx5_core_dev * dev)2563 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev)
2564 {
2565 struct mlx5_vport *vport;
2566
2567 vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
2568 dev->priv.eswitch->manager_vport);
2569
2570 esw_acl_egress_ofld_cleanup(vport);
2571 }
2572
mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2573 int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch *master_esw,
2574 struct mlx5_eswitch *slave_esw)
2575 {
2576 int err;
2577
2578 err = esw_set_slave_root_fdb(master_esw->dev,
2579 slave_esw->dev);
2580 if (err)
2581 return err;
2582
2583 err = esw_set_master_egress_rule(master_esw->dev,
2584 slave_esw->dev);
2585 if (err)
2586 goto err_acl;
2587
2588 return err;
2589
2590 err_acl:
2591 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2592
2593 return err;
2594 }
2595
mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2596 void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw,
2597 struct mlx5_eswitch *slave_esw)
2598 {
2599 esw_unset_master_egress_rule(master_esw->dev);
2600 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2601 }
2602
2603 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
2604 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2605
mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch * esw)2606 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw)
2607 {
2608 const struct mlx5_eswitch_rep_ops *ops;
2609 struct mlx5_eswitch_rep *rep;
2610 unsigned long i;
2611 u8 rep_type;
2612
2613 mlx5_esw_for_each_rep(esw, i, rep) {
2614 rep_type = NUM_REP_TYPES;
2615 while (rep_type--) {
2616 ops = esw->offloads.rep_ops[rep_type];
2617 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2618 ops->event)
2619 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, NULL);
2620 }
2621 }
2622 }
2623
mlx5_esw_offloads_unpair(struct mlx5_eswitch * esw)2624 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw)
2625 {
2626 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2627 mlx5e_tc_clean_fdb_peer_flows(esw);
2628 #endif
2629 mlx5_esw_offloads_rep_event_unpair(esw);
2630 esw_del_fdb_peer_miss_rules(esw);
2631 }
2632
mlx5_esw_offloads_pair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2633 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2634 struct mlx5_eswitch *peer_esw)
2635 {
2636 const struct mlx5_eswitch_rep_ops *ops;
2637 struct mlx5_eswitch_rep *rep;
2638 unsigned long i;
2639 u8 rep_type;
2640 int err;
2641
2642 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2643 if (err)
2644 return err;
2645
2646 mlx5_esw_for_each_rep(esw, i, rep) {
2647 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2648 ops = esw->offloads.rep_ops[rep_type];
2649 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2650 ops->event) {
2651 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2652 if (err)
2653 goto err_out;
2654 }
2655 }
2656 }
2657
2658 return 0;
2659
2660 err_out:
2661 mlx5_esw_offloads_unpair(esw);
2662 return err;
2663 }
2664
mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,bool pair)2665 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2666 struct mlx5_eswitch *peer_esw,
2667 bool pair)
2668 {
2669 struct mlx5_flow_root_namespace *peer_ns;
2670 struct mlx5_flow_root_namespace *ns;
2671 int err;
2672
2673 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2674 ns = esw->dev->priv.steering->fdb_root_ns;
2675
2676 if (pair) {
2677 err = mlx5_flow_namespace_set_peer(ns, peer_ns);
2678 if (err)
2679 return err;
2680
2681 err = mlx5_flow_namespace_set_peer(peer_ns, ns);
2682 if (err) {
2683 mlx5_flow_namespace_set_peer(ns, NULL);
2684 return err;
2685 }
2686 } else {
2687 mlx5_flow_namespace_set_peer(ns, NULL);
2688 mlx5_flow_namespace_set_peer(peer_ns, NULL);
2689 }
2690
2691 return 0;
2692 }
2693
mlx5_esw_offloads_devcom_event(int event,void * my_data,void * event_data)2694 static int mlx5_esw_offloads_devcom_event(int event,
2695 void *my_data,
2696 void *event_data)
2697 {
2698 struct mlx5_eswitch *esw = my_data;
2699 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2700 struct mlx5_eswitch *peer_esw = event_data;
2701 int err;
2702
2703 switch (event) {
2704 case ESW_OFFLOADS_DEVCOM_PAIR:
2705 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2706 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2707 break;
2708
2709 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
2710 if (err)
2711 goto err_out;
2712 err = mlx5_esw_offloads_pair(esw, peer_esw);
2713 if (err)
2714 goto err_peer;
2715
2716 err = mlx5_esw_offloads_pair(peer_esw, esw);
2717 if (err)
2718 goto err_pair;
2719
2720 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true);
2721 break;
2722
2723 case ESW_OFFLOADS_DEVCOM_UNPAIR:
2724 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS))
2725 break;
2726
2727 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false);
2728 mlx5_esw_offloads_unpair(peer_esw);
2729 mlx5_esw_offloads_unpair(esw);
2730 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
2731 break;
2732 }
2733
2734 return 0;
2735
2736 err_pair:
2737 mlx5_esw_offloads_unpair(esw);
2738 err_peer:
2739 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
2740 err_out:
2741 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
2742 event, err);
2743 return err;
2744 }
2745
esw_offloads_devcom_init(struct mlx5_eswitch * esw)2746 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw)
2747 {
2748 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2749
2750 INIT_LIST_HEAD(&esw->offloads.peer_flows);
2751 mutex_init(&esw->offloads.peer_mutex);
2752
2753 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2754 return;
2755
2756 if (!mlx5_is_lag_supported(esw->dev))
2757 return;
2758
2759 mlx5_devcom_register_component(devcom,
2760 MLX5_DEVCOM_ESW_OFFLOADS,
2761 mlx5_esw_offloads_devcom_event,
2762 esw);
2763
2764 mlx5_devcom_send_event(devcom,
2765 MLX5_DEVCOM_ESW_OFFLOADS,
2766 ESW_OFFLOADS_DEVCOM_PAIR, esw);
2767 }
2768
esw_offloads_devcom_cleanup(struct mlx5_eswitch * esw)2769 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
2770 {
2771 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2772
2773 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2774 return;
2775
2776 if (!mlx5_is_lag_supported(esw->dev))
2777 return;
2778
2779 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS,
2780 ESW_OFFLOADS_DEVCOM_UNPAIR, esw);
2781
2782 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
2783 }
2784
mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch * esw)2785 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
2786 {
2787 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
2788 return false;
2789
2790 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
2791 MLX5_FDB_TO_VPORT_REG_C_0))
2792 return false;
2793
2794 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
2795 return false;
2796
2797 return true;
2798 }
2799
2800 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
2801
2802 /* Share the same metadata for uplink's. This is fine because:
2803 * (a) In shared FDB mode (LAG) both uplink's are treated the
2804 * same and tagged with the same metadata.
2805 * (b) In non shared FDB mode, packets from physical port0
2806 * cannot hit eswitch of PF1 and vice versa.
2807 */
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch * esw)2808 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
2809 {
2810 return MLX5_ESW_METADATA_RSVD_UPLINK;
2811 }
2812
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch * esw)2813 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
2814 {
2815 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
2816 /* Reserve 0xf for internal port offload */
2817 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
2818 u32 pf_num;
2819 int id;
2820
2821 /* Only 4 bits of pf_num */
2822 pf_num = mlx5_get_dev_index(esw->dev);
2823 if (pf_num > max_pf_num)
2824 return 0;
2825
2826 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
2827 /* Use only non-zero vport_id (2-4095) for all PF's */
2828 id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
2829 MLX5_ESW_METADATA_RSVD_UPLINK + 1,
2830 vport_end_ida, GFP_KERNEL);
2831 if (id < 0)
2832 return 0;
2833 id = (pf_num << ESW_VPORT_BITS) | id;
2834 return id;
2835 }
2836
mlx5_esw_match_metadata_free(struct mlx5_eswitch * esw,u32 metadata)2837 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
2838 {
2839 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
2840
2841 /* Metadata contains only 12 bits of actual ida id */
2842 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
2843 }
2844
esw_offloads_vport_metadata_setup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2845 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
2846 struct mlx5_vport *vport)
2847 {
2848 if (vport->vport == MLX5_VPORT_UPLINK)
2849 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
2850 else
2851 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
2852
2853 vport->metadata = vport->default_metadata;
2854 return vport->metadata ? 0 : -ENOSPC;
2855 }
2856
esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2857 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
2858 struct mlx5_vport *vport)
2859 {
2860 if (!vport->default_metadata)
2861 return;
2862
2863 if (vport->vport == MLX5_VPORT_UPLINK)
2864 return;
2865
2866 WARN_ON(vport->metadata != vport->default_metadata);
2867 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
2868 }
2869
esw_offloads_metadata_uninit(struct mlx5_eswitch * esw)2870 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
2871 {
2872 struct mlx5_vport *vport;
2873 unsigned long i;
2874
2875 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
2876 return;
2877
2878 mlx5_esw_for_each_vport(esw, i, vport)
2879 esw_offloads_vport_metadata_cleanup(esw, vport);
2880 }
2881
esw_offloads_metadata_init(struct mlx5_eswitch * esw)2882 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
2883 {
2884 struct mlx5_vport *vport;
2885 unsigned long i;
2886 int err;
2887
2888 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
2889 return 0;
2890
2891 mlx5_esw_for_each_vport(esw, i, vport) {
2892 err = esw_offloads_vport_metadata_setup(esw, vport);
2893 if (err)
2894 goto metadata_err;
2895 }
2896
2897 return 0;
2898
2899 metadata_err:
2900 esw_offloads_metadata_uninit(esw);
2901 return err;
2902 }
2903
mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch * esw,bool enable)2904 int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable)
2905 {
2906 int err = 0;
2907
2908 down_write(&esw->mode_lock);
2909 if (esw->mode != MLX5_ESWITCH_NONE) {
2910 err = -EBUSY;
2911 goto done;
2912 }
2913 if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2914 err = -EOPNOTSUPP;
2915 goto done;
2916 }
2917 if (enable)
2918 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2919 else
2920 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2921 done:
2922 up_write(&esw->mode_lock);
2923 return err;
2924 }
2925
2926 int
esw_vport_create_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2927 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
2928 struct mlx5_vport *vport)
2929 {
2930 int err;
2931
2932 err = esw_acl_ingress_ofld_setup(esw, vport);
2933 if (err)
2934 return err;
2935
2936 err = esw_acl_egress_ofld_setup(esw, vport);
2937 if (err)
2938 goto egress_err;
2939
2940 return 0;
2941
2942 egress_err:
2943 esw_acl_ingress_ofld_cleanup(esw, vport);
2944 return err;
2945 }
2946
2947 void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2948 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
2949 struct mlx5_vport *vport)
2950 {
2951 esw_acl_egress_ofld_cleanup(vport);
2952 esw_acl_ingress_ofld_cleanup(esw, vport);
2953 }
2954
esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch * esw)2955 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
2956 {
2957 struct mlx5_vport *vport;
2958
2959 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
2960 if (IS_ERR(vport))
2961 return PTR_ERR(vport);
2962
2963 return esw_vport_create_offloads_acl_tables(esw, vport);
2964 }
2965
esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch * esw)2966 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
2967 {
2968 struct mlx5_vport *vport;
2969
2970 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
2971 if (IS_ERR(vport))
2972 return;
2973
2974 esw_vport_destroy_offloads_acl_tables(esw, vport);
2975 }
2976
mlx5_eswitch_reload_reps(struct mlx5_eswitch * esw)2977 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
2978 {
2979 struct mlx5_eswitch_rep *rep;
2980 unsigned long i;
2981 int ret;
2982
2983 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
2984 return 0;
2985
2986 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
2987 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
2988 return 0;
2989
2990 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
2991 if (ret)
2992 return ret;
2993
2994 mlx5_esw_for_each_rep(esw, i, rep) {
2995 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
2996 mlx5_esw_offloads_rep_load(esw, rep->vport);
2997 }
2998
2999 return 0;
3000 }
3001
esw_offloads_steering_init(struct mlx5_eswitch * esw)3002 static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3003 {
3004 struct mlx5_esw_indir_table *indir;
3005 int err;
3006
3007 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3008 mutex_init(&esw->fdb_table.offloads.vports.lock);
3009 hash_init(esw->fdb_table.offloads.vports.table);
3010 atomic64_set(&esw->user_count, 0);
3011
3012 indir = mlx5_esw_indir_table_init();
3013 if (IS_ERR(indir)) {
3014 err = PTR_ERR(indir);
3015 goto create_indir_err;
3016 }
3017 esw->fdb_table.offloads.indir = indir;
3018
3019 err = esw_create_uplink_offloads_acl_tables(esw);
3020 if (err)
3021 goto create_acl_err;
3022
3023 err = esw_create_offloads_table(esw);
3024 if (err)
3025 goto create_offloads_err;
3026
3027 err = esw_create_restore_table(esw);
3028 if (err)
3029 goto create_restore_err;
3030
3031 err = esw_create_offloads_fdb_tables(esw);
3032 if (err)
3033 goto create_fdb_err;
3034
3035 err = esw_create_vport_rx_group(esw);
3036 if (err)
3037 goto create_fg_err;
3038
3039 return 0;
3040
3041 create_fg_err:
3042 esw_destroy_offloads_fdb_tables(esw);
3043 create_fdb_err:
3044 esw_destroy_restore_table(esw);
3045 create_restore_err:
3046 esw_destroy_offloads_table(esw);
3047 create_offloads_err:
3048 esw_destroy_uplink_offloads_acl_tables(esw);
3049 create_acl_err:
3050 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3051 create_indir_err:
3052 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3053 return err;
3054 }
3055
esw_offloads_steering_cleanup(struct mlx5_eswitch * esw)3056 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3057 {
3058 esw_destroy_vport_rx_group(esw);
3059 esw_destroy_offloads_fdb_tables(esw);
3060 esw_destroy_restore_table(esw);
3061 esw_destroy_offloads_table(esw);
3062 esw_destroy_uplink_offloads_acl_tables(esw);
3063 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3064 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3065 }
3066
3067 static void
esw_vfs_changed_event_handler(struct mlx5_eswitch * esw,const u32 * out)3068 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3069 {
3070 bool host_pf_disabled;
3071 u16 new_num_vfs;
3072
3073 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3074 host_params_context.host_num_of_vfs);
3075 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3076 host_params_context.host_pf_disabled);
3077
3078 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3079 return;
3080
3081 /* Number of VFs can only change from "0 to x" or "x to 0". */
3082 if (esw->esw_funcs.num_vfs > 0) {
3083 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
3084 } else {
3085 int err;
3086
3087 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3088 MLX5_VPORT_UC_ADDR_CHANGE);
3089 if (err)
3090 return;
3091 }
3092 esw->esw_funcs.num_vfs = new_num_vfs;
3093 }
3094
esw_functions_changed_event_handler(struct work_struct * work)3095 static void esw_functions_changed_event_handler(struct work_struct *work)
3096 {
3097 struct mlx5_host_work *host_work;
3098 struct mlx5_eswitch *esw;
3099 const u32 *out;
3100
3101 host_work = container_of(work, struct mlx5_host_work, work);
3102 esw = host_work->esw;
3103
3104 out = mlx5_esw_query_functions(esw->dev);
3105 if (IS_ERR(out))
3106 goto out;
3107
3108 esw_vfs_changed_event_handler(esw, out);
3109 kvfree(out);
3110 out:
3111 kfree(host_work);
3112 }
3113
mlx5_esw_funcs_changed_handler(struct notifier_block * nb,unsigned long type,void * data)3114 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3115 {
3116 struct mlx5_esw_functions *esw_funcs;
3117 struct mlx5_host_work *host_work;
3118 struct mlx5_eswitch *esw;
3119
3120 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3121 if (!host_work)
3122 return NOTIFY_DONE;
3123
3124 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3125 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3126
3127 host_work->esw = esw;
3128
3129 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3130 queue_work(esw->work_queue, &host_work->work);
3131
3132 return NOTIFY_OK;
3133 }
3134
mlx5_esw_host_number_init(struct mlx5_eswitch * esw)3135 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3136 {
3137 const u32 *query_host_out;
3138
3139 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3140 return 0;
3141
3142 query_host_out = mlx5_esw_query_functions(esw->dev);
3143 if (IS_ERR(query_host_out))
3144 return PTR_ERR(query_host_out);
3145
3146 /* Mark non local controller with non zero controller number. */
3147 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3148 host_params_context.host_number);
3149 kvfree(query_host_out);
3150 return 0;
3151 }
3152
mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch * esw,u32 controller)3153 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3154 {
3155 /* Local controller is always valid */
3156 if (controller == 0)
3157 return true;
3158
3159 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3160 return false;
3161
3162 /* External host number starts with zero in device */
3163 return (controller == esw->offloads.host_number + 1);
3164 }
3165
esw_offloads_enable(struct mlx5_eswitch * esw)3166 int esw_offloads_enable(struct mlx5_eswitch *esw)
3167 {
3168 struct mapping_ctx *reg_c0_obj_pool;
3169 struct mlx5_vport *vport;
3170 unsigned long i;
3171 u64 mapping_id;
3172 int err;
3173
3174 mutex_init(&esw->offloads.termtbl_mutex);
3175 mlx5_rdma_enable_roce(esw->dev);
3176
3177 err = mlx5_esw_host_number_init(esw);
3178 if (err)
3179 goto err_metadata;
3180
3181 err = esw_offloads_metadata_init(esw);
3182 if (err)
3183 goto err_metadata;
3184
3185 err = esw_set_passing_vport_metadata(esw, true);
3186 if (err)
3187 goto err_vport_metadata;
3188
3189 mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
3190
3191 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN,
3192 sizeof(struct mlx5_mapped_obj),
3193 ESW_REG_C0_USER_DATA_METADATA_MASK,
3194 true);
3195
3196 if (IS_ERR(reg_c0_obj_pool)) {
3197 err = PTR_ERR(reg_c0_obj_pool);
3198 goto err_pool;
3199 }
3200 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3201
3202 err = esw_offloads_steering_init(esw);
3203 if (err)
3204 goto err_steering_init;
3205
3206 /* Representor will control the vport link state */
3207 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3208 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3209
3210 /* Uplink vport rep must load first. */
3211 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK);
3212 if (err)
3213 goto err_uplink;
3214
3215 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
3216 if (err)
3217 goto err_vports;
3218
3219 esw_offloads_devcom_init(esw);
3220
3221 return 0;
3222
3223 err_vports:
3224 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
3225 err_uplink:
3226 esw_offloads_steering_cleanup(esw);
3227 err_steering_init:
3228 mapping_destroy(reg_c0_obj_pool);
3229 err_pool:
3230 esw_set_passing_vport_metadata(esw, false);
3231 err_vport_metadata:
3232 esw_offloads_metadata_uninit(esw);
3233 err_metadata:
3234 mlx5_rdma_disable_roce(esw->dev);
3235 mutex_destroy(&esw->offloads.termtbl_mutex);
3236 return err;
3237 }
3238
esw_offloads_stop(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)3239 static int esw_offloads_stop(struct mlx5_eswitch *esw,
3240 struct netlink_ext_ack *extack)
3241 {
3242 int err, err1;
3243
3244 mlx5_eswitch_disable_locked(esw, false);
3245 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY,
3246 MLX5_ESWITCH_IGNORE_NUM_VFS);
3247 if (err) {
3248 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3249 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS,
3250 MLX5_ESWITCH_IGNORE_NUM_VFS);
3251 if (err1) {
3252 NL_SET_ERR_MSG_MOD(extack,
3253 "Failed setting eswitch back to offloads");
3254 }
3255 }
3256
3257 return err;
3258 }
3259
esw_offloads_disable(struct mlx5_eswitch * esw)3260 void esw_offloads_disable(struct mlx5_eswitch *esw)
3261 {
3262 esw_offloads_devcom_cleanup(esw);
3263 mlx5_eswitch_disable_pf_vf_vports(esw);
3264 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
3265 esw_set_passing_vport_metadata(esw, false);
3266 esw_offloads_steering_cleanup(esw);
3267 mapping_destroy(esw->offloads.reg_c0_obj_pool);
3268 esw_offloads_metadata_uninit(esw);
3269 mlx5_rdma_disable_roce(esw->dev);
3270 mutex_destroy(&esw->offloads.termtbl_mutex);
3271 }
3272
esw_mode_from_devlink(u16 mode,u16 * mlx5_mode)3273 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3274 {
3275 switch (mode) {
3276 case DEVLINK_ESWITCH_MODE_LEGACY:
3277 *mlx5_mode = MLX5_ESWITCH_LEGACY;
3278 break;
3279 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3280 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3281 break;
3282 default:
3283 return -EINVAL;
3284 }
3285
3286 return 0;
3287 }
3288
esw_mode_to_devlink(u16 mlx5_mode,u16 * mode)3289 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3290 {
3291 switch (mlx5_mode) {
3292 case MLX5_ESWITCH_LEGACY:
3293 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
3294 break;
3295 case MLX5_ESWITCH_OFFLOADS:
3296 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3297 break;
3298 default:
3299 return -EINVAL;
3300 }
3301
3302 return 0;
3303 }
3304
esw_inline_mode_from_devlink(u8 mode,u8 * mlx5_mode)3305 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3306 {
3307 switch (mode) {
3308 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3309 *mlx5_mode = MLX5_INLINE_MODE_NONE;
3310 break;
3311 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3312 *mlx5_mode = MLX5_INLINE_MODE_L2;
3313 break;
3314 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3315 *mlx5_mode = MLX5_INLINE_MODE_IP;
3316 break;
3317 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3318 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3319 break;
3320 default:
3321 return -EINVAL;
3322 }
3323
3324 return 0;
3325 }
3326
esw_inline_mode_to_devlink(u8 mlx5_mode,u8 * mode)3327 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3328 {
3329 switch (mlx5_mode) {
3330 case MLX5_INLINE_MODE_NONE:
3331 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3332 break;
3333 case MLX5_INLINE_MODE_L2:
3334 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3335 break;
3336 case MLX5_INLINE_MODE_IP:
3337 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3338 break;
3339 case MLX5_INLINE_MODE_TCP_UDP:
3340 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3341 break;
3342 default:
3343 return -EINVAL;
3344 }
3345
3346 return 0;
3347 }
3348
eswitch_devlink_esw_mode_check(const struct mlx5_eswitch * esw)3349 static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw)
3350 {
3351 /* devlink commands in NONE eswitch mode are currently supported only
3352 * on ECPF.
3353 */
3354 return (esw->mode == MLX5_ESWITCH_NONE &&
3355 !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0;
3356 }
3357
3358 /* FIXME: devl_unlock() followed by devl_lock() inside driver callback
3359 * is never correct and prone to races. It's a transitional workaround,
3360 * never repeat this pattern.
3361 *
3362 * This code MUST be fixed before removing devlink_mutex as it is safe
3363 * to do only because of that mutex.
3364 */
mlx5_eswtich_mode_callback_enter(struct devlink * devlink,struct mlx5_eswitch * esw)3365 static void mlx5_eswtich_mode_callback_enter(struct devlink *devlink,
3366 struct mlx5_eswitch *esw)
3367 {
3368 devl_unlock(devlink);
3369 down_write(&esw->mode_lock);
3370 }
3371
mlx5_eswtich_mode_callback_exit(struct devlink * devlink,struct mlx5_eswitch * esw)3372 static void mlx5_eswtich_mode_callback_exit(struct devlink *devlink,
3373 struct mlx5_eswitch *esw)
3374 {
3375 up_write(&esw->mode_lock);
3376 devl_lock(devlink);
3377 }
3378
mlx5_devlink_eswitch_mode_set(struct devlink * devlink,u16 mode,struct netlink_ext_ack * extack)3379 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3380 struct netlink_ext_ack *extack)
3381 {
3382 u16 cur_mlx5_mode, mlx5_mode = 0;
3383 struct mlx5_eswitch *esw;
3384 int err = 0;
3385
3386 esw = mlx5_devlink_eswitch_get(devlink);
3387 if (IS_ERR(esw))
3388 return PTR_ERR(esw);
3389
3390 if (esw_mode_from_devlink(mode, &mlx5_mode))
3391 return -EINVAL;
3392
3393 /* FIXME: devl_unlock() followed by devl_lock() inside driver callback
3394 * is never correct and prone to races. It's a transitional workaround,
3395 * never repeat this pattern.
3396 *
3397 * This code MUST be fixed before removing devlink_mutex as it is safe
3398 * to do only because of that mutex.
3399 */
3400 devl_unlock(devlink);
3401
3402 mlx5_lag_disable_change(esw->dev);
3403 err = mlx5_esw_try_lock(esw);
3404 if (err < 0) {
3405 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
3406 goto enable_lag;
3407 }
3408 cur_mlx5_mode = err;
3409 err = 0;
3410
3411 if (cur_mlx5_mode == mlx5_mode)
3412 goto unlock;
3413
3414 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3415 if (mlx5_devlink_trap_get_num_active(esw->dev)) {
3416 NL_SET_ERR_MSG_MOD(extack,
3417 "Can't change mode while devlink traps are active");
3418 err = -EOPNOTSUPP;
3419 goto unlock;
3420 }
3421 err = esw_offloads_start(esw, extack);
3422 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
3423 err = esw_offloads_stop(esw, extack);
3424 } else {
3425 err = -EINVAL;
3426 }
3427
3428 unlock:
3429 mlx5_esw_unlock(esw);
3430 enable_lag:
3431 mlx5_lag_enable_change(esw->dev);
3432 devl_lock(devlink);
3433 return err;
3434 }
3435
mlx5_devlink_eswitch_mode_get(struct devlink * devlink,u16 * mode)3436 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3437 {
3438 struct mlx5_eswitch *esw;
3439 int err;
3440
3441 esw = mlx5_devlink_eswitch_get(devlink);
3442 if (IS_ERR(esw))
3443 return PTR_ERR(esw);
3444
3445 mlx5_eswtich_mode_callback_enter(devlink, esw);
3446 err = eswitch_devlink_esw_mode_check(esw);
3447 if (err)
3448 goto unlock;
3449
3450 err = esw_mode_to_devlink(esw->mode, mode);
3451 unlock:
3452 mlx5_eswtich_mode_callback_exit(devlink, esw);
3453 return err;
3454 }
3455
mlx5_esw_vports_inline_set(struct mlx5_eswitch * esw,u8 mlx5_mode,struct netlink_ext_ack * extack)3456 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3457 struct netlink_ext_ack *extack)
3458 {
3459 struct mlx5_core_dev *dev = esw->dev;
3460 struct mlx5_vport *vport;
3461 u16 err_vport_num = 0;
3462 unsigned long i;
3463 int err = 0;
3464
3465 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3466 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3467 if (err) {
3468 err_vport_num = vport->vport;
3469 NL_SET_ERR_MSG_MOD(extack,
3470 "Failed to set min inline on vport");
3471 goto revert_inline_mode;
3472 }
3473 }
3474 return 0;
3475
3476 revert_inline_mode:
3477 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3478 if (vport->vport == err_vport_num)
3479 break;
3480 mlx5_modify_nic_vport_min_inline(dev,
3481 vport->vport,
3482 esw->offloads.inline_mode);
3483 }
3484 return err;
3485 }
3486
mlx5_devlink_eswitch_inline_mode_set(struct devlink * devlink,u8 mode,struct netlink_ext_ack * extack)3487 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3488 struct netlink_ext_ack *extack)
3489 {
3490 struct mlx5_core_dev *dev = devlink_priv(devlink);
3491 struct mlx5_eswitch *esw;
3492 u8 mlx5_mode;
3493 int err;
3494
3495 esw = mlx5_devlink_eswitch_get(devlink);
3496 if (IS_ERR(esw))
3497 return PTR_ERR(esw);
3498
3499 mlx5_eswtich_mode_callback_enter(devlink, esw);
3500 err = eswitch_devlink_esw_mode_check(esw);
3501 if (err)
3502 goto out;
3503
3504 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3505 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3506 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3507 err = 0;
3508 goto out;
3509 }
3510
3511 fallthrough;
3512 case MLX5_CAP_INLINE_MODE_L2:
3513 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
3514 err = -EOPNOTSUPP;
3515 goto out;
3516 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3517 break;
3518 }
3519
3520 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3521 NL_SET_ERR_MSG_MOD(extack,
3522 "Can't set inline mode when flows are configured");
3523 err = -EOPNOTSUPP;
3524 goto out;
3525 }
3526
3527 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
3528 if (err)
3529 goto out;
3530
3531 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3532 if (err)
3533 goto out;
3534
3535 esw->offloads.inline_mode = mlx5_mode;
3536 mlx5_eswtich_mode_callback_exit(devlink, esw);
3537 return 0;
3538
3539 out:
3540 mlx5_eswtich_mode_callback_exit(devlink, esw);
3541 return err;
3542 }
3543
mlx5_devlink_eswitch_inline_mode_get(struct devlink * devlink,u8 * mode)3544 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3545 {
3546 struct mlx5_eswitch *esw;
3547 int err;
3548
3549 esw = mlx5_devlink_eswitch_get(devlink);
3550 if (IS_ERR(esw))
3551 return PTR_ERR(esw);
3552
3553 mlx5_eswtich_mode_callback_enter(devlink, esw);
3554 err = eswitch_devlink_esw_mode_check(esw);
3555 if (err)
3556 goto unlock;
3557
3558 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3559 unlock:
3560 mlx5_eswtich_mode_callback_exit(devlink, esw);
3561 return err;
3562 }
3563
mlx5_devlink_eswitch_encap_mode_set(struct devlink * devlink,enum devlink_eswitch_encap_mode encap,struct netlink_ext_ack * extack)3564 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3565 enum devlink_eswitch_encap_mode encap,
3566 struct netlink_ext_ack *extack)
3567 {
3568 struct mlx5_core_dev *dev = devlink_priv(devlink);
3569 struct mlx5_eswitch *esw;
3570 int err;
3571
3572 esw = mlx5_devlink_eswitch_get(devlink);
3573 if (IS_ERR(esw))
3574 return PTR_ERR(esw);
3575
3576 mlx5_eswtich_mode_callback_enter(devlink, esw);
3577 err = eswitch_devlink_esw_mode_check(esw);
3578 if (err)
3579 goto unlock;
3580
3581 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
3582 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
3583 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3584 err = -EOPNOTSUPP;
3585 goto unlock;
3586 }
3587
3588 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3589 err = -EOPNOTSUPP;
3590 goto unlock;
3591 }
3592
3593 if (esw->mode == MLX5_ESWITCH_LEGACY) {
3594 esw->offloads.encap = encap;
3595 goto unlock;
3596 }
3597
3598 if (esw->offloads.encap == encap)
3599 goto unlock;
3600
3601 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3602 NL_SET_ERR_MSG_MOD(extack,
3603 "Can't set encapsulation when flows are configured");
3604 err = -EOPNOTSUPP;
3605 goto unlock;
3606 }
3607
3608 esw_destroy_offloads_fdb_tables(esw);
3609
3610 esw->offloads.encap = encap;
3611
3612 err = esw_create_offloads_fdb_tables(esw);
3613
3614 if (err) {
3615 NL_SET_ERR_MSG_MOD(extack,
3616 "Failed re-creating fast FDB table");
3617 esw->offloads.encap = !encap;
3618 (void)esw_create_offloads_fdb_tables(esw);
3619 }
3620
3621 unlock:
3622 mlx5_eswtich_mode_callback_exit(devlink, esw);
3623 return err;
3624 }
3625
mlx5_devlink_eswitch_encap_mode_get(struct devlink * devlink,enum devlink_eswitch_encap_mode * encap)3626 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
3627 enum devlink_eswitch_encap_mode *encap)
3628 {
3629 struct mlx5_eswitch *esw;
3630 int err;
3631
3632 esw = mlx5_devlink_eswitch_get(devlink);
3633 if (IS_ERR(esw))
3634 return PTR_ERR(esw);
3635
3636 mlx5_eswtich_mode_callback_enter(devlink, esw);
3637 err = eswitch_devlink_esw_mode_check(esw);
3638 if (err)
3639 goto unlock;
3640
3641 *encap = esw->offloads.encap;
3642 unlock:
3643 mlx5_eswtich_mode_callback_exit(devlink, esw);
3644 return err;
3645 }
3646
3647 static bool
mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch * esw,u16 vport_num)3648 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
3649 {
3650 /* Currently, only ECPF based device has representor for host PF. */
3651 if (vport_num == MLX5_VPORT_PF &&
3652 !mlx5_core_is_ecpf_esw_manager(esw->dev))
3653 return false;
3654
3655 if (vport_num == MLX5_VPORT_ECPF &&
3656 !mlx5_ecpf_vport_exists(esw->dev))
3657 return false;
3658
3659 return true;
3660 }
3661
mlx5_eswitch_register_vport_reps(struct mlx5_eswitch * esw,const struct mlx5_eswitch_rep_ops * ops,u8 rep_type)3662 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
3663 const struct mlx5_eswitch_rep_ops *ops,
3664 u8 rep_type)
3665 {
3666 struct mlx5_eswitch_rep_data *rep_data;
3667 struct mlx5_eswitch_rep *rep;
3668 unsigned long i;
3669
3670 esw->offloads.rep_ops[rep_type] = ops;
3671 mlx5_esw_for_each_rep(esw, i, rep) {
3672 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
3673 rep->esw = esw;
3674 rep_data = &rep->rep_data[rep_type];
3675 atomic_set(&rep_data->state, REP_REGISTERED);
3676 }
3677 }
3678 }
3679 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
3680
mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch * esw,u8 rep_type)3681 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
3682 {
3683 struct mlx5_eswitch_rep *rep;
3684 unsigned long i;
3685
3686 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
3687 __unload_reps_all_vport(esw, rep_type);
3688
3689 mlx5_esw_for_each_rep(esw, i, rep)
3690 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
3691 }
3692 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
3693
mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch * esw,u8 rep_type)3694 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
3695 {
3696 struct mlx5_eswitch_rep *rep;
3697
3698 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3699 return rep->rep_data[rep_type].priv;
3700 }
3701
mlx5_eswitch_get_proto_dev(struct mlx5_eswitch * esw,u16 vport,u8 rep_type)3702 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
3703 u16 vport,
3704 u8 rep_type)
3705 {
3706 struct mlx5_eswitch_rep *rep;
3707
3708 rep = mlx5_eswitch_get_rep(esw, vport);
3709
3710 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
3711 esw->offloads.rep_ops[rep_type]->get_proto_dev)
3712 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
3713 return NULL;
3714 }
3715 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
3716
mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch * esw,u8 rep_type)3717 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
3718 {
3719 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
3720 }
3721 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
3722
mlx5_eswitch_vport_rep(struct mlx5_eswitch * esw,u16 vport)3723 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
3724 u16 vport)
3725 {
3726 return mlx5_eswitch_get_rep(esw, vport);
3727 }
3728 EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
3729
mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch * esw)3730 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
3731 {
3732 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
3733 }
3734 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
3735
mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch * esw)3736 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
3737 {
3738 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
3739 }
3740 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
3741
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch * esw,u16 vport_num)3742 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
3743 u16 vport_num)
3744 {
3745 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3746
3747 if (WARN_ON_ONCE(IS_ERR(vport)))
3748 return 0;
3749
3750 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
3751 }
3752 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
3753
mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch * esw,struct devlink_port * dl_port,u16 vport_num,u32 controller,u32 sfnum)3754 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port,
3755 u16 vport_num, u32 controller, u32 sfnum)
3756 {
3757 int err;
3758
3759 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE);
3760 if (err)
3761 return err;
3762
3763 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, controller, sfnum);
3764 if (err)
3765 goto devlink_err;
3766
3767 err = mlx5_esw_offloads_rep_load(esw, vport_num);
3768 if (err)
3769 goto rep_err;
3770 return 0;
3771
3772 rep_err:
3773 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3774 devlink_err:
3775 mlx5_esw_vport_disable(esw, vport_num);
3776 return err;
3777 }
3778
mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch * esw,u16 vport_num)3779 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num)
3780 {
3781 mlx5_esw_offloads_rep_unload(esw, vport_num);
3782 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3783 mlx5_esw_vport_disable(esw, vport_num);
3784 }
3785
mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch * esw,u16 vport_num,u16 * vhca_id)3786 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
3787 {
3788 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
3789 void *query_ctx;
3790 void *hca_caps;
3791 int err;
3792
3793 *vhca_id = 0;
3794 if (mlx5_esw_is_manager_vport(esw, vport_num) ||
3795 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
3796 return -EPERM;
3797
3798 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
3799 if (!query_ctx)
3800 return -ENOMEM;
3801
3802 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx);
3803 if (err)
3804 goto out_free;
3805
3806 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
3807 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
3808
3809 out_free:
3810 kfree(query_ctx);
3811 return err;
3812 }
3813
mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch * esw,u16 vport_num)3814 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
3815 {
3816 u16 *old_entry, *vhca_map_entry, vhca_id;
3817 int err;
3818
3819 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3820 if (err) {
3821 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
3822 vport_num, err);
3823 return err;
3824 }
3825
3826 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
3827 if (!vhca_map_entry)
3828 return -ENOMEM;
3829
3830 *vhca_map_entry = vport_num;
3831 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
3832 if (xa_is_err(old_entry)) {
3833 kfree(vhca_map_entry);
3834 return xa_err(old_entry);
3835 }
3836 kfree(old_entry);
3837 return 0;
3838 }
3839
mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch * esw,u16 vport_num)3840 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
3841 {
3842 u16 *vhca_map_entry, vhca_id;
3843 int err;
3844
3845 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3846 if (err)
3847 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
3848 vport_num, err);
3849
3850 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
3851 kfree(vhca_map_entry);
3852 }
3853
mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch * esw,u16 vhca_id,u16 * vport_num)3854 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
3855 {
3856 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
3857
3858 if (!res)
3859 return -ENOENT;
3860
3861 *vport_num = *res;
3862 return 0;
3863 }
3864
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch * esw,u16 vport_num)3865 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
3866 u16 vport_num)
3867 {
3868 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3869
3870 if (WARN_ON_ONCE(IS_ERR(vport)))
3871 return 0;
3872
3873 return vport->metadata;
3874 }
3875 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
3876
3877 static bool
is_port_function_supported(struct mlx5_eswitch * esw,u16 vport_num)3878 is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num)
3879 {
3880 return vport_num == MLX5_VPORT_PF ||
3881 mlx5_eswitch_is_vf_vport(esw, vport_num) ||
3882 mlx5_esw_is_sf_vport(esw, vport_num);
3883 }
3884
mlx5_devlink_port_function_hw_addr_get(struct devlink_port * port,u8 * hw_addr,int * hw_addr_len,struct netlink_ext_ack * extack)3885 int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port,
3886 u8 *hw_addr, int *hw_addr_len,
3887 struct netlink_ext_ack *extack)
3888 {
3889 struct mlx5_eswitch *esw;
3890 struct mlx5_vport *vport;
3891 u16 vport_num;
3892
3893 esw = mlx5_devlink_eswitch_get(port->devlink);
3894 if (IS_ERR(esw))
3895 return PTR_ERR(esw);
3896
3897 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
3898 if (!is_port_function_supported(esw, vport_num))
3899 return -EOPNOTSUPP;
3900
3901 vport = mlx5_eswitch_get_vport(esw, vport_num);
3902 if (IS_ERR(vport)) {
3903 NL_SET_ERR_MSG_MOD(extack, "Invalid port");
3904 return PTR_ERR(vport);
3905 }
3906
3907 mutex_lock(&esw->state_lock);
3908 ether_addr_copy(hw_addr, vport->info.mac);
3909 *hw_addr_len = ETH_ALEN;
3910 mutex_unlock(&esw->state_lock);
3911 return 0;
3912 }
3913
mlx5_devlink_port_function_hw_addr_set(struct devlink_port * port,const u8 * hw_addr,int hw_addr_len,struct netlink_ext_ack * extack)3914 int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port,
3915 const u8 *hw_addr, int hw_addr_len,
3916 struct netlink_ext_ack *extack)
3917 {
3918 struct mlx5_eswitch *esw;
3919 u16 vport_num;
3920
3921 esw = mlx5_devlink_eswitch_get(port->devlink);
3922 if (IS_ERR(esw)) {
3923 NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr");
3924 return PTR_ERR(esw);
3925 }
3926
3927 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
3928 if (!is_port_function_supported(esw, vport_num)) {
3929 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr");
3930 return -EINVAL;
3931 }
3932
3933 return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr);
3934 }
3935