1 /*
2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
43 #include "rdma.h"
44 #include "en.h"
45 #include "fs_core.h"
46 #include "lib/devcom.h"
47 #include "lib/eq.h"
48 #include "lib/fs_chains.h"
49 #include "en_tc.h"
50 #include "en/mapping.h"
51 #include "devlink.h"
52 #include "lag/lag.h"
53 #include "en/tc/post_meter.h"
54 
55 #define mlx5_esw_for_each_rep(esw, i, rep) \
56 	xa_for_each(&((esw)->offloads.vport_reps), i, rep)
57 
58 /* There are two match-all miss flows, one for unicast dst mac and
59  * one for multicast.
60  */
61 #define MLX5_ESW_MISS_FLOWS (2)
62 #define UPLINK_REP_INDEX 0
63 
64 #define MLX5_ESW_VPORT_TBL_SIZE 128
65 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS  4
66 
67 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
68 
69 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
70 	.max_fte = MLX5_ESW_VPORT_TBL_SIZE,
71 	.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
72 	.flags = 0,
73 };
74 
mlx5_eswitch_get_rep(struct mlx5_eswitch * esw,u16 vport_num)75 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
76 						     u16 vport_num)
77 {
78 	return xa_load(&esw->offloads.vport_reps, vport_num);
79 }
80 
81 static void
mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr)82 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
83 				  struct mlx5_flow_spec *spec,
84 				  struct mlx5_esw_flow_attr *attr)
85 {
86 	if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
87 		return;
88 
89 	if (attr->int_port) {
90 		spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port);
91 
92 		return;
93 	}
94 
95 	spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
96 					 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
97 					 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
98 }
99 
100 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
101  * are not needed as well in the following process. So clear them all for simplicity.
102  */
103 void
mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec)104 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
105 {
106 	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
107 		void *misc2;
108 
109 		misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
110 		MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
111 
112 		misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
113 		MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
114 
115 		if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
116 			spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
117 	}
118 }
119 
120 static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct mlx5_eswitch * src_esw,u16 vport)121 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
122 				  struct mlx5_flow_spec *spec,
123 				  struct mlx5_flow_attr *attr,
124 				  struct mlx5_eswitch *src_esw,
125 				  u16 vport)
126 {
127 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
128 	u32 metadata;
129 	void *misc2;
130 	void *misc;
131 
132 	/* Use metadata matching because vport is not represented by single
133 	 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
134 	 */
135 	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
136 		if (mlx5_esw_indir_table_decap_vport(attr))
137 			vport = mlx5_esw_indir_table_decap_vport(attr);
138 
139 		if (!attr->chain && esw_attr && esw_attr->int_port)
140 			metadata =
141 				mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port);
142 		else
143 			metadata =
144 				mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport);
145 
146 		misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
147 		MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
148 
149 		misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
150 		MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
151 			 mlx5_eswitch_get_vport_metadata_mask());
152 
153 		spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
154 	} else {
155 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
156 		MLX5_SET(fte_match_set_misc, misc, source_port, vport);
157 
158 		if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
159 			MLX5_SET(fte_match_set_misc, misc,
160 				 source_eswitch_owner_vhca_id,
161 				 MLX5_CAP_GEN(src_esw->dev, vhca_id));
162 
163 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
164 		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
165 		if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
166 			MLX5_SET_TO_ONES(fte_match_set_misc, misc,
167 					 source_eswitch_owner_vhca_id);
168 
169 		spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
170 	}
171 }
172 
173 static int
esw_setup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)174 esw_setup_decap_indir(struct mlx5_eswitch *esw,
175 		      struct mlx5_flow_attr *attr)
176 {
177 	struct mlx5_flow_table *ft;
178 
179 	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
180 		return -EOPNOTSUPP;
181 
182 	ft = mlx5_esw_indir_table_get(esw, attr,
183 				      mlx5_esw_indir_table_decap_vport(attr), true);
184 	return PTR_ERR_OR_ZERO(ft);
185 }
186 
187 static void
esw_cleanup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)188 esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
189 			struct mlx5_flow_attr *attr)
190 {
191 	if (mlx5_esw_indir_table_decap_vport(attr))
192 		mlx5_esw_indir_table_put(esw,
193 					 mlx5_esw_indir_table_decap_vport(attr),
194 					 true);
195 }
196 
197 static int
esw_setup_mtu_dest(struct mlx5_flow_destination * dest,struct mlx5e_meter_attr * meter,int i)198 esw_setup_mtu_dest(struct mlx5_flow_destination *dest,
199 		   struct mlx5e_meter_attr *meter,
200 		   int i)
201 {
202 	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE;
203 	dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN;
204 	dest[i].range.min = 0;
205 	dest[i].range.max = meter->params.mtu;
206 	dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter);
207 	dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter);
208 
209 	return 0;
210 }
211 
212 static int
esw_setup_sampler_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,u32 sampler_id,int i)213 esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
214 		       struct mlx5_flow_act *flow_act,
215 		       u32 sampler_id,
216 		       int i)
217 {
218 	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
219 	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
220 	dest[i].sampler_id = sampler_id;
221 
222 	return 0;
223 }
224 
225 static int
esw_setup_ft_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int i)226 esw_setup_ft_dest(struct mlx5_flow_destination *dest,
227 		  struct mlx5_flow_act *flow_act,
228 		  struct mlx5_eswitch *esw,
229 		  struct mlx5_flow_attr *attr,
230 		  int i)
231 {
232 	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
233 	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
234 	dest[i].ft = attr->dest_ft;
235 
236 	if (mlx5_esw_indir_table_decap_vport(attr))
237 		return esw_setup_decap_indir(esw, attr);
238 	return 0;
239 }
240 
241 static void
esw_setup_accept_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,int i)242 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
243 		      struct mlx5_fs_chains *chains, int i)
244 {
245 	if (mlx5_chains_ignore_flow_level_supported(chains))
246 		flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
247 	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
248 	dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
249 }
250 
251 static void
esw_setup_slow_path_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,int i)252 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
253 			 struct mlx5_eswitch *esw, int i)
254 {
255 	if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
256 		flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
257 	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
258 	dest[i].ft = mlx5_eswitch_get_slow_fdb(esw);
259 }
260 
261 static int
esw_setup_chain_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level,int i)262 esw_setup_chain_dest(struct mlx5_flow_destination *dest,
263 		     struct mlx5_flow_act *flow_act,
264 		     struct mlx5_fs_chains *chains,
265 		     u32 chain, u32 prio, u32 level,
266 		     int i)
267 {
268 	struct mlx5_flow_table *ft;
269 
270 	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
271 	ft = mlx5_chains_get_table(chains, chain, prio, level);
272 	if (IS_ERR(ft))
273 		return PTR_ERR(ft);
274 
275 	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
276 	dest[i].ft = ft;
277 	return  0;
278 }
279 
esw_put_dest_tables_loop(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int from,int to)280 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
281 				     int from, int to)
282 {
283 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
284 	struct mlx5_fs_chains *chains = esw_chains(esw);
285 	int i;
286 
287 	for (i = from; i < to; i++)
288 		if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
289 			mlx5_chains_put_table(chains, 0, 1, 0);
290 		else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
291 						     esw_attr->dests[i].mdev))
292 			mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false);
293 }
294 
295 static bool
esw_is_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)296 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
297 {
298 	int i;
299 
300 	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
301 		if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
302 			return true;
303 	return false;
304 }
305 
306 static int
esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains,struct mlx5_flow_attr * attr,int * i)307 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
308 				 struct mlx5_flow_act *flow_act,
309 				 struct mlx5_eswitch *esw,
310 				 struct mlx5_fs_chains *chains,
311 				 struct mlx5_flow_attr *attr,
312 				 int *i)
313 {
314 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
315 	int err;
316 
317 	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
318 		return -EOPNOTSUPP;
319 
320 	/* flow steering cannot handle more than one dest with the same ft
321 	 * in a single flow
322 	 */
323 	if (esw_attr->out_count - esw_attr->split_count > 1)
324 		return -EOPNOTSUPP;
325 
326 	err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
327 	if (err)
328 		return err;
329 
330 	if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
331 		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
332 		flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
333 	}
334 	(*i)++;
335 
336 	return 0;
337 }
338 
esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)339 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
340 					       struct mlx5_flow_attr *attr)
341 {
342 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
343 
344 	esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
345 }
346 
347 static bool
esw_is_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)348 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
349 {
350 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
351 	bool result = false;
352 	int i;
353 
354 	/* Indirect table is supported only for flows with in_port uplink
355 	 * and the destination is vport on the same eswitch as the uplink,
356 	 * return false in case at least one of destinations doesn't meet
357 	 * this criteria.
358 	 */
359 	for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
360 		if (esw_attr->dests[i].vport_valid &&
361 		    mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
362 						esw_attr->dests[i].mdev)) {
363 			result = true;
364 		} else {
365 			result = false;
366 			break;
367 		}
368 	}
369 	return result;
370 }
371 
372 static int
esw_setup_indir_table(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int * i)373 esw_setup_indir_table(struct mlx5_flow_destination *dest,
374 		      struct mlx5_flow_act *flow_act,
375 		      struct mlx5_eswitch *esw,
376 		      struct mlx5_flow_attr *attr,
377 		      int *i)
378 {
379 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
380 	int j, err;
381 
382 	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
383 		return -EOPNOTSUPP;
384 
385 	for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
386 		flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
387 		dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
388 
389 		dest[*i].ft = mlx5_esw_indir_table_get(esw, attr,
390 						       esw_attr->dests[j].vport, false);
391 		if (IS_ERR(dest[*i].ft)) {
392 			err = PTR_ERR(dest[*i].ft);
393 			goto err_indir_tbl_get;
394 		}
395 	}
396 
397 	if (mlx5_esw_indir_table_decap_vport(attr)) {
398 		err = esw_setup_decap_indir(esw, attr);
399 		if (err)
400 			goto err_indir_tbl_get;
401 	}
402 
403 	return 0;
404 
405 err_indir_tbl_get:
406 	esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
407 	return err;
408 }
409 
esw_cleanup_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)410 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
411 {
412 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
413 
414 	esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
415 	esw_cleanup_decap_indir(esw, attr);
416 }
417 
418 static void
esw_cleanup_chain_dest(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)419 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
420 {
421 	mlx5_chains_put_table(chains, chain, prio, level);
422 }
423 
esw_same_vhca_id(struct mlx5_core_dev * mdev1,struct mlx5_core_dev * mdev2)424 static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2)
425 {
426 	return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id);
427 }
428 
esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)429 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw,
430 					      struct mlx5_esw_flow_attr *esw_attr,
431 					      int attr_idx)
432 {
433 	if (esw->offloads.ft_ipsec_tx_pol &&
434 	    esw_attr->dests[attr_idx].vport_valid &&
435 	    esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK &&
436 	    /* To be aligned with software, encryption is needed only for tunnel device */
437 	    (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) &&
438 	    esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport &&
439 	    esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev))
440 		return true;
441 
442 	return false;
443 }
444 
esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)445 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw,
446 					   struct mlx5_esw_flow_attr *esw_attr)
447 {
448 	int i;
449 
450 	if (!esw->offloads.ft_ipsec_tx_pol)
451 		return true;
452 
453 	for (i = 0; i < esw_attr->split_count; i++)
454 		if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i))
455 			return false;
456 
457 	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
458 		if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) &&
459 		    (esw_attr->out_count - esw_attr->split_count > 1))
460 			return false;
461 
462 	return true;
463 }
464 
465 static void
esw_setup_dest_fwd_vport(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)466 esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
467 			 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
468 			 int attr_idx, int dest_idx, bool pkt_reformat)
469 {
470 	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
471 	dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport;
472 	if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
473 		dest[dest_idx].vport.vhca_id =
474 			MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
475 		dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
476 		if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
477 		    mlx5_lag_is_mpesw(esw->dev))
478 			dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
479 	}
480 	if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
481 		if (pkt_reformat) {
482 			flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
483 			flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
484 		}
485 		dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
486 		dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
487 	}
488 }
489 
490 static void
esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)491 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
492 			 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
493 			 int attr_idx, int dest_idx, bool pkt_reformat)
494 {
495 	dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol;
496 	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
497 	if (pkt_reformat &&
498 	    esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
499 		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
500 		flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
501 	}
502 }
503 
504 static void
esw_setup_vport_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)505 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
506 		     struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
507 		     int attr_idx, int dest_idx, bool pkt_reformat)
508 {
509 	if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
510 		esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr,
511 					 attr_idx, dest_idx, pkt_reformat);
512 	else
513 		esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr,
514 					 attr_idx, dest_idx, pkt_reformat);
515 }
516 
517 static int
esw_setup_vport_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int i)518 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
519 		      struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
520 		      int i)
521 {
522 	int j;
523 
524 	for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
525 		esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
526 	return i;
527 }
528 
529 static bool
esw_src_port_rewrite_supported(struct mlx5_eswitch * esw)530 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
531 {
532 	return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
533 	       mlx5_eswitch_vport_match_metadata_enabled(esw) &&
534 	       MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
535 }
536 
537 static bool
esw_dests_to_vf_pf_vports(struct mlx5_flow_destination * dests,int max_dest)538 esw_dests_to_vf_pf_vports(struct mlx5_flow_destination *dests, int max_dest)
539 {
540 	bool vf_dest = false, pf_dest = false;
541 	int i;
542 
543 	for (i = 0; i < max_dest; i++) {
544 		if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT)
545 			continue;
546 
547 		if (dests[i].vport.num == MLX5_VPORT_UPLINK)
548 			pf_dest = true;
549 		else
550 			vf_dest = true;
551 
552 		if (vf_dest && pf_dest)
553 			return true;
554 	}
555 
556 	return false;
557 }
558 
559 static int
esw_setup_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int * i)560 esw_setup_dests(struct mlx5_flow_destination *dest,
561 		struct mlx5_flow_act *flow_act,
562 		struct mlx5_eswitch *esw,
563 		struct mlx5_flow_attr *attr,
564 		struct mlx5_flow_spec *spec,
565 		int *i)
566 {
567 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
568 	struct mlx5_fs_chains *chains = esw_chains(esw);
569 	int err = 0;
570 
571 	if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
572 	    esw_src_port_rewrite_supported(esw))
573 		attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
574 
575 	if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
576 		esw_setup_slow_path_dest(dest, flow_act, esw, *i);
577 		(*i)++;
578 		goto out;
579 	}
580 
581 	if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) {
582 		esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
583 		(*i)++;
584 	} else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
585 		esw_setup_accept_dest(dest, flow_act, chains, *i);
586 		(*i)++;
587 	} else if (attr->flags & MLX5_ATTR_FLAG_MTU) {
588 		err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i);
589 		(*i)++;
590 	} else if (esw_is_indir_table(esw, attr)) {
591 		err = esw_setup_indir_table(dest, flow_act, esw, attr, i);
592 	} else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
593 		err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
594 	} else {
595 		*i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
596 
597 		if (attr->dest_ft) {
598 			err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i);
599 			(*i)++;
600 		} else if (attr->dest_chain) {
601 			err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
602 						   1, 0, *i);
603 			(*i)++;
604 		}
605 	}
606 
607 out:
608 	return err;
609 }
610 
611 static void
esw_cleanup_dests(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)612 esw_cleanup_dests(struct mlx5_eswitch *esw,
613 		  struct mlx5_flow_attr *attr)
614 {
615 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
616 	struct mlx5_fs_chains *chains = esw_chains(esw);
617 
618 	if (attr->dest_ft) {
619 		esw_cleanup_decap_indir(esw, attr);
620 	} else if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
621 		if (attr->dest_chain)
622 			esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
623 		else if (esw_is_indir_table(esw, attr))
624 			esw_cleanup_indir_table(esw, attr);
625 		else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
626 			esw_cleanup_chain_src_port_rewrite(esw, attr);
627 	}
628 }
629 
630 static void
esw_setup_meter(struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act)631 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
632 {
633 	struct mlx5e_flow_meter_handle *meter;
634 
635 	meter = attr->meter_attr.meter;
636 	flow_act->exe_aso.type = attr->exe_aso_type;
637 	flow_act->exe_aso.object_id = meter->obj_id;
638 	flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
639 	flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
640 	/* use metadata reg 5 for packet color */
641 	flow_act->exe_aso.return_reg_id = 5;
642 }
643 
644 struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)645 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
646 				struct mlx5_flow_spec *spec,
647 				struct mlx5_flow_attr *attr)
648 {
649 	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
650 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
651 	struct mlx5_fs_chains *chains = esw_chains(esw);
652 	bool split = !!(esw_attr->split_count);
653 	struct mlx5_vport_tbl_attr fwd_attr;
654 	struct mlx5_flow_destination *dest;
655 	struct mlx5_flow_handle *rule;
656 	struct mlx5_flow_table *fdb;
657 	int i = 0;
658 
659 	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
660 		return ERR_PTR(-EOPNOTSUPP);
661 
662 	if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
663 		return ERR_PTR(-EOPNOTSUPP);
664 
665 	if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr))
666 		return ERR_PTR(-EOPNOTSUPP);
667 
668 	dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
669 	if (!dest)
670 		return ERR_PTR(-ENOMEM);
671 
672 	flow_act.action = attr->action;
673 
674 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
675 		flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
676 		flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
677 		flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
678 		if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
679 			flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
680 			flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
681 			flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
682 		}
683 	}
684 
685 	mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
686 
687 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
688 		int err;
689 
690 		err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
691 		if (err) {
692 			rule = ERR_PTR(err);
693 			goto err_create_goto_table;
694 		}
695 
696 		/* Header rewrite with combined wire+loopback in FDB is not allowed */
697 		if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) &&
698 		    esw_dests_to_vf_pf_vports(dest, i)) {
699 			esw_warn(esw->dev,
700 				 "FDB: Header rewrite with forwarding to both PF and VF is not allowed\n");
701 			rule = ERR_PTR(-EINVAL);
702 			goto err_esw_get;
703 		}
704 	}
705 
706 	if (esw_attr->decap_pkt_reformat)
707 		flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
708 
709 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
710 		dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
711 		dest[i].counter_id = mlx5_fc_id(attr->counter);
712 		i++;
713 	}
714 
715 	if (attr->outer_match_level != MLX5_MATCH_NONE)
716 		spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
717 	if (attr->inner_match_level != MLX5_MATCH_NONE)
718 		spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
719 
720 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
721 		flow_act.modify_hdr = attr->modify_hdr;
722 
723 	if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
724 	    attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
725 		esw_setup_meter(attr, &flow_act);
726 
727 	if (split) {
728 		fwd_attr.chain = attr->chain;
729 		fwd_attr.prio = attr->prio;
730 		fwd_attr.vport = esw_attr->in_rep->vport;
731 		fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
732 
733 		fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
734 	} else {
735 		if (attr->chain || attr->prio)
736 			fdb = mlx5_chains_get_table(chains, attr->chain,
737 						    attr->prio, 0);
738 		else
739 			fdb = attr->ft;
740 
741 		if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
742 			mlx5_eswitch_set_rule_source_port(esw, spec, attr,
743 							  esw_attr->in_mdev->priv.eswitch,
744 							  esw_attr->in_rep->vport);
745 	}
746 	if (IS_ERR(fdb)) {
747 		rule = ERR_CAST(fdb);
748 		goto err_esw_get;
749 	}
750 
751 	if (!i) {
752 		kfree(dest);
753 		dest = NULL;
754 	}
755 
756 	if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
757 		rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
758 						     &flow_act, dest, i);
759 	else
760 		rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
761 	if (IS_ERR(rule))
762 		goto err_add_rule;
763 	else
764 		atomic64_inc(&esw->offloads.num_flows);
765 
766 	kfree(dest);
767 	return rule;
768 
769 err_add_rule:
770 	if (split)
771 		mlx5_esw_vporttbl_put(esw, &fwd_attr);
772 	else if (attr->chain || attr->prio)
773 		mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
774 err_esw_get:
775 	esw_cleanup_dests(esw, attr);
776 err_create_goto_table:
777 	kfree(dest);
778 	return rule;
779 }
780 
781 struct mlx5_flow_handle *
mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)782 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
783 			  struct mlx5_flow_spec *spec,
784 			  struct mlx5_flow_attr *attr)
785 {
786 	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
787 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
788 	struct mlx5_fs_chains *chains = esw_chains(esw);
789 	struct mlx5_vport_tbl_attr fwd_attr;
790 	struct mlx5_flow_destination *dest;
791 	struct mlx5_flow_table *fast_fdb;
792 	struct mlx5_flow_table *fwd_fdb;
793 	struct mlx5_flow_handle *rule;
794 	int i, err = 0;
795 
796 	dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
797 	if (!dest)
798 		return ERR_PTR(-ENOMEM);
799 
800 	fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
801 	if (IS_ERR(fast_fdb)) {
802 		rule = ERR_CAST(fast_fdb);
803 		goto err_get_fast;
804 	}
805 
806 	fwd_attr.chain = attr->chain;
807 	fwd_attr.prio = attr->prio;
808 	fwd_attr.vport = esw_attr->in_rep->vport;
809 	fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
810 	fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
811 	if (IS_ERR(fwd_fdb)) {
812 		rule = ERR_CAST(fwd_fdb);
813 		goto err_get_fwd;
814 	}
815 
816 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
817 	for (i = 0; i < esw_attr->split_count; i++) {
818 		if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
819 			/* Source port rewrite (forward to ovs internal port or statck device) isn't
820 			 * supported in the rule of split action.
821 			 */
822 			err = -EOPNOTSUPP;
823 		else
824 			esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
825 
826 		if (err) {
827 			rule = ERR_PTR(err);
828 			goto err_chain_src_rewrite;
829 		}
830 	}
831 	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
832 	dest[i].ft = fwd_fdb;
833 	i++;
834 
835 	mlx5_eswitch_set_rule_source_port(esw, spec, attr,
836 					  esw_attr->in_mdev->priv.eswitch,
837 					  esw_attr->in_rep->vport);
838 
839 	if (attr->outer_match_level != MLX5_MATCH_NONE)
840 		spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
841 
842 	flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
843 	rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
844 
845 	if (IS_ERR(rule)) {
846 		i = esw_attr->split_count;
847 		goto err_chain_src_rewrite;
848 	}
849 
850 	atomic64_inc(&esw->offloads.num_flows);
851 
852 	kfree(dest);
853 	return rule;
854 err_chain_src_rewrite:
855 	mlx5_esw_vporttbl_put(esw, &fwd_attr);
856 err_get_fwd:
857 	mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
858 err_get_fast:
859 	kfree(dest);
860 	return rule;
861 }
862 
863 static void
__mlx5_eswitch_del_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr,bool fwd_rule)864 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
865 			struct mlx5_flow_handle *rule,
866 			struct mlx5_flow_attr *attr,
867 			bool fwd_rule)
868 {
869 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
870 	struct mlx5_fs_chains *chains = esw_chains(esw);
871 	bool split = (esw_attr->split_count > 0);
872 	struct mlx5_vport_tbl_attr fwd_attr;
873 	int i;
874 
875 	mlx5_del_flow_rules(rule);
876 
877 	if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
878 		/* unref the term table */
879 		for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
880 			if (esw_attr->dests[i].termtbl)
881 				mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
882 		}
883 	}
884 
885 	atomic64_dec(&esw->offloads.num_flows);
886 
887 	if (fwd_rule || split) {
888 		fwd_attr.chain = attr->chain;
889 		fwd_attr.prio = attr->prio;
890 		fwd_attr.vport = esw_attr->in_rep->vport;
891 		fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
892 	}
893 
894 	if (fwd_rule)  {
895 		mlx5_esw_vporttbl_put(esw, &fwd_attr);
896 		mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
897 	} else {
898 		if (split)
899 			mlx5_esw_vporttbl_put(esw, &fwd_attr);
900 		else if (attr->chain || attr->prio)
901 			mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
902 		esw_cleanup_dests(esw, attr);
903 	}
904 }
905 
906 void
mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)907 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
908 				struct mlx5_flow_handle *rule,
909 				struct mlx5_flow_attr *attr)
910 {
911 	__mlx5_eswitch_del_rule(esw, rule, attr, false);
912 }
913 
914 void
mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)915 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
916 			  struct mlx5_flow_handle *rule,
917 			  struct mlx5_flow_attr *attr)
918 {
919 	__mlx5_eswitch_del_rule(esw, rule, attr, true);
920 }
921 
922 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch * on_esw,struct mlx5_eswitch * from_esw,struct mlx5_eswitch_rep * rep,u32 sqn)923 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
924 				    struct mlx5_eswitch *from_esw,
925 				    struct mlx5_eswitch_rep *rep,
926 				    u32 sqn)
927 {
928 	struct mlx5_flow_act flow_act = {0};
929 	struct mlx5_flow_destination dest = {};
930 	struct mlx5_flow_handle *flow_rule;
931 	struct mlx5_flow_spec *spec;
932 	void *misc;
933 	u16 vport;
934 
935 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
936 	if (!spec) {
937 		flow_rule = ERR_PTR(-ENOMEM);
938 		goto out;
939 	}
940 
941 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
942 	MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
943 
944 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
945 	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
946 
947 	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
948 
949 	/* source vport is the esw manager */
950 	vport = from_esw->manager_vport;
951 
952 	if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) {
953 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
954 		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
955 			 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport));
956 
957 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
958 		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
959 			 mlx5_eswitch_get_vport_metadata_mask());
960 
961 		spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
962 	} else {
963 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
964 		MLX5_SET(fte_match_set_misc, misc, source_port, vport);
965 
966 		if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
967 			MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
968 				 MLX5_CAP_GEN(from_esw->dev, vhca_id));
969 
970 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
971 		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
972 
973 		if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
974 			MLX5_SET_TO_ONES(fte_match_set_misc, misc,
975 					 source_eswitch_owner_vhca_id);
976 
977 		spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
978 	}
979 
980 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
981 	dest.vport.num = rep->vport;
982 	dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
983 	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
984 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
985 
986 	if (rep->vport == MLX5_VPORT_UPLINK &&
987 	    on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) {
988 		dest.ft = on_esw->offloads.ft_ipsec_tx_pol;
989 		flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL;
990 		dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
991 	} else {
992 		dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
993 		dest.vport.num = rep->vport;
994 		dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
995 		dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
996 	}
997 
998 	if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) &&
999 	    rep->vport == MLX5_VPORT_UPLINK)
1000 		spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1001 
1002 	flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw),
1003 					spec, &flow_act, &dest, 1);
1004 	if (IS_ERR(flow_rule))
1005 		esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
1006 			 PTR_ERR(flow_rule));
1007 out:
1008 	kvfree(spec);
1009 	return flow_rule;
1010 }
1011 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1012 
mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle * rule)1013 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1014 {
1015 	mlx5_del_flow_rules(rule);
1016 }
1017 
mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle * rule)1018 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1019 {
1020 	if (rule)
1021 		mlx5_del_flow_rules(rule);
1022 }
1023 
1024 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch * esw,u16 vport_num)1025 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1026 {
1027 	struct mlx5_flow_destination dest = {};
1028 	struct mlx5_flow_act flow_act = {0};
1029 	struct mlx5_flow_handle *flow_rule;
1030 	struct mlx5_flow_spec *spec;
1031 
1032 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1033 	if (!spec)
1034 		return ERR_PTR(-ENOMEM);
1035 
1036 	MLX5_SET(fte_match_param, spec->match_criteria,
1037 		 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1038 	MLX5_SET(fte_match_param, spec->match_criteria,
1039 		 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1040 	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1041 		 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1042 
1043 	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1044 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1045 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1046 
1047 	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1048 		 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1049 	dest.vport.num = vport_num;
1050 
1051 	flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1052 					spec, &flow_act, &dest, 1);
1053 	if (IS_ERR(flow_rule))
1054 		esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
1055 			 vport_num, PTR_ERR(flow_rule));
1056 
1057 	kvfree(spec);
1058 	return flow_rule;
1059 }
1060 
mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch * esw)1061 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1062 {
1063 	return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1064 	       MLX5_FDB_TO_VPORT_REG_C_1;
1065 }
1066 
esw_set_passing_vport_metadata(struct mlx5_eswitch * esw,bool enable)1067 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1068 {
1069 	u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1070 	u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1071 	u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1072 	u8 curr, wanted;
1073 	int err;
1074 
1075 	if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1076 	    !mlx5_eswitch_vport_match_metadata_enabled(esw))
1077 		return 0;
1078 
1079 	MLX5_SET(query_esw_vport_context_in, in, opcode,
1080 		 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1081 	err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1082 	if (err)
1083 		return err;
1084 
1085 	curr = MLX5_GET(query_esw_vport_context_out, out,
1086 			esw_vport_context.fdb_to_vport_reg_c_id);
1087 	wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1088 	if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1089 		wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1090 
1091 	if (enable)
1092 		curr |= wanted;
1093 	else
1094 		curr &= ~wanted;
1095 
1096 	MLX5_SET(modify_esw_vport_context_in, min,
1097 		 esw_vport_context.fdb_to_vport_reg_c_id, curr);
1098 	MLX5_SET(modify_esw_vport_context_in, min,
1099 		 field_select.fdb_to_vport_reg_c_id, 1);
1100 
1101 	err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
1102 	if (!err) {
1103 		if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1104 			esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1105 		else
1106 			esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1107 	}
1108 
1109 	return err;
1110 }
1111 
peer_miss_rules_setup(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev,struct mlx5_flow_spec * spec,struct mlx5_flow_destination * dest)1112 static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1113 				  struct mlx5_core_dev *peer_dev,
1114 				  struct mlx5_flow_spec *spec,
1115 				  struct mlx5_flow_destination *dest)
1116 {
1117 	void *misc;
1118 
1119 	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1120 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1121 				    misc_parameters_2);
1122 		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1123 			 mlx5_eswitch_get_vport_metadata_mask());
1124 
1125 		spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1126 	} else {
1127 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1128 				    misc_parameters);
1129 
1130 		MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1131 			 MLX5_CAP_GEN(peer_dev, vhca_id));
1132 
1133 		spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1134 
1135 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1136 				    misc_parameters);
1137 		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1138 		MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1139 				 source_eswitch_owner_vhca_id);
1140 	}
1141 
1142 	dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1143 	dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1144 	dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1145 	dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1146 }
1147 
esw_set_peer_miss_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,struct mlx5_flow_spec * spec,u16 vport)1148 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1149 					       struct mlx5_eswitch *peer_esw,
1150 					       struct mlx5_flow_spec *spec,
1151 					       u16 vport)
1152 {
1153 	void *misc;
1154 
1155 	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1156 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1157 				    misc_parameters_2);
1158 		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1159 			 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1160 								   vport));
1161 	} else {
1162 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1163 				    misc_parameters);
1164 		MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1165 	}
1166 }
1167 
esw_add_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1168 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1169 				       struct mlx5_core_dev *peer_dev)
1170 {
1171 	struct mlx5_flow_destination dest = {};
1172 	struct mlx5_flow_act flow_act = {0};
1173 	struct mlx5_flow_handle **flows;
1174 	/* total vports is the same for both e-switches */
1175 	int nvports = esw->total_vports;
1176 	struct mlx5_flow_handle *flow;
1177 	struct mlx5_flow_spec *spec;
1178 	struct mlx5_vport *vport;
1179 	int err, pfindex;
1180 	unsigned long i;
1181 	void *misc;
1182 
1183 	if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(esw->dev))
1184 		return 0;
1185 
1186 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1187 	if (!spec)
1188 		return -ENOMEM;
1189 
1190 	peer_miss_rules_setup(esw, peer_dev, spec, &dest);
1191 
1192 	flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL);
1193 	if (!flows) {
1194 		err = -ENOMEM;
1195 		goto alloc_flows_err;
1196 	}
1197 
1198 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1199 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1200 			    misc_parameters);
1201 
1202 	if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1203 		vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1204 		esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1205 						   spec, MLX5_VPORT_PF);
1206 
1207 		flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1208 					   spec, &flow_act, &dest, 1);
1209 		if (IS_ERR(flow)) {
1210 			err = PTR_ERR(flow);
1211 			goto add_pf_flow_err;
1212 		}
1213 		flows[vport->index] = flow;
1214 	}
1215 
1216 	if (mlx5_ecpf_vport_exists(esw->dev)) {
1217 		vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1218 		MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1219 		flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1220 					   spec, &flow_act, &dest, 1);
1221 		if (IS_ERR(flow)) {
1222 			err = PTR_ERR(flow);
1223 			goto add_ecpf_flow_err;
1224 		}
1225 		flows[vport->index] = flow;
1226 	}
1227 
1228 	mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1229 		esw_set_peer_miss_rule_source_port(esw,
1230 						   peer_dev->priv.eswitch,
1231 						   spec, vport->vport);
1232 
1233 		flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1234 					   spec, &flow_act, &dest, 1);
1235 		if (IS_ERR(flow)) {
1236 			err = PTR_ERR(flow);
1237 			goto add_vf_flow_err;
1238 		}
1239 		flows[vport->index] = flow;
1240 	}
1241 
1242 	if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1243 		mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1244 			if (i >= mlx5_core_max_ec_vfs(peer_dev))
1245 				break;
1246 			esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1247 							   spec, vport->vport);
1248 			flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1249 						   spec, &flow_act, &dest, 1);
1250 			if (IS_ERR(flow)) {
1251 				err = PTR_ERR(flow);
1252 				goto add_ec_vf_flow_err;
1253 			}
1254 			flows[vport->index] = flow;
1255 		}
1256 	}
1257 
1258 	pfindex = mlx5_get_dev_index(peer_dev);
1259 	if (pfindex >= MLX5_MAX_PORTS) {
1260 		esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n",
1261 			 pfindex, MLX5_MAX_PORTS);
1262 		err = -EINVAL;
1263 		goto add_ec_vf_flow_err;
1264 	}
1265 	esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows;
1266 
1267 	kvfree(spec);
1268 	return 0;
1269 
1270 add_ec_vf_flow_err:
1271 	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1272 		if (!flows[vport->index])
1273 			continue;
1274 		mlx5_del_flow_rules(flows[vport->index]);
1275 	}
1276 add_vf_flow_err:
1277 	mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1278 		if (!flows[vport->index])
1279 			continue;
1280 		mlx5_del_flow_rules(flows[vport->index]);
1281 	}
1282 	if (mlx5_ecpf_vport_exists(esw->dev)) {
1283 		vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1284 		mlx5_del_flow_rules(flows[vport->index]);
1285 	}
1286 add_ecpf_flow_err:
1287 	if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1288 		vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1289 		mlx5_del_flow_rules(flows[vport->index]);
1290 	}
1291 add_pf_flow_err:
1292 	esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1293 	kvfree(flows);
1294 alloc_flows_err:
1295 	kvfree(spec);
1296 	return err;
1297 }
1298 
esw_del_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1299 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1300 					struct mlx5_core_dev *peer_dev)
1301 {
1302 	u16 peer_index = mlx5_get_dev_index(peer_dev);
1303 	struct mlx5_flow_handle **flows;
1304 	struct mlx5_vport *vport;
1305 	unsigned long i;
1306 
1307 	flows = esw->fdb_table.offloads.peer_miss_rules[peer_index];
1308 	if (!flows)
1309 		return;
1310 
1311 	if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1312 		mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1313 			/* The flow for a particular vport could be NULL if the other ECPF
1314 			 * has fewer or no VFs enabled
1315 			 */
1316 			if (!flows[vport->index])
1317 				continue;
1318 			mlx5_del_flow_rules(flows[vport->index]);
1319 		}
1320 	}
1321 
1322 	mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1323 		mlx5_del_flow_rules(flows[vport->index]);
1324 
1325 	if (mlx5_ecpf_vport_exists(esw->dev)) {
1326 		vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1327 		mlx5_del_flow_rules(flows[vport->index]);
1328 	}
1329 
1330 	if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1331 		vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1332 		mlx5_del_flow_rules(flows[vport->index]);
1333 	}
1334 
1335 	kvfree(flows);
1336 	esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL;
1337 }
1338 
esw_add_fdb_miss_rule(struct mlx5_eswitch * esw)1339 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1340 {
1341 	struct mlx5_flow_act flow_act = {0};
1342 	struct mlx5_flow_destination dest = {};
1343 	struct mlx5_flow_handle *flow_rule = NULL;
1344 	struct mlx5_flow_spec *spec;
1345 	void *headers_c;
1346 	void *headers_v;
1347 	int err = 0;
1348 	u8 *dmac_c;
1349 	u8 *dmac_v;
1350 
1351 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1352 	if (!spec) {
1353 		err = -ENOMEM;
1354 		goto out;
1355 	}
1356 
1357 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1358 	headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1359 				 outer_headers);
1360 	dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1361 			      outer_headers.dmac_47_16);
1362 	dmac_c[0] = 0x01;
1363 
1364 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1365 	dest.vport.num = esw->manager_vport;
1366 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1367 
1368 	flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1369 					spec, &flow_act, &dest, 1);
1370 	if (IS_ERR(flow_rule)) {
1371 		err = PTR_ERR(flow_rule);
1372 		esw_warn(esw->dev,  "FDB: Failed to add unicast miss flow rule err %d\n", err);
1373 		goto out;
1374 	}
1375 
1376 	esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1377 
1378 	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1379 				 outer_headers);
1380 	dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1381 			      outer_headers.dmac_47_16);
1382 	dmac_v[0] = 0x01;
1383 	flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1384 					spec, &flow_act, &dest, 1);
1385 	if (IS_ERR(flow_rule)) {
1386 		err = PTR_ERR(flow_rule);
1387 		esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1388 		mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1389 		goto out;
1390 	}
1391 
1392 	esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1393 
1394 out:
1395 	kvfree(spec);
1396 	return err;
1397 }
1398 
1399 struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch * esw,u32 tag)1400 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1401 {
1402 	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1403 	struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1404 	struct mlx5_flow_context *flow_context;
1405 	struct mlx5_flow_handle *flow_rule;
1406 	struct mlx5_flow_destination dest;
1407 	struct mlx5_flow_spec *spec;
1408 	void *misc;
1409 
1410 	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1411 		return ERR_PTR(-EOPNOTSUPP);
1412 
1413 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1414 	if (!spec)
1415 		return ERR_PTR(-ENOMEM);
1416 
1417 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1418 			    misc_parameters_2);
1419 	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1420 		 ESW_REG_C0_USER_DATA_METADATA_MASK);
1421 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1422 			    misc_parameters_2);
1423 	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1424 	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1425 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1426 			  MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1427 	flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1428 
1429 	flow_context = &spec->flow_context;
1430 	flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1431 	flow_context->flow_tag = tag;
1432 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1433 	dest.ft = esw->offloads.ft_offloads;
1434 
1435 	flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1436 	kvfree(spec);
1437 
1438 	if (IS_ERR(flow_rule))
1439 		esw_warn(esw->dev,
1440 			 "Failed to create restore rule for tag: %d, err(%d)\n",
1441 			 tag, (int)PTR_ERR(flow_rule));
1442 
1443 	return flow_rule;
1444 }
1445 
1446 #define MAX_PF_SQ 256
1447 #define MAX_SQ_NVPORTS 32
1448 
1449 void
mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch * esw,u32 * flow_group_in,int match_params)1450 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1451 				    u32 *flow_group_in,
1452 				    int match_params)
1453 {
1454 	void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1455 					    flow_group_in,
1456 					    match_criteria);
1457 
1458 	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1459 		MLX5_SET(create_flow_group_in, flow_group_in,
1460 			 match_criteria_enable,
1461 			 MLX5_MATCH_MISC_PARAMETERS_2 | match_params);
1462 
1463 		MLX5_SET(fte_match_param, match_criteria,
1464 			 misc_parameters_2.metadata_reg_c_0,
1465 			 mlx5_eswitch_get_vport_metadata_mask());
1466 	} else {
1467 		MLX5_SET(create_flow_group_in, flow_group_in,
1468 			 match_criteria_enable,
1469 			 MLX5_MATCH_MISC_PARAMETERS | match_params);
1470 
1471 		MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1472 				 misc_parameters.source_port);
1473 	}
1474 }
1475 
1476 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
esw_vport_tbl_put(struct mlx5_eswitch * esw)1477 static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1478 {
1479 	struct mlx5_vport_tbl_attr attr;
1480 	struct mlx5_vport *vport;
1481 	unsigned long i;
1482 
1483 	attr.chain = 0;
1484 	attr.prio = 1;
1485 	mlx5_esw_for_each_vport(esw, i, vport) {
1486 		attr.vport = vport->vport;
1487 		attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1488 		mlx5_esw_vporttbl_put(esw, &attr);
1489 	}
1490 }
1491 
esw_vport_tbl_get(struct mlx5_eswitch * esw)1492 static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1493 {
1494 	struct mlx5_vport_tbl_attr attr;
1495 	struct mlx5_flow_table *fdb;
1496 	struct mlx5_vport *vport;
1497 	unsigned long i;
1498 
1499 	attr.chain = 0;
1500 	attr.prio = 1;
1501 	mlx5_esw_for_each_vport(esw, i, vport) {
1502 		attr.vport = vport->vport;
1503 		attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1504 		fdb = mlx5_esw_vporttbl_get(esw, &attr);
1505 		if (IS_ERR(fdb))
1506 			goto out;
1507 	}
1508 	return 0;
1509 
1510 out:
1511 	esw_vport_tbl_put(esw);
1512 	return PTR_ERR(fdb);
1513 }
1514 
1515 #define fdb_modify_header_fwd_to_table_supported(esw) \
1516 	(MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
esw_init_chains_offload_flags(struct mlx5_eswitch * esw,u32 * flags)1517 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1518 {
1519 	struct mlx5_core_dev *dev = esw->dev;
1520 
1521 	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1522 		*flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1523 
1524 	if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1525 	    esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1526 		*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1527 		esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1528 	} else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1529 		*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1530 		esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1531 	} else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1532 		/* Disabled when ttl workaround is needed, e.g
1533 		 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1534 		 */
1535 		esw_warn(dev,
1536 			 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1537 		*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1538 	} else {
1539 		*flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1540 		esw_info(dev, "Supported tc chains and prios offload\n");
1541 	}
1542 
1543 	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1544 		*flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1545 }
1546 
1547 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1548 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1549 {
1550 	struct mlx5_core_dev *dev = esw->dev;
1551 	struct mlx5_flow_table *nf_ft, *ft;
1552 	struct mlx5_chains_attr attr = {};
1553 	struct mlx5_fs_chains *chains;
1554 	int err;
1555 
1556 	esw_init_chains_offload_flags(esw, &attr.flags);
1557 	attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1558 	attr.max_grp_num = esw->params.large_group_num;
1559 	attr.default_ft = miss_fdb;
1560 	attr.mapping = esw->offloads.reg_c0_obj_pool;
1561 
1562 	chains = mlx5_chains_create(dev, &attr);
1563 	if (IS_ERR(chains)) {
1564 		err = PTR_ERR(chains);
1565 		esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1566 		return err;
1567 	}
1568 	mlx5_chains_print_info(chains);
1569 
1570 	esw->fdb_table.offloads.esw_chains_priv = chains;
1571 
1572 	/* Create tc_end_ft which is the always created ft chain */
1573 	nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1574 				      1, 0);
1575 	if (IS_ERR(nf_ft)) {
1576 		err = PTR_ERR(nf_ft);
1577 		goto nf_ft_err;
1578 	}
1579 
1580 	/* Always open the root for fast path */
1581 	ft = mlx5_chains_get_table(chains, 0, 1, 0);
1582 	if (IS_ERR(ft)) {
1583 		err = PTR_ERR(ft);
1584 		goto level_0_err;
1585 	}
1586 
1587 	/* Open level 1 for split fdb rules now if prios isn't supported  */
1588 	if (!mlx5_chains_prios_supported(chains)) {
1589 		err = esw_vport_tbl_get(esw);
1590 		if (err)
1591 			goto level_1_err;
1592 	}
1593 
1594 	mlx5_chains_set_end_ft(chains, nf_ft);
1595 
1596 	return 0;
1597 
1598 level_1_err:
1599 	mlx5_chains_put_table(chains, 0, 1, 0);
1600 level_0_err:
1601 	mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1602 nf_ft_err:
1603 	mlx5_chains_destroy(chains);
1604 	esw->fdb_table.offloads.esw_chains_priv = NULL;
1605 
1606 	return err;
1607 }
1608 
1609 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1610 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1611 {
1612 	if (!mlx5_chains_prios_supported(chains))
1613 		esw_vport_tbl_put(esw);
1614 	mlx5_chains_put_table(chains, 0, 1, 0);
1615 	mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1616 	mlx5_chains_destroy(chains);
1617 }
1618 
1619 #else /* CONFIG_MLX5_CLS_ACT */
1620 
1621 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1622 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1623 { return 0; }
1624 
1625 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1626 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1627 {}
1628 
1629 #endif
1630 
1631 static int
esw_create_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1632 esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1633 			       struct mlx5_flow_table *fdb,
1634 			       u32 *flow_group_in,
1635 			       int *ix)
1636 {
1637 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1638 	struct mlx5_flow_group *g;
1639 	void *match_criteria;
1640 	int count, err = 0;
1641 
1642 	memset(flow_group_in, 0, inlen);
1643 
1644 	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS);
1645 
1646 	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1647 	MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1648 
1649 	if (!mlx5_eswitch_vport_match_metadata_enabled(esw) &&
1650 	    MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1651 		MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1652 				 misc_parameters.source_eswitch_owner_vhca_id);
1653 		MLX5_SET(create_flow_group_in, flow_group_in,
1654 			 source_eswitch_owner_vhca_id_valid, 1);
1655 	}
1656 
1657 	/* See comment at table_size calculation */
1658 	count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1659 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1660 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1661 	*ix += count;
1662 
1663 	g = mlx5_create_flow_group(fdb, flow_group_in);
1664 	if (IS_ERR(g)) {
1665 		err = PTR_ERR(g);
1666 		esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1667 		goto out;
1668 	}
1669 	esw->fdb_table.offloads.send_to_vport_grp = g;
1670 
1671 out:
1672 	return err;
1673 }
1674 
1675 static int
esw_create_meta_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1676 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1677 				    struct mlx5_flow_table *fdb,
1678 				    u32 *flow_group_in,
1679 				    int *ix)
1680 {
1681 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1682 	struct mlx5_flow_group *g;
1683 	void *match_criteria;
1684 	int err = 0;
1685 
1686 	if (!esw_src_port_rewrite_supported(esw))
1687 		return 0;
1688 
1689 	memset(flow_group_in, 0, inlen);
1690 
1691 	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1692 		 MLX5_MATCH_MISC_PARAMETERS_2);
1693 
1694 	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1695 
1696 	MLX5_SET(fte_match_param, match_criteria,
1697 		 misc_parameters_2.metadata_reg_c_0,
1698 		 mlx5_eswitch_get_vport_metadata_mask());
1699 	MLX5_SET(fte_match_param, match_criteria,
1700 		 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1701 
1702 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1703 	MLX5_SET(create_flow_group_in, flow_group_in,
1704 		 end_flow_index, *ix + esw->total_vports - 1);
1705 	*ix += esw->total_vports;
1706 
1707 	g = mlx5_create_flow_group(fdb, flow_group_in);
1708 	if (IS_ERR(g)) {
1709 		err = PTR_ERR(g);
1710 		esw_warn(esw->dev,
1711 			 "Failed to create send-to-vport meta flow group err(%d)\n", err);
1712 		goto send_vport_meta_err;
1713 	}
1714 	esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1715 
1716 	return 0;
1717 
1718 send_vport_meta_err:
1719 	return err;
1720 }
1721 
1722 static int
esw_create_peer_esw_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1723 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1724 			       struct mlx5_flow_table *fdb,
1725 			       u32 *flow_group_in,
1726 			       int *ix)
1727 {
1728 	int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1);
1729 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1730 	struct mlx5_flow_group *g;
1731 	void *match_criteria;
1732 	int err = 0;
1733 
1734 	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1735 		return 0;
1736 
1737 	memset(flow_group_in, 0, inlen);
1738 
1739 	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
1740 
1741 	if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1742 		match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1743 					      flow_group_in,
1744 					      match_criteria);
1745 
1746 		MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1747 				 misc_parameters.source_eswitch_owner_vhca_id);
1748 
1749 		MLX5_SET(create_flow_group_in, flow_group_in,
1750 			 source_eswitch_owner_vhca_id_valid, 1);
1751 	}
1752 
1753 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1754 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1755 		 *ix + max_peer_ports);
1756 	*ix += max_peer_ports + 1;
1757 
1758 	g = mlx5_create_flow_group(fdb, flow_group_in);
1759 	if (IS_ERR(g)) {
1760 		err = PTR_ERR(g);
1761 		esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1762 		goto out;
1763 	}
1764 	esw->fdb_table.offloads.peer_miss_grp = g;
1765 
1766 out:
1767 	return err;
1768 }
1769 
1770 static int
esw_create_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1771 esw_create_miss_group(struct mlx5_eswitch *esw,
1772 		      struct mlx5_flow_table *fdb,
1773 		      u32 *flow_group_in,
1774 		      int *ix)
1775 {
1776 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1777 	struct mlx5_flow_group *g;
1778 	void *match_criteria;
1779 	int err = 0;
1780 	u8 *dmac;
1781 
1782 	memset(flow_group_in, 0, inlen);
1783 
1784 	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1785 		 MLX5_MATCH_OUTER_HEADERS);
1786 	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1787 				      match_criteria);
1788 	dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1789 			    outer_headers.dmac_47_16);
1790 	dmac[0] = 0x01;
1791 
1792 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1793 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1794 		 *ix + MLX5_ESW_MISS_FLOWS);
1795 
1796 	g = mlx5_create_flow_group(fdb, flow_group_in);
1797 	if (IS_ERR(g)) {
1798 		err = PTR_ERR(g);
1799 		esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1800 		goto miss_err;
1801 	}
1802 	esw->fdb_table.offloads.miss_grp = g;
1803 
1804 	err = esw_add_fdb_miss_rule(esw);
1805 	if (err)
1806 		goto miss_rule_err;
1807 
1808 	return 0;
1809 
1810 miss_rule_err:
1811 	mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1812 miss_err:
1813 	return err;
1814 }
1815 
esw_create_offloads_fdb_tables(struct mlx5_eswitch * esw)1816 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1817 {
1818 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1819 	struct mlx5_flow_table_attr ft_attr = {};
1820 	struct mlx5_core_dev *dev = esw->dev;
1821 	struct mlx5_flow_namespace *root_ns;
1822 	struct mlx5_flow_table *fdb = NULL;
1823 	int table_size, ix = 0, err = 0;
1824 	u32 flags = 0, *flow_group_in;
1825 
1826 	esw_debug(esw->dev, "Create offloads FDB Tables\n");
1827 
1828 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1829 	if (!flow_group_in)
1830 		return -ENOMEM;
1831 
1832 	root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1833 	if (!root_ns) {
1834 		esw_warn(dev, "Failed to get FDB flow namespace\n");
1835 		err = -EOPNOTSUPP;
1836 		goto ns_err;
1837 	}
1838 	esw->fdb_table.offloads.ns = root_ns;
1839 	err = mlx5_flow_namespace_set_mode(root_ns,
1840 					   esw->dev->priv.steering->mode);
1841 	if (err) {
1842 		esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1843 		goto ns_err;
1844 	}
1845 
1846 	/* To be strictly correct:
1847 	 *	MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1848 	 * should be:
1849 	 *	esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1850 	 *	peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1851 	 * but as the peer device might not be in switchdev mode it's not
1852 	 * possible. We use the fact that by default FW sets max vfs and max sfs
1853 	 * to the same value on both devices. If it needs to be changed in the future note
1854 	 * the peer miss group should also be created based on the number of
1855 	 * total vports of the peer (currently is also uses esw->total_vports).
1856 	 */
1857 	table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1858 		     esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS;
1859 
1860 	/* create the slow path fdb with encap set, so further table instances
1861 	 * can be created at run time while VFs are probed if the FW allows that.
1862 	 */
1863 	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1864 		flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1865 			  MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1866 
1867 	ft_attr.flags = flags;
1868 	ft_attr.max_fte = table_size;
1869 	ft_attr.prio = FDB_SLOW_PATH;
1870 
1871 	fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1872 	if (IS_ERR(fdb)) {
1873 		err = PTR_ERR(fdb);
1874 		esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1875 		goto slow_fdb_err;
1876 	}
1877 	esw->fdb_table.offloads.slow_fdb = fdb;
1878 
1879 	/* Create empty TC-miss managed table. This allows plugging in following
1880 	 * priorities without directly exposing their level 0 table to
1881 	 * eswitch_offloads and passing it as miss_fdb to following call to
1882 	 * esw_chains_create().
1883 	 */
1884 	memset(&ft_attr, 0, sizeof(ft_attr));
1885 	ft_attr.prio = FDB_TC_MISS;
1886 	esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1887 	if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1888 		err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1889 		esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1890 		goto tc_miss_table_err;
1891 	}
1892 
1893 	err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
1894 	if (err) {
1895 		esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1896 		goto fdb_chains_err;
1897 	}
1898 
1899 	err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1900 	if (err)
1901 		goto send_vport_err;
1902 
1903 	err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1904 	if (err)
1905 		goto send_vport_meta_err;
1906 
1907 	err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix);
1908 	if (err)
1909 		goto peer_miss_err;
1910 
1911 	err = esw_create_miss_group(esw, fdb, flow_group_in, &ix);
1912 	if (err)
1913 		goto miss_err;
1914 
1915 	kvfree(flow_group_in);
1916 	return 0;
1917 
1918 miss_err:
1919 	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1920 		mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1921 peer_miss_err:
1922 	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1923 		mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1924 send_vport_meta_err:
1925 	mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1926 send_vport_err:
1927 	esw_chains_destroy(esw, esw_chains(esw));
1928 fdb_chains_err:
1929 	mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1930 tc_miss_table_err:
1931 	mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
1932 slow_fdb_err:
1933 	/* Holds true only as long as DMFS is the default */
1934 	mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
1935 ns_err:
1936 	kvfree(flow_group_in);
1937 	return err;
1938 }
1939 
esw_destroy_offloads_fdb_tables(struct mlx5_eswitch * esw)1940 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1941 {
1942 	if (!mlx5_eswitch_get_slow_fdb(esw))
1943 		return;
1944 
1945 	esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1946 	mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1947 	mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1948 	mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1949 	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1950 		mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1951 	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1952 		mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1953 	mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1954 
1955 	esw_chains_destroy(esw, esw_chains(esw));
1956 
1957 	mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1958 	mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
1959 	/* Holds true only as long as DMFS is the default */
1960 	mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1961 				     MLX5_FLOW_STEERING_MODE_DMFS);
1962 	atomic64_set(&esw->user_count, 0);
1963 }
1964 
esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch * esw)1965 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
1966 {
1967 	int nvports;
1968 
1969 	nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1970 	if (mlx5e_tc_int_port_supported(esw))
1971 		nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1972 
1973 	return nvports;
1974 }
1975 
esw_create_offloads_table(struct mlx5_eswitch * esw)1976 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1977 {
1978 	struct mlx5_flow_table_attr ft_attr = {};
1979 	struct mlx5_core_dev *dev = esw->dev;
1980 	struct mlx5_flow_table *ft_offloads;
1981 	struct mlx5_flow_namespace *ns;
1982 	int err = 0;
1983 
1984 	ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1985 	if (!ns) {
1986 		esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1987 		return -EOPNOTSUPP;
1988 	}
1989 
1990 	ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
1991 			  MLX5_ESW_FT_OFFLOADS_DROP_RULE;
1992 	ft_attr.prio = 1;
1993 
1994 	ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
1995 	if (IS_ERR(ft_offloads)) {
1996 		err = PTR_ERR(ft_offloads);
1997 		esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
1998 		return err;
1999 	}
2000 
2001 	esw->offloads.ft_offloads = ft_offloads;
2002 	return 0;
2003 }
2004 
esw_destroy_offloads_table(struct mlx5_eswitch * esw)2005 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2006 {
2007 	struct mlx5_esw_offload *offloads = &esw->offloads;
2008 
2009 	mlx5_destroy_flow_table(offloads->ft_offloads);
2010 }
2011 
esw_create_vport_rx_group(struct mlx5_eswitch * esw)2012 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2013 {
2014 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2015 	struct mlx5_flow_group *g;
2016 	u32 *flow_group_in;
2017 	int nvports;
2018 	int err = 0;
2019 
2020 	nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2021 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2022 	if (!flow_group_in)
2023 		return -ENOMEM;
2024 
2025 	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
2026 
2027 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2028 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2029 
2030 	g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2031 
2032 	if (IS_ERR(g)) {
2033 		err = PTR_ERR(g);
2034 		mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
2035 		goto out;
2036 	}
2037 
2038 	esw->offloads.vport_rx_group = g;
2039 out:
2040 	kvfree(flow_group_in);
2041 	return err;
2042 }
2043 
esw_destroy_vport_rx_group(struct mlx5_eswitch * esw)2044 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2045 {
2046 	mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
2047 }
2048 
esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch * esw)2049 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2050 {
2051 	/* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2052 	 * for the drop rule, which is placed at the end of the table.
2053 	 * So return the total of vport and int_port as rule index.
2054 	 */
2055 	return esw_get_nr_ft_offloads_steering_src_ports(esw);
2056 }
2057 
esw_create_vport_rx_drop_group(struct mlx5_eswitch * esw)2058 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2059 {
2060 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2061 	struct mlx5_flow_group *g;
2062 	u32 *flow_group_in;
2063 	int flow_index;
2064 	int err = 0;
2065 
2066 	flow_index = esw_create_vport_rx_drop_rule_index(esw);
2067 
2068 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2069 	if (!flow_group_in)
2070 		return -ENOMEM;
2071 
2072 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2073 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2074 
2075 	g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2076 
2077 	if (IS_ERR(g)) {
2078 		err = PTR_ERR(g);
2079 		mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
2080 		goto out;
2081 	}
2082 
2083 	esw->offloads.vport_rx_drop_group = g;
2084 out:
2085 	kvfree(flow_group_in);
2086 	return err;
2087 }
2088 
esw_destroy_vport_rx_drop_group(struct mlx5_eswitch * esw)2089 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2090 {
2091 	if (esw->offloads.vport_rx_drop_group)
2092 		mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group);
2093 }
2094 
2095 void
mlx5_esw_set_spec_source_port(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_spec * spec)2096 mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw,
2097 			      u16 vport,
2098 			      struct mlx5_flow_spec *spec)
2099 {
2100 	void *misc;
2101 
2102 	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2103 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2104 		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2105 			 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2106 
2107 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2108 		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2109 			 mlx5_eswitch_get_vport_metadata_mask());
2110 
2111 		spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2112 	} else {
2113 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2114 		MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2115 
2116 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2117 		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2118 
2119 		spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2120 	}
2121 }
2122 
2123 struct mlx5_flow_handle *
mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_destination * dest)2124 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2125 				  struct mlx5_flow_destination *dest)
2126 {
2127 	struct mlx5_flow_act flow_act = {0};
2128 	struct mlx5_flow_handle *flow_rule;
2129 	struct mlx5_flow_spec *spec;
2130 
2131 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2132 	if (!spec) {
2133 		flow_rule = ERR_PTR(-ENOMEM);
2134 		goto out;
2135 	}
2136 
2137 	mlx5_esw_set_spec_source_port(esw, vport, spec);
2138 
2139 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2140 	flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
2141 					&flow_act, dest, 1);
2142 	if (IS_ERR(flow_rule)) {
2143 		esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
2144 		goto out;
2145 	}
2146 
2147 out:
2148 	kvfree(spec);
2149 	return flow_rule;
2150 }
2151 
esw_create_vport_rx_drop_rule(struct mlx5_eswitch * esw)2152 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2153 {
2154 	struct mlx5_flow_act flow_act = {};
2155 	struct mlx5_flow_handle *flow_rule;
2156 
2157 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2158 	flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL,
2159 					&flow_act, NULL, 0);
2160 	if (IS_ERR(flow_rule)) {
2161 		esw_warn(esw->dev,
2162 			 "fs offloads: Failed to add vport rx drop rule err %ld\n",
2163 			 PTR_ERR(flow_rule));
2164 		return PTR_ERR(flow_rule);
2165 	}
2166 
2167 	esw->offloads.vport_rx_drop_rule = flow_rule;
2168 
2169 	return 0;
2170 }
2171 
esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch * esw)2172 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2173 {
2174 	if (esw->offloads.vport_rx_drop_rule)
2175 		mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule);
2176 }
2177 
mlx5_eswitch_inline_mode_get(struct mlx5_eswitch * esw,u8 * mode)2178 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2179 {
2180 	u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2181 	struct mlx5_core_dev *dev = esw->dev;
2182 	struct mlx5_vport *vport;
2183 	unsigned long i;
2184 
2185 	if (!MLX5_CAP_GEN(dev, vport_group_manager))
2186 		return -EOPNOTSUPP;
2187 
2188 	if (!mlx5_esw_is_fdb_created(esw))
2189 		return -EOPNOTSUPP;
2190 
2191 	switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2192 	case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2193 		mlx5_mode = MLX5_INLINE_MODE_NONE;
2194 		goto out;
2195 	case MLX5_CAP_INLINE_MODE_L2:
2196 		mlx5_mode = MLX5_INLINE_MODE_L2;
2197 		goto out;
2198 	case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2199 		goto query_vports;
2200 	}
2201 
2202 query_vports:
2203 	mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
2204 	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2205 		mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
2206 		if (prev_mlx5_mode != mlx5_mode)
2207 			return -EINVAL;
2208 		prev_mlx5_mode = mlx5_mode;
2209 	}
2210 
2211 out:
2212 	*mode = mlx5_mode;
2213 	return 0;
2214 }
2215 
esw_destroy_restore_table(struct mlx5_eswitch * esw)2216 static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2217 {
2218 	struct mlx5_esw_offload *offloads = &esw->offloads;
2219 
2220 	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2221 		return;
2222 
2223 	mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
2224 	mlx5_destroy_flow_group(offloads->restore_group);
2225 	mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2226 }
2227 
esw_create_restore_table(struct mlx5_eswitch * esw)2228 static int esw_create_restore_table(struct mlx5_eswitch *esw)
2229 {
2230 	u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2231 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2232 	struct mlx5_flow_table_attr ft_attr = {};
2233 	struct mlx5_core_dev *dev = esw->dev;
2234 	struct mlx5_flow_namespace *ns;
2235 	struct mlx5_modify_hdr *mod_hdr;
2236 	void *match_criteria, *misc;
2237 	struct mlx5_flow_table *ft;
2238 	struct mlx5_flow_group *g;
2239 	u32 *flow_group_in;
2240 	int err = 0;
2241 
2242 	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2243 		return 0;
2244 
2245 	ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2246 	if (!ns) {
2247 		esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2248 		return -EOPNOTSUPP;
2249 	}
2250 
2251 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2252 	if (!flow_group_in) {
2253 		err = -ENOMEM;
2254 		goto out_free;
2255 	}
2256 
2257 	ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2258 	ft = mlx5_create_flow_table(ns, &ft_attr);
2259 	if (IS_ERR(ft)) {
2260 		err = PTR_ERR(ft);
2261 		esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2262 			 err);
2263 		goto out_free;
2264 	}
2265 
2266 	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2267 				      match_criteria);
2268 	misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2269 			    misc_parameters_2);
2270 
2271 	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2272 		 ESW_REG_C0_USER_DATA_METADATA_MASK);
2273 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2274 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2275 		 ft_attr.max_fte - 1);
2276 	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2277 		 MLX5_MATCH_MISC_PARAMETERS_2);
2278 	g = mlx5_create_flow_group(ft, flow_group_in);
2279 	if (IS_ERR(g)) {
2280 		err = PTR_ERR(g);
2281 		esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2282 			 err);
2283 		goto err_group;
2284 	}
2285 
2286 	MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2287 	MLX5_SET(copy_action_in, modact, src_field,
2288 		 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2289 	MLX5_SET(copy_action_in, modact, dst_field,
2290 		 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2291 	mod_hdr = mlx5_modify_header_alloc(esw->dev,
2292 					   MLX5_FLOW_NAMESPACE_KERNEL, 1,
2293 					   modact);
2294 	if (IS_ERR(mod_hdr)) {
2295 		err = PTR_ERR(mod_hdr);
2296 		esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2297 			 err);
2298 		goto err_mod_hdr;
2299 	}
2300 
2301 	esw->offloads.ft_offloads_restore = ft;
2302 	esw->offloads.restore_group = g;
2303 	esw->offloads.restore_copy_hdr_id = mod_hdr;
2304 
2305 	kvfree(flow_group_in);
2306 
2307 	return 0;
2308 
2309 err_mod_hdr:
2310 	mlx5_destroy_flow_group(g);
2311 err_group:
2312 	mlx5_destroy_flow_table(ft);
2313 out_free:
2314 	kvfree(flow_group_in);
2315 
2316 	return err;
2317 }
2318 
esw_offloads_start(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)2319 static int esw_offloads_start(struct mlx5_eswitch *esw,
2320 			      struct netlink_ext_ack *extack)
2321 {
2322 	int err;
2323 
2324 	esw->mode = MLX5_ESWITCH_OFFLOADS;
2325 	err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs);
2326 	if (err) {
2327 		NL_SET_ERR_MSG_MOD(extack,
2328 				   "Failed setting eswitch to offloads");
2329 		esw->mode = MLX5_ESWITCH_LEGACY;
2330 		mlx5_rescan_drivers(esw->dev);
2331 		return err;
2332 	}
2333 	if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2334 		if (mlx5_eswitch_inline_mode_get(esw,
2335 						 &esw->offloads.inline_mode)) {
2336 			esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2337 			NL_SET_ERR_MSG_MOD(extack,
2338 					   "Inline mode is different between vports");
2339 		}
2340 	}
2341 	return 0;
2342 }
2343 
mlx5_esw_offloads_rep_init(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2344 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
2345 {
2346 	struct mlx5_eswitch_rep *rep;
2347 	int rep_type;
2348 	int err;
2349 
2350 	rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2351 	if (!rep)
2352 		return -ENOMEM;
2353 
2354 	rep->vport = vport->vport;
2355 	rep->vport_index = vport->index;
2356 	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2357 		atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
2358 
2359 	err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2360 	if (err)
2361 		goto insert_err;
2362 
2363 	return 0;
2364 
2365 insert_err:
2366 	kfree(rep);
2367 	return err;
2368 }
2369 
mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep)2370 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2371 					  struct mlx5_eswitch_rep *rep)
2372 {
2373 	xa_erase(&esw->offloads.vport_reps, rep->vport);
2374 	kfree(rep);
2375 }
2376 
esw_offloads_cleanup_reps(struct mlx5_eswitch * esw)2377 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2378 {
2379 	struct mlx5_eswitch_rep *rep;
2380 	unsigned long i;
2381 
2382 	mlx5_esw_for_each_rep(esw, i, rep)
2383 		mlx5_esw_offloads_rep_cleanup(esw, rep);
2384 	xa_destroy(&esw->offloads.vport_reps);
2385 }
2386 
esw_offloads_init_reps(struct mlx5_eswitch * esw)2387 static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2388 {
2389 	struct mlx5_vport *vport;
2390 	unsigned long i;
2391 	int err;
2392 
2393 	xa_init(&esw->offloads.vport_reps);
2394 
2395 	mlx5_esw_for_each_vport(esw, i, vport) {
2396 		err = mlx5_esw_offloads_rep_init(esw, vport);
2397 		if (err)
2398 			goto err;
2399 	}
2400 	return 0;
2401 
2402 err:
2403 	esw_offloads_cleanup_reps(esw);
2404 	return err;
2405 }
2406 
esw_port_metadata_set(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx)2407 static int esw_port_metadata_set(struct devlink *devlink, u32 id,
2408 				 struct devlink_param_gset_ctx *ctx)
2409 {
2410 	struct mlx5_core_dev *dev = devlink_priv(devlink);
2411 	struct mlx5_eswitch *esw = dev->priv.eswitch;
2412 	int err = 0;
2413 
2414 	down_write(&esw->mode_lock);
2415 	if (mlx5_esw_is_fdb_created(esw)) {
2416 		err = -EBUSY;
2417 		goto done;
2418 	}
2419 	if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2420 		err = -EOPNOTSUPP;
2421 		goto done;
2422 	}
2423 	if (ctx->val.vbool)
2424 		esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2425 	else
2426 		esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2427 done:
2428 	up_write(&esw->mode_lock);
2429 	return err;
2430 }
2431 
esw_port_metadata_get(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx)2432 static int esw_port_metadata_get(struct devlink *devlink, u32 id,
2433 				 struct devlink_param_gset_ctx *ctx)
2434 {
2435 	struct mlx5_core_dev *dev = devlink_priv(devlink);
2436 
2437 	ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch);
2438 	return 0;
2439 }
2440 
esw_port_metadata_validate(struct devlink * devlink,u32 id,union devlink_param_value val,struct netlink_ext_ack * extack)2441 static int esw_port_metadata_validate(struct devlink *devlink, u32 id,
2442 				      union devlink_param_value val,
2443 				      struct netlink_ext_ack *extack)
2444 {
2445 	struct mlx5_core_dev *dev = devlink_priv(devlink);
2446 	u8 esw_mode;
2447 
2448 	esw_mode = mlx5_eswitch_mode(dev);
2449 	if (esw_mode == MLX5_ESWITCH_OFFLOADS) {
2450 		NL_SET_ERR_MSG_MOD(extack,
2451 				   "E-Switch must either disabled or non switchdev mode");
2452 		return -EBUSY;
2453 	}
2454 	return 0;
2455 }
2456 
2457 static const struct devlink_param esw_devlink_params[] = {
2458 	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
2459 			     "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL,
2460 			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2461 			     esw_port_metadata_get,
2462 			     esw_port_metadata_set,
2463 			     esw_port_metadata_validate),
2464 };
2465 
esw_offloads_init(struct mlx5_eswitch * esw)2466 int esw_offloads_init(struct mlx5_eswitch *esw)
2467 {
2468 	int err;
2469 
2470 	err = esw_offloads_init_reps(esw);
2471 	if (err)
2472 		return err;
2473 
2474 	err = devl_params_register(priv_to_devlink(esw->dev),
2475 				   esw_devlink_params,
2476 				   ARRAY_SIZE(esw_devlink_params));
2477 	if (err)
2478 		goto err_params;
2479 
2480 	return 0;
2481 
2482 err_params:
2483 	esw_offloads_cleanup_reps(esw);
2484 	return err;
2485 }
2486 
esw_offloads_cleanup(struct mlx5_eswitch * esw)2487 void esw_offloads_cleanup(struct mlx5_eswitch *esw)
2488 {
2489 	devl_params_unregister(priv_to_devlink(esw->dev),
2490 			       esw_devlink_params,
2491 			       ARRAY_SIZE(esw_devlink_params));
2492 	esw_offloads_cleanup_reps(esw);
2493 }
2494 
__esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2495 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2496 				      struct mlx5_eswitch_rep *rep, u8 rep_type)
2497 {
2498 	if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2499 			   REP_LOADED, REP_REGISTERED) == REP_LOADED)
2500 		esw->offloads.rep_ops[rep_type]->unload(rep);
2501 }
2502 
__unload_reps_all_vport(struct mlx5_eswitch * esw,u8 rep_type)2503 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2504 {
2505 	struct mlx5_eswitch_rep *rep;
2506 	unsigned long i;
2507 
2508 	mlx5_esw_for_each_rep(esw, i, rep)
2509 		__esw_offloads_unload_rep(esw, rep, rep_type);
2510 }
2511 
mlx5_esw_offloads_rep_load(struct mlx5_eswitch * esw,u16 vport_num)2512 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2513 {
2514 	struct mlx5_eswitch_rep *rep;
2515 	int rep_type;
2516 	int err;
2517 
2518 	rep = mlx5_eswitch_get_rep(esw, vport_num);
2519 	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2520 		if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2521 				   REP_REGISTERED, REP_LOADED) == REP_REGISTERED) {
2522 			err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2523 			if (err)
2524 				goto err_reps;
2525 		}
2526 
2527 	return 0;
2528 
2529 err_reps:
2530 	atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2531 	for (--rep_type; rep_type >= 0; rep_type--)
2532 		__esw_offloads_unload_rep(esw, rep, rep_type);
2533 	return err;
2534 }
2535 
mlx5_esw_offloads_rep_unload(struct mlx5_eswitch * esw,u16 vport_num)2536 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2537 {
2538 	struct mlx5_eswitch_rep *rep;
2539 	int rep_type;
2540 
2541 	rep = mlx5_eswitch_get_rep(esw, vport_num);
2542 	for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2543 		__esw_offloads_unload_rep(esw, rep, rep_type);
2544 }
2545 
mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2546 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2547 {
2548 	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2549 		return 0;
2550 
2551 	return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport);
2552 }
2553 
mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2554 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2555 {
2556 	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2557 		return;
2558 
2559 	mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport);
2560 }
2561 
mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport,struct mlx5_devlink_port * dl_port,u32 controller,u32 sfnum)2562 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
2563 				  struct mlx5_devlink_port *dl_port,
2564 				  u32 controller, u32 sfnum)
2565 {
2566 	return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum);
2567 }
2568 
mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2569 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2570 {
2571 	mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport);
2572 }
2573 
mlx5_esw_offloads_load_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2574 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2575 {
2576 	int err;
2577 
2578 	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2579 		return 0;
2580 
2581 	err = mlx5_esw_offloads_devlink_port_register(esw, vport);
2582 	if (err)
2583 		return err;
2584 
2585 	err = mlx5_esw_offloads_rep_load(esw, vport->vport);
2586 	if (err)
2587 		goto load_err;
2588 	return err;
2589 
2590 load_err:
2591 	mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2592 	return err;
2593 }
2594 
mlx5_esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2595 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2596 {
2597 	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2598 		return;
2599 
2600 	mlx5_esw_offloads_rep_unload(esw, vport->vport);
2601 
2602 	mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2603 }
2604 
esw_set_slave_root_fdb(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2605 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2606 				  struct mlx5_core_dev *slave)
2607 {
2608 	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)]   = {};
2609 	u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2610 	struct mlx5_flow_root_namespace *root;
2611 	struct mlx5_flow_namespace *ns;
2612 	int err;
2613 
2614 	MLX5_SET(set_flow_table_root_in, in, opcode,
2615 		 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2616 	MLX5_SET(set_flow_table_root_in, in, table_type,
2617 		 FS_FT_FDB);
2618 
2619 	if (master) {
2620 		ns = mlx5_get_flow_namespace(master,
2621 					     MLX5_FLOW_NAMESPACE_FDB);
2622 		root = find_root(&ns->node);
2623 		mutex_lock(&root->chain_lock);
2624 		MLX5_SET(set_flow_table_root_in, in,
2625 			 table_eswitch_owner_vhca_id_valid, 1);
2626 		MLX5_SET(set_flow_table_root_in, in,
2627 			 table_eswitch_owner_vhca_id,
2628 			 MLX5_CAP_GEN(master, vhca_id));
2629 		MLX5_SET(set_flow_table_root_in, in, table_id,
2630 			 root->root_ft->id);
2631 	} else {
2632 		ns = mlx5_get_flow_namespace(slave,
2633 					     MLX5_FLOW_NAMESPACE_FDB);
2634 		root = find_root(&ns->node);
2635 		mutex_lock(&root->chain_lock);
2636 		MLX5_SET(set_flow_table_root_in, in, table_id,
2637 			 root->root_ft->id);
2638 	}
2639 
2640 	err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2641 	mutex_unlock(&root->chain_lock);
2642 
2643 	return err;
2644 }
2645 
__esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,struct mlx5_vport * vport,struct mlx5_flow_table * acl)2646 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2647 					struct mlx5_core_dev *slave,
2648 					struct mlx5_vport *vport,
2649 					struct mlx5_flow_table *acl)
2650 {
2651 	u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2652 	struct mlx5_flow_handle *flow_rule = NULL;
2653 	struct mlx5_flow_destination dest = {};
2654 	struct mlx5_flow_act flow_act = {};
2655 	struct mlx5_flow_spec *spec;
2656 	int err = 0;
2657 	void *misc;
2658 
2659 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2660 	if (!spec)
2661 		return -ENOMEM;
2662 
2663 	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2664 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2665 			    misc_parameters);
2666 	MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2667 	MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index);
2668 
2669 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2670 	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2671 	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2672 			 source_eswitch_owner_vhca_id);
2673 
2674 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2675 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2676 	dest.vport.num = slave->priv.eswitch->manager_vport;
2677 	dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2678 	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2679 
2680 	flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2681 					&dest, 1);
2682 	if (IS_ERR(flow_rule)) {
2683 		err = PTR_ERR(flow_rule);
2684 	} else {
2685 		err = xa_insert(&vport->egress.offloads.bounce_rules,
2686 				slave_index, flow_rule, GFP_KERNEL);
2687 		if (err)
2688 			mlx5_del_flow_rules(flow_rule);
2689 	}
2690 
2691 	kvfree(spec);
2692 	return err;
2693 }
2694 
esw_master_egress_create_resources(struct mlx5_eswitch * esw,struct mlx5_flow_namespace * egress_ns,struct mlx5_vport * vport,size_t count)2695 static int esw_master_egress_create_resources(struct mlx5_eswitch *esw,
2696 					      struct mlx5_flow_namespace *egress_ns,
2697 					      struct mlx5_vport *vport, size_t count)
2698 {
2699 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2700 	struct mlx5_flow_table_attr ft_attr = {
2701 		.max_fte = count, .prio = 0, .level = 0,
2702 	};
2703 	struct mlx5_flow_table *acl;
2704 	struct mlx5_flow_group *g;
2705 	void *match_criteria;
2706 	u32 *flow_group_in;
2707 	int err;
2708 
2709 	if (vport->egress.acl)
2710 		return 0;
2711 
2712 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2713 	if (!flow_group_in)
2714 		return -ENOMEM;
2715 
2716 	if (vport->vport || mlx5_core_is_ecpf(esw->dev))
2717 		ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT;
2718 
2719 	acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2720 	if (IS_ERR(acl)) {
2721 		err = PTR_ERR(acl);
2722 		goto out;
2723 	}
2724 
2725 	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2726 				      match_criteria);
2727 	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2728 			 misc_parameters.source_port);
2729 	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2730 			 misc_parameters.source_eswitch_owner_vhca_id);
2731 	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2732 		 MLX5_MATCH_MISC_PARAMETERS);
2733 
2734 	MLX5_SET(create_flow_group_in, flow_group_in,
2735 		 source_eswitch_owner_vhca_id_valid, 1);
2736 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2737 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count);
2738 
2739 	g = mlx5_create_flow_group(acl, flow_group_in);
2740 	if (IS_ERR(g)) {
2741 		err = PTR_ERR(g);
2742 		goto err_group;
2743 	}
2744 
2745 	vport->egress.acl = acl;
2746 	vport->egress.offloads.bounce_grp = g;
2747 	vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB;
2748 	xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC);
2749 
2750 	kvfree(flow_group_in);
2751 
2752 	return 0;
2753 
2754 err_group:
2755 	mlx5_destroy_flow_table(acl);
2756 out:
2757 	kvfree(flow_group_in);
2758 	return err;
2759 }
2760 
esw_master_egress_destroy_resources(struct mlx5_vport * vport)2761 static void esw_master_egress_destroy_resources(struct mlx5_vport *vport)
2762 {
2763 	if (!xa_empty(&vport->egress.offloads.bounce_rules))
2764 		return;
2765 	mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp);
2766 	vport->egress.offloads.bounce_grp = NULL;
2767 	mlx5_destroy_flow_table(vport->egress.acl);
2768 	vport->egress.acl = NULL;
2769 }
2770 
esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,size_t count)2771 static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2772 				      struct mlx5_core_dev *slave, size_t count)
2773 {
2774 	struct mlx5_eswitch *esw = master->priv.eswitch;
2775 	u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2776 	struct mlx5_flow_namespace *egress_ns;
2777 	struct mlx5_vport *vport;
2778 	int err;
2779 
2780 	vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
2781 	if (IS_ERR(vport))
2782 		return PTR_ERR(vport);
2783 
2784 	egress_ns = mlx5_get_flow_vport_acl_namespace(master,
2785 						      MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2786 						      vport->index);
2787 	if (!egress_ns)
2788 		return -EINVAL;
2789 
2790 	if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB)
2791 		return 0;
2792 
2793 	err = esw_master_egress_create_resources(esw, egress_ns, vport, count);
2794 	if (err)
2795 		return err;
2796 
2797 	if (xa_load(&vport->egress.offloads.bounce_rules, slave_index))
2798 		return -EINVAL;
2799 
2800 	err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl);
2801 	if (err)
2802 		goto err_rule;
2803 
2804 	return 0;
2805 
2806 err_rule:
2807 	esw_master_egress_destroy_resources(vport);
2808 	return err;
2809 }
2810 
esw_unset_master_egress_rule(struct mlx5_core_dev * dev,struct mlx5_core_dev * slave_dev)2811 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev,
2812 					 struct mlx5_core_dev *slave_dev)
2813 {
2814 	struct mlx5_vport *vport;
2815 
2816 	vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
2817 				       dev->priv.eswitch->manager_vport);
2818 
2819 	esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id));
2820 
2821 	if (xa_empty(&vport->egress.offloads.bounce_rules)) {
2822 		esw_acl_egress_ofld_cleanup(vport);
2823 		xa_destroy(&vport->egress.offloads.bounce_rules);
2824 	}
2825 }
2826 
mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw,int max_slaves)2827 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw,
2828 					     struct mlx5_eswitch *slave_esw, int max_slaves)
2829 {
2830 	int err;
2831 
2832 	err = esw_set_slave_root_fdb(master_esw->dev,
2833 				     slave_esw->dev);
2834 	if (err)
2835 		return err;
2836 
2837 	err = esw_set_master_egress_rule(master_esw->dev,
2838 					 slave_esw->dev, max_slaves);
2839 	if (err)
2840 		goto err_acl;
2841 
2842 	return err;
2843 
2844 err_acl:
2845 	esw_set_slave_root_fdb(NULL, slave_esw->dev);
2846 	return err;
2847 }
2848 
mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2849 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw,
2850 					      struct mlx5_eswitch *slave_esw)
2851 {
2852 	esw_set_slave_root_fdb(NULL, slave_esw->dev);
2853 	esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev);
2854 }
2855 
2856 #define ESW_OFFLOADS_DEVCOM_PAIR	(0)
2857 #define ESW_OFFLOADS_DEVCOM_UNPAIR	(1)
2858 
mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2859 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw,
2860 					       struct mlx5_eswitch *peer_esw)
2861 {
2862 	const struct mlx5_eswitch_rep_ops *ops;
2863 	struct mlx5_eswitch_rep *rep;
2864 	unsigned long i;
2865 	u8 rep_type;
2866 
2867 	mlx5_esw_for_each_rep(esw, i, rep) {
2868 		rep_type = NUM_REP_TYPES;
2869 		while (rep_type--) {
2870 			ops = esw->offloads.rep_ops[rep_type];
2871 			if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2872 			    ops->event)
2873 				ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw);
2874 		}
2875 	}
2876 }
2877 
mlx5_esw_offloads_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2878 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw,
2879 				     struct mlx5_eswitch *peer_esw)
2880 {
2881 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2882 	mlx5e_tc_clean_fdb_peer_flows(esw);
2883 #endif
2884 	mlx5_esw_offloads_rep_event_unpair(esw, peer_esw);
2885 	esw_del_fdb_peer_miss_rules(esw, peer_esw->dev);
2886 }
2887 
mlx5_esw_offloads_pair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2888 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2889 				  struct mlx5_eswitch *peer_esw)
2890 {
2891 	const struct mlx5_eswitch_rep_ops *ops;
2892 	struct mlx5_eswitch_rep *rep;
2893 	unsigned long i;
2894 	u8 rep_type;
2895 	int err;
2896 
2897 	err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2898 	if (err)
2899 		return err;
2900 
2901 	mlx5_esw_for_each_rep(esw, i, rep) {
2902 		for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2903 			ops = esw->offloads.rep_ops[rep_type];
2904 			if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2905 			    ops->event) {
2906 				err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2907 				if (err)
2908 					goto err_out;
2909 			}
2910 		}
2911 	}
2912 
2913 	return 0;
2914 
2915 err_out:
2916 	mlx5_esw_offloads_unpair(esw, peer_esw);
2917 	return err;
2918 }
2919 
mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,bool pair)2920 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2921 					 struct mlx5_eswitch *peer_esw,
2922 					 bool pair)
2923 {
2924 	u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2925 	u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
2926 	struct mlx5_flow_root_namespace *peer_ns;
2927 	struct mlx5_flow_root_namespace *ns;
2928 	int err;
2929 
2930 	peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2931 	ns = esw->dev->priv.steering->fdb_root_ns;
2932 
2933 	if (pair) {
2934 		err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id);
2935 		if (err)
2936 			return err;
2937 
2938 		err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id);
2939 		if (err) {
2940 			mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2941 			return err;
2942 		}
2943 	} else {
2944 		mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2945 		mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id);
2946 	}
2947 
2948 	return 0;
2949 }
2950 
mlx5_esw_offloads_devcom_event(int event,void * my_data,void * event_data)2951 static int mlx5_esw_offloads_devcom_event(int event,
2952 					  void *my_data,
2953 					  void *event_data)
2954 {
2955 	struct mlx5_eswitch *esw = my_data;
2956 	struct mlx5_eswitch *peer_esw = event_data;
2957 	u16 esw_i, peer_esw_i;
2958 	bool esw_paired;
2959 	int err;
2960 
2961 	peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2962 	esw_i = MLX5_CAP_GEN(esw->dev, vhca_id);
2963 	esw_paired = !!xa_load(&esw->paired, peer_esw_i);
2964 
2965 	switch (event) {
2966 	case ESW_OFFLOADS_DEVCOM_PAIR:
2967 		if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2968 		    mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2969 			break;
2970 
2971 		if (esw_paired)
2972 			break;
2973 
2974 		err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
2975 		if (err)
2976 			goto err_out;
2977 
2978 		err = mlx5_esw_offloads_pair(esw, peer_esw);
2979 		if (err)
2980 			goto err_peer;
2981 
2982 		err = mlx5_esw_offloads_pair(peer_esw, esw);
2983 		if (err)
2984 			goto err_pair;
2985 
2986 		err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL);
2987 		if (err)
2988 			goto err_xa;
2989 
2990 		err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL);
2991 		if (err)
2992 			goto err_peer_xa;
2993 
2994 		esw->num_peers++;
2995 		peer_esw->num_peers++;
2996 		mlx5_devcom_comp_set_ready(esw->devcom, true);
2997 		break;
2998 
2999 	case ESW_OFFLOADS_DEVCOM_UNPAIR:
3000 		if (!esw_paired)
3001 			break;
3002 
3003 		peer_esw->num_peers--;
3004 		esw->num_peers--;
3005 		if (!esw->num_peers && !peer_esw->num_peers)
3006 			mlx5_devcom_comp_set_ready(esw->devcom, false);
3007 		xa_erase(&peer_esw->paired, esw_i);
3008 		xa_erase(&esw->paired, peer_esw_i);
3009 		mlx5_esw_offloads_unpair(peer_esw, esw);
3010 		mlx5_esw_offloads_unpair(esw, peer_esw);
3011 		mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3012 		break;
3013 	}
3014 
3015 	return 0;
3016 
3017 err_peer_xa:
3018 	xa_erase(&esw->paired, peer_esw_i);
3019 err_xa:
3020 	mlx5_esw_offloads_unpair(peer_esw, esw);
3021 err_pair:
3022 	mlx5_esw_offloads_unpair(esw, peer_esw);
3023 err_peer:
3024 	mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3025 err_out:
3026 	mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
3027 		      event, err);
3028 	return err;
3029 }
3030 
mlx5_esw_offloads_devcom_init(struct mlx5_eswitch * esw,u64 key)3031 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key)
3032 {
3033 	int i;
3034 
3035 	for (i = 0; i < MLX5_MAX_PORTS; i++)
3036 		INIT_LIST_HEAD(&esw->offloads.peer_flows[i]);
3037 	mutex_init(&esw->offloads.peer_mutex);
3038 
3039 	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
3040 		return;
3041 
3042 	if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) &&
3043 	    !mlx5_lag_is_supported(esw->dev))
3044 		return;
3045 
3046 	xa_init(&esw->paired);
3047 	esw->num_peers = 0;
3048 	esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc,
3049 						     MLX5_DEVCOM_ESW_OFFLOADS,
3050 						     key,
3051 						     mlx5_esw_offloads_devcom_event,
3052 						     esw);
3053 	if (IS_ERR_OR_NULL(esw->devcom))
3054 		return;
3055 
3056 	mlx5_devcom_send_event(esw->devcom,
3057 			       ESW_OFFLOADS_DEVCOM_PAIR,
3058 			       ESW_OFFLOADS_DEVCOM_UNPAIR,
3059 			       esw);
3060 }
3061 
mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch * esw)3062 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
3063 {
3064 	if (IS_ERR_OR_NULL(esw->devcom))
3065 		return;
3066 
3067 	mlx5_devcom_send_event(esw->devcom,
3068 			       ESW_OFFLOADS_DEVCOM_UNPAIR,
3069 			       ESW_OFFLOADS_DEVCOM_UNPAIR,
3070 			       esw);
3071 
3072 	mlx5_devcom_unregister_component(esw->devcom);
3073 	xa_destroy(&esw->paired);
3074 	esw->devcom = NULL;
3075 }
3076 
mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch * esw)3077 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw)
3078 {
3079 	return mlx5_devcom_comp_is_ready(esw->devcom);
3080 }
3081 
mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch * esw)3082 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
3083 {
3084 	if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
3085 		return false;
3086 
3087 	if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
3088 	      MLX5_FDB_TO_VPORT_REG_C_0))
3089 		return false;
3090 
3091 	return true;
3092 }
3093 
3094 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
3095 
3096 /* Share the same metadata for uplink's. This is fine because:
3097  * (a) In shared FDB mode (LAG) both uplink's are treated the
3098  *     same and tagged with the same metadata.
3099  * (b) In non shared FDB mode, packets from physical port0
3100  *     cannot hit eswitch of PF1 and vice versa.
3101  */
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch * esw)3102 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
3103 {
3104 	return MLX5_ESW_METADATA_RSVD_UPLINK;
3105 }
3106 
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch * esw)3107 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
3108 {
3109 	u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
3110 	/* Reserve 0xf for internal port offload */
3111 	u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
3112 	u32 pf_num;
3113 	int id;
3114 
3115 	/* Only 4 bits of pf_num */
3116 	pf_num = mlx5_get_dev_index(esw->dev);
3117 	if (pf_num > max_pf_num)
3118 		return 0;
3119 
3120 	/* Metadata is 4 bits of PFNUM and 12 bits of unique id */
3121 	/* Use only non-zero vport_id (2-4095) for all PF's */
3122 	id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
3123 			     MLX5_ESW_METADATA_RSVD_UPLINK + 1,
3124 			     vport_end_ida, GFP_KERNEL);
3125 	if (id < 0)
3126 		return 0;
3127 	id = (pf_num << ESW_VPORT_BITS) | id;
3128 	return id;
3129 }
3130 
mlx5_esw_match_metadata_free(struct mlx5_eswitch * esw,u32 metadata)3131 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
3132 {
3133 	u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
3134 
3135 	/* Metadata contains only 12 bits of actual ida id */
3136 	ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
3137 }
3138 
esw_offloads_vport_metadata_setup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3139 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
3140 					     struct mlx5_vport *vport)
3141 {
3142 	if (vport->vport == MLX5_VPORT_UPLINK)
3143 		vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
3144 	else
3145 		vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
3146 
3147 	vport->metadata = vport->default_metadata;
3148 	return vport->metadata ? 0 : -ENOSPC;
3149 }
3150 
esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3151 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
3152 						struct mlx5_vport *vport)
3153 {
3154 	if (!vport->default_metadata)
3155 		return;
3156 
3157 	if (vport->vport == MLX5_VPORT_UPLINK)
3158 		return;
3159 
3160 	WARN_ON(vport->metadata != vport->default_metadata);
3161 	mlx5_esw_match_metadata_free(esw, vport->default_metadata);
3162 }
3163 
esw_offloads_metadata_uninit(struct mlx5_eswitch * esw)3164 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
3165 {
3166 	struct mlx5_vport *vport;
3167 	unsigned long i;
3168 
3169 	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3170 		return;
3171 
3172 	mlx5_esw_for_each_vport(esw, i, vport)
3173 		esw_offloads_vport_metadata_cleanup(esw, vport);
3174 }
3175 
esw_offloads_metadata_init(struct mlx5_eswitch * esw)3176 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3177 {
3178 	struct mlx5_vport *vport;
3179 	unsigned long i;
3180 	int err;
3181 
3182 	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3183 		return 0;
3184 
3185 	mlx5_esw_for_each_vport(esw, i, vport) {
3186 		err = esw_offloads_vport_metadata_setup(esw, vport);
3187 		if (err)
3188 			goto metadata_err;
3189 	}
3190 
3191 	return 0;
3192 
3193 metadata_err:
3194 	esw_offloads_metadata_uninit(esw);
3195 	return err;
3196 }
3197 
3198 int
esw_vport_create_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3199 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3200 				     struct mlx5_vport *vport)
3201 {
3202 	int err;
3203 
3204 	err = esw_acl_ingress_ofld_setup(esw, vport);
3205 	if (err)
3206 		return err;
3207 
3208 	err = esw_acl_egress_ofld_setup(esw, vport);
3209 	if (err)
3210 		goto egress_err;
3211 
3212 	return 0;
3213 
3214 egress_err:
3215 	esw_acl_ingress_ofld_cleanup(esw, vport);
3216 	return err;
3217 }
3218 
3219 void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3220 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3221 				      struct mlx5_vport *vport)
3222 {
3223 	esw_acl_egress_ofld_cleanup(vport);
3224 	esw_acl_ingress_ofld_cleanup(esw, vport);
3225 }
3226 
esw_create_offloads_acl_tables(struct mlx5_eswitch * esw)3227 static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw)
3228 {
3229 	struct mlx5_vport *uplink, *manager;
3230 	int ret;
3231 
3232 	uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3233 	if (IS_ERR(uplink))
3234 		return PTR_ERR(uplink);
3235 
3236 	ret = esw_vport_create_offloads_acl_tables(esw, uplink);
3237 	if (ret)
3238 		return ret;
3239 
3240 	manager = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3241 	if (IS_ERR(manager)) {
3242 		ret = PTR_ERR(manager);
3243 		goto err_manager;
3244 	}
3245 
3246 	ret = esw_vport_create_offloads_acl_tables(esw, manager);
3247 	if (ret)
3248 		goto err_manager;
3249 
3250 	return 0;
3251 
3252 err_manager:
3253 	esw_vport_destroy_offloads_acl_tables(esw, uplink);
3254 	return ret;
3255 }
3256 
esw_destroy_offloads_acl_tables(struct mlx5_eswitch * esw)3257 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw)
3258 {
3259 	struct mlx5_vport *vport;
3260 
3261 	vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3262 	if (!IS_ERR(vport))
3263 		esw_vport_destroy_offloads_acl_tables(esw, vport);
3264 
3265 	vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3266 	if (!IS_ERR(vport))
3267 		esw_vport_destroy_offloads_acl_tables(esw, vport);
3268 }
3269 
mlx5_eswitch_reload_reps(struct mlx5_eswitch * esw)3270 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
3271 {
3272 	struct mlx5_eswitch_rep *rep;
3273 	unsigned long i;
3274 	int ret;
3275 
3276 	if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3277 		return 0;
3278 
3279 	rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3280 	if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
3281 		return 0;
3282 
3283 	ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
3284 	if (ret)
3285 		return ret;
3286 
3287 	mlx5_esw_for_each_rep(esw, i, rep) {
3288 		if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
3289 			mlx5_esw_offloads_rep_load(esw, rep->vport);
3290 	}
3291 
3292 	return 0;
3293 }
3294 
esw_offloads_steering_init(struct mlx5_eswitch * esw)3295 static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3296 {
3297 	struct mlx5_esw_indir_table *indir;
3298 	int err;
3299 
3300 	memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3301 	mutex_init(&esw->fdb_table.offloads.vports.lock);
3302 	hash_init(esw->fdb_table.offloads.vports.table);
3303 	atomic64_set(&esw->user_count, 0);
3304 
3305 	indir = mlx5_esw_indir_table_init();
3306 	if (IS_ERR(indir)) {
3307 		err = PTR_ERR(indir);
3308 		goto create_indir_err;
3309 	}
3310 	esw->fdb_table.offloads.indir = indir;
3311 
3312 	err = esw_create_offloads_acl_tables(esw);
3313 	if (err)
3314 		goto create_acl_err;
3315 
3316 	err = esw_create_offloads_table(esw);
3317 	if (err)
3318 		goto create_offloads_err;
3319 
3320 	err = esw_create_restore_table(esw);
3321 	if (err)
3322 		goto create_restore_err;
3323 
3324 	err = esw_create_offloads_fdb_tables(esw);
3325 	if (err)
3326 		goto create_fdb_err;
3327 
3328 	err = esw_create_vport_rx_group(esw);
3329 	if (err)
3330 		goto create_fg_err;
3331 
3332 	err = esw_create_vport_rx_drop_group(esw);
3333 	if (err)
3334 		goto create_rx_drop_fg_err;
3335 
3336 	err = esw_create_vport_rx_drop_rule(esw);
3337 	if (err)
3338 		goto create_rx_drop_rule_err;
3339 
3340 	return 0;
3341 
3342 create_rx_drop_rule_err:
3343 	esw_destroy_vport_rx_drop_group(esw);
3344 create_rx_drop_fg_err:
3345 	esw_destroy_vport_rx_group(esw);
3346 create_fg_err:
3347 	esw_destroy_offloads_fdb_tables(esw);
3348 create_fdb_err:
3349 	esw_destroy_restore_table(esw);
3350 create_restore_err:
3351 	esw_destroy_offloads_table(esw);
3352 create_offloads_err:
3353 	esw_destroy_offloads_acl_tables(esw);
3354 create_acl_err:
3355 	mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3356 create_indir_err:
3357 	mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3358 	return err;
3359 }
3360 
esw_offloads_steering_cleanup(struct mlx5_eswitch * esw)3361 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3362 {
3363 	esw_destroy_vport_rx_drop_rule(esw);
3364 	esw_destroy_vport_rx_drop_group(esw);
3365 	esw_destroy_vport_rx_group(esw);
3366 	esw_destroy_offloads_fdb_tables(esw);
3367 	esw_destroy_restore_table(esw);
3368 	esw_destroy_offloads_table(esw);
3369 	esw_destroy_offloads_acl_tables(esw);
3370 	mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3371 	mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3372 }
3373 
3374 static void
esw_vfs_changed_event_handler(struct mlx5_eswitch * esw,const u32 * out)3375 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3376 {
3377 	struct devlink *devlink;
3378 	bool host_pf_disabled;
3379 	u16 new_num_vfs;
3380 
3381 	new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3382 			       host_params_context.host_num_of_vfs);
3383 	host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3384 				    host_params_context.host_pf_disabled);
3385 
3386 	if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3387 		return;
3388 
3389 	devlink = priv_to_devlink(esw->dev);
3390 	devl_lock(devlink);
3391 	/* Number of VFs can only change from "0 to x" or "x to 0". */
3392 	if (esw->esw_funcs.num_vfs > 0) {
3393 		mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
3394 	} else {
3395 		int err;
3396 
3397 		err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3398 						  MLX5_VPORT_UC_ADDR_CHANGE);
3399 		if (err) {
3400 			devl_unlock(devlink);
3401 			return;
3402 		}
3403 	}
3404 	esw->esw_funcs.num_vfs = new_num_vfs;
3405 	devl_unlock(devlink);
3406 }
3407 
esw_functions_changed_event_handler(struct work_struct * work)3408 static void esw_functions_changed_event_handler(struct work_struct *work)
3409 {
3410 	struct mlx5_host_work *host_work;
3411 	struct mlx5_eswitch *esw;
3412 	const u32 *out;
3413 
3414 	host_work = container_of(work, struct mlx5_host_work, work);
3415 	esw = host_work->esw;
3416 
3417 	out = mlx5_esw_query_functions(esw->dev);
3418 	if (IS_ERR(out))
3419 		goto out;
3420 
3421 	esw_vfs_changed_event_handler(esw, out);
3422 	kvfree(out);
3423 out:
3424 	kfree(host_work);
3425 }
3426 
mlx5_esw_funcs_changed_handler(struct notifier_block * nb,unsigned long type,void * data)3427 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3428 {
3429 	struct mlx5_esw_functions *esw_funcs;
3430 	struct mlx5_host_work *host_work;
3431 	struct mlx5_eswitch *esw;
3432 
3433 	host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3434 	if (!host_work)
3435 		return NOTIFY_DONE;
3436 
3437 	esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3438 	esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3439 
3440 	host_work->esw = esw;
3441 
3442 	INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3443 	queue_work(esw->work_queue, &host_work->work);
3444 
3445 	return NOTIFY_OK;
3446 }
3447 
mlx5_esw_host_number_init(struct mlx5_eswitch * esw)3448 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3449 {
3450 	const u32 *query_host_out;
3451 
3452 	if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3453 		return 0;
3454 
3455 	query_host_out = mlx5_esw_query_functions(esw->dev);
3456 	if (IS_ERR(query_host_out))
3457 		return PTR_ERR(query_host_out);
3458 
3459 	/* Mark non local controller with non zero controller number. */
3460 	esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3461 					     host_params_context.host_number);
3462 	kvfree(query_host_out);
3463 	return 0;
3464 }
3465 
mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch * esw,u32 controller)3466 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3467 {
3468 	/* Local controller is always valid */
3469 	if (controller == 0)
3470 		return true;
3471 
3472 	if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3473 		return false;
3474 
3475 	/* External host number starts with zero in device */
3476 	return (controller == esw->offloads.host_number + 1);
3477 }
3478 
esw_offloads_enable(struct mlx5_eswitch * esw)3479 int esw_offloads_enable(struct mlx5_eswitch *esw)
3480 {
3481 	struct mapping_ctx *reg_c0_obj_pool;
3482 	struct mlx5_vport *vport;
3483 	unsigned long i;
3484 	u64 mapping_id;
3485 	int err;
3486 
3487 	mutex_init(&esw->offloads.termtbl_mutex);
3488 	mlx5_rdma_enable_roce(esw->dev);
3489 
3490 	err = mlx5_esw_host_number_init(esw);
3491 	if (err)
3492 		goto err_metadata;
3493 
3494 	err = esw_offloads_metadata_init(esw);
3495 	if (err)
3496 		goto err_metadata;
3497 
3498 	err = esw_set_passing_vport_metadata(esw, true);
3499 	if (err)
3500 		goto err_vport_metadata;
3501 
3502 	mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
3503 
3504 	reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN,
3505 						sizeof(struct mlx5_mapped_obj),
3506 						ESW_REG_C0_USER_DATA_METADATA_MASK,
3507 						true);
3508 
3509 	if (IS_ERR(reg_c0_obj_pool)) {
3510 		err = PTR_ERR(reg_c0_obj_pool);
3511 		goto err_pool;
3512 	}
3513 	esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3514 
3515 	err = esw_offloads_steering_init(esw);
3516 	if (err)
3517 		goto err_steering_init;
3518 
3519 	/* Representor will control the vport link state */
3520 	mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3521 		vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3522 	if (mlx5_core_ec_sriov_enabled(esw->dev))
3523 		mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs)
3524 			vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3525 
3526 	/* Uplink vport rep must load first. */
3527 	err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
3528 	if (err)
3529 		goto err_uplink;
3530 
3531 	err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
3532 	if (err)
3533 		goto err_vports;
3534 
3535 	return 0;
3536 
3537 err_vports:
3538 	mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3539 err_uplink:
3540 	esw_offloads_steering_cleanup(esw);
3541 err_steering_init:
3542 	mapping_destroy(reg_c0_obj_pool);
3543 err_pool:
3544 	esw_set_passing_vport_metadata(esw, false);
3545 err_vport_metadata:
3546 	esw_offloads_metadata_uninit(esw);
3547 err_metadata:
3548 	mlx5_rdma_disable_roce(esw->dev);
3549 	mutex_destroy(&esw->offloads.termtbl_mutex);
3550 	return err;
3551 }
3552 
esw_offloads_stop(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)3553 static int esw_offloads_stop(struct mlx5_eswitch *esw,
3554 			     struct netlink_ext_ack *extack)
3555 {
3556 	int err;
3557 
3558 	esw->mode = MLX5_ESWITCH_LEGACY;
3559 
3560 	/* If changing from switchdev to legacy mode without sriov enabled,
3561 	 * no need to create legacy fdb.
3562 	 */
3563 	if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev))
3564 		return 0;
3565 
3566 	err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3567 	if (err)
3568 		NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3569 
3570 	return err;
3571 }
3572 
esw_offloads_disable(struct mlx5_eswitch * esw)3573 void esw_offloads_disable(struct mlx5_eswitch *esw)
3574 {
3575 	mlx5_eswitch_disable_pf_vf_vports(esw);
3576 	mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3577 	esw_set_passing_vport_metadata(esw, false);
3578 	esw_offloads_steering_cleanup(esw);
3579 	mapping_destroy(esw->offloads.reg_c0_obj_pool);
3580 	esw_offloads_metadata_uninit(esw);
3581 	mlx5_rdma_disable_roce(esw->dev);
3582 	mutex_destroy(&esw->offloads.termtbl_mutex);
3583 }
3584 
esw_mode_from_devlink(u16 mode,u16 * mlx5_mode)3585 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3586 {
3587 	switch (mode) {
3588 	case DEVLINK_ESWITCH_MODE_LEGACY:
3589 		*mlx5_mode = MLX5_ESWITCH_LEGACY;
3590 		break;
3591 	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3592 		*mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3593 		break;
3594 	default:
3595 		return -EINVAL;
3596 	}
3597 
3598 	return 0;
3599 }
3600 
esw_mode_to_devlink(u16 mlx5_mode,u16 * mode)3601 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3602 {
3603 	switch (mlx5_mode) {
3604 	case MLX5_ESWITCH_LEGACY:
3605 		*mode = DEVLINK_ESWITCH_MODE_LEGACY;
3606 		break;
3607 	case MLX5_ESWITCH_OFFLOADS:
3608 		*mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3609 		break;
3610 	default:
3611 		return -EINVAL;
3612 	}
3613 
3614 	return 0;
3615 }
3616 
esw_inline_mode_from_devlink(u8 mode,u8 * mlx5_mode)3617 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3618 {
3619 	switch (mode) {
3620 	case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3621 		*mlx5_mode = MLX5_INLINE_MODE_NONE;
3622 		break;
3623 	case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3624 		*mlx5_mode = MLX5_INLINE_MODE_L2;
3625 		break;
3626 	case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3627 		*mlx5_mode = MLX5_INLINE_MODE_IP;
3628 		break;
3629 	case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3630 		*mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3631 		break;
3632 	default:
3633 		return -EINVAL;
3634 	}
3635 
3636 	return 0;
3637 }
3638 
esw_inline_mode_to_devlink(u8 mlx5_mode,u8 * mode)3639 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3640 {
3641 	switch (mlx5_mode) {
3642 	case MLX5_INLINE_MODE_NONE:
3643 		*mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3644 		break;
3645 	case MLX5_INLINE_MODE_L2:
3646 		*mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3647 		break;
3648 	case MLX5_INLINE_MODE_IP:
3649 		*mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3650 		break;
3651 	case MLX5_INLINE_MODE_TCP_UDP:
3652 		*mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3653 		break;
3654 	default:
3655 		return -EINVAL;
3656 	}
3657 
3658 	return 0;
3659 }
3660 
esw_offloads_devlink_ns_eq_netdev_ns(struct devlink * devlink)3661 static bool esw_offloads_devlink_ns_eq_netdev_ns(struct devlink *devlink)
3662 {
3663 	struct mlx5_core_dev *dev = devlink_priv(devlink);
3664 	struct net *devl_net, *netdev_net;
3665 	bool ret = false;
3666 
3667 	mutex_lock(&dev->mlx5e_res.uplink_netdev_lock);
3668 	if (dev->mlx5e_res.uplink_netdev) {
3669 		netdev_net = dev_net(dev->mlx5e_res.uplink_netdev);
3670 		devl_net = devlink_net(devlink);
3671 		ret = net_eq(devl_net, netdev_net);
3672 	}
3673 	mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock);
3674 	return ret;
3675 }
3676 
mlx5_eswitch_block_mode(struct mlx5_core_dev * dev)3677 int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev)
3678 {
3679 	struct mlx5_eswitch *esw = dev->priv.eswitch;
3680 	int err;
3681 
3682 	if (!mlx5_esw_allowed(esw))
3683 		return 0;
3684 
3685 	/* Take TC into account */
3686 	err = mlx5_esw_try_lock(esw);
3687 	if (err < 0)
3688 		return err;
3689 
3690 	esw->offloads.num_block_mode++;
3691 	mlx5_esw_unlock(esw);
3692 	return 0;
3693 }
3694 
mlx5_eswitch_unblock_mode(struct mlx5_core_dev * dev)3695 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev)
3696 {
3697 	struct mlx5_eswitch *esw = dev->priv.eswitch;
3698 
3699 	if (!mlx5_esw_allowed(esw))
3700 		return;
3701 
3702 	down_write(&esw->mode_lock);
3703 	esw->offloads.num_block_mode--;
3704 	up_write(&esw->mode_lock);
3705 }
3706 
mlx5_devlink_eswitch_mode_set(struct devlink * devlink,u16 mode,struct netlink_ext_ack * extack)3707 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3708 				  struct netlink_ext_ack *extack)
3709 {
3710 	u16 cur_mlx5_mode, mlx5_mode = 0;
3711 	struct mlx5_eswitch *esw;
3712 	int err = 0;
3713 
3714 	esw = mlx5_devlink_eswitch_get(devlink);
3715 	if (IS_ERR(esw))
3716 		return PTR_ERR(esw);
3717 
3718 	if (esw_mode_from_devlink(mode, &mlx5_mode))
3719 		return -EINVAL;
3720 
3721 	if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV &&
3722 	    !esw_offloads_devlink_ns_eq_netdev_ns(devlink)) {
3723 		NL_SET_ERR_MSG_MOD(extack,
3724 				   "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's.");
3725 		return -EPERM;
3726 	}
3727 
3728 	mlx5_lag_disable_change(esw->dev);
3729 	err = mlx5_esw_try_lock(esw);
3730 	if (err < 0) {
3731 		NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
3732 		goto enable_lag;
3733 	}
3734 	cur_mlx5_mode = err;
3735 	err = 0;
3736 
3737 	if (cur_mlx5_mode == mlx5_mode)
3738 		goto unlock;
3739 
3740 	if (esw->offloads.num_block_mode) {
3741 		NL_SET_ERR_MSG_MOD(extack,
3742 				   "Can't change eswitch mode when IPsec SA and/or policies are configured");
3743 		err = -EOPNOTSUPP;
3744 		goto unlock;
3745 	}
3746 
3747 	esw->eswitch_operation_in_progress = true;
3748 	up_write(&esw->mode_lock);
3749 
3750 	mlx5_eswitch_disable_locked(esw);
3751 	if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3752 		if (mlx5_devlink_trap_get_num_active(esw->dev)) {
3753 			NL_SET_ERR_MSG_MOD(extack,
3754 					   "Can't change mode while devlink traps are active");
3755 			err = -EOPNOTSUPP;
3756 			goto skip;
3757 		}
3758 		err = esw_offloads_start(esw, extack);
3759 	} else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
3760 		err = esw_offloads_stop(esw, extack);
3761 		mlx5_rescan_drivers(esw->dev);
3762 	} else {
3763 		err = -EINVAL;
3764 	}
3765 
3766 skip:
3767 	down_write(&esw->mode_lock);
3768 	esw->eswitch_operation_in_progress = false;
3769 unlock:
3770 	mlx5_esw_unlock(esw);
3771 enable_lag:
3772 	mlx5_lag_enable_change(esw->dev);
3773 	return err;
3774 }
3775 
mlx5_devlink_eswitch_mode_get(struct devlink * devlink,u16 * mode)3776 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3777 {
3778 	struct mlx5_eswitch *esw;
3779 
3780 	esw = mlx5_devlink_eswitch_get(devlink);
3781 	if (IS_ERR(esw))
3782 		return PTR_ERR(esw);
3783 
3784 	return esw_mode_to_devlink(esw->mode, mode);
3785 }
3786 
mlx5_esw_vports_inline_set(struct mlx5_eswitch * esw,u8 mlx5_mode,struct netlink_ext_ack * extack)3787 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3788 				      struct netlink_ext_ack *extack)
3789 {
3790 	struct mlx5_core_dev *dev = esw->dev;
3791 	struct mlx5_vport *vport;
3792 	u16 err_vport_num = 0;
3793 	unsigned long i;
3794 	int err = 0;
3795 
3796 	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3797 		err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3798 		if (err) {
3799 			err_vport_num = vport->vport;
3800 			NL_SET_ERR_MSG_MOD(extack,
3801 					   "Failed to set min inline on vport");
3802 			goto revert_inline_mode;
3803 		}
3804 	}
3805 	if (mlx5_core_ec_sriov_enabled(esw->dev)) {
3806 		mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3807 			err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3808 			if (err) {
3809 				err_vport_num = vport->vport;
3810 				NL_SET_ERR_MSG_MOD(extack,
3811 						   "Failed to set min inline on vport");
3812 				goto revert_ec_vf_inline_mode;
3813 			}
3814 		}
3815 	}
3816 	return 0;
3817 
3818 revert_ec_vf_inline_mode:
3819 	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3820 		if (vport->vport == err_vport_num)
3821 			break;
3822 		mlx5_modify_nic_vport_min_inline(dev,
3823 						 vport->vport,
3824 						 esw->offloads.inline_mode);
3825 	}
3826 revert_inline_mode:
3827 	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3828 		if (vport->vport == err_vport_num)
3829 			break;
3830 		mlx5_modify_nic_vport_min_inline(dev,
3831 						 vport->vport,
3832 						 esw->offloads.inline_mode);
3833 	}
3834 	return err;
3835 }
3836 
mlx5_devlink_eswitch_inline_mode_set(struct devlink * devlink,u8 mode,struct netlink_ext_ack * extack)3837 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3838 					 struct netlink_ext_ack *extack)
3839 {
3840 	struct mlx5_core_dev *dev = devlink_priv(devlink);
3841 	struct mlx5_eswitch *esw;
3842 	u8 mlx5_mode;
3843 	int err;
3844 
3845 	esw = mlx5_devlink_eswitch_get(devlink);
3846 	if (IS_ERR(esw))
3847 		return PTR_ERR(esw);
3848 
3849 	down_write(&esw->mode_lock);
3850 
3851 	switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3852 	case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3853 		if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3854 			err = 0;
3855 			goto out;
3856 		}
3857 
3858 		fallthrough;
3859 	case MLX5_CAP_INLINE_MODE_L2:
3860 		NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
3861 		err = -EOPNOTSUPP;
3862 		goto out;
3863 	case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3864 		break;
3865 	}
3866 
3867 	if (atomic64_read(&esw->offloads.num_flows) > 0) {
3868 		NL_SET_ERR_MSG_MOD(extack,
3869 				   "Can't set inline mode when flows are configured");
3870 		err = -EOPNOTSUPP;
3871 		goto out;
3872 	}
3873 
3874 	err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
3875 	if (err)
3876 		goto out;
3877 
3878 	esw->eswitch_operation_in_progress = true;
3879 	up_write(&esw->mode_lock);
3880 
3881 	err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3882 	if (!err)
3883 		esw->offloads.inline_mode = mlx5_mode;
3884 
3885 	down_write(&esw->mode_lock);
3886 	esw->eswitch_operation_in_progress = false;
3887 	up_write(&esw->mode_lock);
3888 	return 0;
3889 
3890 out:
3891 	up_write(&esw->mode_lock);
3892 	return err;
3893 }
3894 
mlx5_devlink_eswitch_inline_mode_get(struct devlink * devlink,u8 * mode)3895 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3896 {
3897 	struct mlx5_eswitch *esw;
3898 
3899 	esw = mlx5_devlink_eswitch_get(devlink);
3900 	if (IS_ERR(esw))
3901 		return PTR_ERR(esw);
3902 
3903 	return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3904 }
3905 
mlx5_eswitch_block_encap(struct mlx5_core_dev * dev)3906 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev)
3907 {
3908 	struct mlx5_eswitch *esw = dev->priv.eswitch;
3909 
3910 	if (!mlx5_esw_allowed(esw))
3911 		return true;
3912 
3913 	down_write(&esw->mode_lock);
3914 	if (esw->mode != MLX5_ESWITCH_LEGACY &&
3915 	    esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
3916 		up_write(&esw->mode_lock);
3917 		return false;
3918 	}
3919 
3920 	esw->offloads.num_block_encap++;
3921 	up_write(&esw->mode_lock);
3922 	return true;
3923 }
3924 
mlx5_eswitch_unblock_encap(struct mlx5_core_dev * dev)3925 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
3926 {
3927 	struct mlx5_eswitch *esw = dev->priv.eswitch;
3928 
3929 	if (!mlx5_esw_allowed(esw))
3930 		return;
3931 
3932 	down_write(&esw->mode_lock);
3933 	esw->offloads.num_block_encap--;
3934 	up_write(&esw->mode_lock);
3935 }
3936 
mlx5_devlink_eswitch_encap_mode_set(struct devlink * devlink,enum devlink_eswitch_encap_mode encap,struct netlink_ext_ack * extack)3937 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3938 					enum devlink_eswitch_encap_mode encap,
3939 					struct netlink_ext_ack *extack)
3940 {
3941 	struct mlx5_core_dev *dev = devlink_priv(devlink);
3942 	struct mlx5_eswitch *esw;
3943 	int err = 0;
3944 
3945 	esw = mlx5_devlink_eswitch_get(devlink);
3946 	if (IS_ERR(esw))
3947 		return PTR_ERR(esw);
3948 
3949 	down_write(&esw->mode_lock);
3950 
3951 	if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
3952 	    (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
3953 	     !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3954 		err = -EOPNOTSUPP;
3955 		goto unlock;
3956 	}
3957 
3958 	if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3959 		err = -EOPNOTSUPP;
3960 		goto unlock;
3961 	}
3962 
3963 	if (esw->mode == MLX5_ESWITCH_LEGACY) {
3964 		esw->offloads.encap = encap;
3965 		goto unlock;
3966 	}
3967 
3968 	if (esw->offloads.encap == encap)
3969 		goto unlock;
3970 
3971 	if (atomic64_read(&esw->offloads.num_flows) > 0) {
3972 		NL_SET_ERR_MSG_MOD(extack,
3973 				   "Can't set encapsulation when flows are configured");
3974 		err = -EOPNOTSUPP;
3975 		goto unlock;
3976 	}
3977 
3978 	if (esw->offloads.num_block_encap) {
3979 		NL_SET_ERR_MSG_MOD(extack,
3980 				   "Can't set encapsulation when IPsec SA and/or policies are configured");
3981 		err = -EOPNOTSUPP;
3982 		goto unlock;
3983 	}
3984 
3985 	esw->eswitch_operation_in_progress = true;
3986 	up_write(&esw->mode_lock);
3987 
3988 	esw_destroy_offloads_fdb_tables(esw);
3989 
3990 	esw->offloads.encap = encap;
3991 
3992 	err = esw_create_offloads_fdb_tables(esw);
3993 
3994 	if (err) {
3995 		NL_SET_ERR_MSG_MOD(extack,
3996 				   "Failed re-creating fast FDB table");
3997 		esw->offloads.encap = !encap;
3998 		(void)esw_create_offloads_fdb_tables(esw);
3999 	}
4000 
4001 	down_write(&esw->mode_lock);
4002 	esw->eswitch_operation_in_progress = false;
4003 
4004 unlock:
4005 	up_write(&esw->mode_lock);
4006 	return err;
4007 }
4008 
mlx5_devlink_eswitch_encap_mode_get(struct devlink * devlink,enum devlink_eswitch_encap_mode * encap)4009 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
4010 					enum devlink_eswitch_encap_mode *encap)
4011 {
4012 	struct mlx5_eswitch *esw;
4013 
4014 	esw = mlx5_devlink_eswitch_get(devlink);
4015 	if (IS_ERR(esw))
4016 		return PTR_ERR(esw);
4017 
4018 	*encap = esw->offloads.encap;
4019 	return 0;
4020 }
4021 
4022 static bool
mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch * esw,u16 vport_num)4023 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
4024 {
4025 	/* Currently, only ECPF based device has representor for host PF. */
4026 	if (vport_num == MLX5_VPORT_PF &&
4027 	    !mlx5_core_is_ecpf_esw_manager(esw->dev))
4028 		return false;
4029 
4030 	if (vport_num == MLX5_VPORT_ECPF &&
4031 	    !mlx5_ecpf_vport_exists(esw->dev))
4032 		return false;
4033 
4034 	return true;
4035 }
4036 
mlx5_eswitch_register_vport_reps(struct mlx5_eswitch * esw,const struct mlx5_eswitch_rep_ops * ops,u8 rep_type)4037 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
4038 				      const struct mlx5_eswitch_rep_ops *ops,
4039 				      u8 rep_type)
4040 {
4041 	struct mlx5_eswitch_rep_data *rep_data;
4042 	struct mlx5_eswitch_rep *rep;
4043 	unsigned long i;
4044 
4045 	esw->offloads.rep_ops[rep_type] = ops;
4046 	mlx5_esw_for_each_rep(esw, i, rep) {
4047 		if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
4048 			rep->esw = esw;
4049 			rep_data = &rep->rep_data[rep_type];
4050 			atomic_set(&rep_data->state, REP_REGISTERED);
4051 		}
4052 	}
4053 }
4054 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
4055 
mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch * esw,u8 rep_type)4056 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
4057 {
4058 	struct mlx5_eswitch_rep *rep;
4059 	unsigned long i;
4060 
4061 	if (esw->mode == MLX5_ESWITCH_OFFLOADS)
4062 		__unload_reps_all_vport(esw, rep_type);
4063 
4064 	mlx5_esw_for_each_rep(esw, i, rep)
4065 		atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
4066 }
4067 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
4068 
mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch * esw,u8 rep_type)4069 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
4070 {
4071 	struct mlx5_eswitch_rep *rep;
4072 
4073 	rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
4074 	return rep->rep_data[rep_type].priv;
4075 }
4076 
mlx5_eswitch_get_proto_dev(struct mlx5_eswitch * esw,u16 vport,u8 rep_type)4077 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
4078 				 u16 vport,
4079 				 u8 rep_type)
4080 {
4081 	struct mlx5_eswitch_rep *rep;
4082 
4083 	rep = mlx5_eswitch_get_rep(esw, vport);
4084 
4085 	if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
4086 	    esw->offloads.rep_ops[rep_type]->get_proto_dev)
4087 		return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
4088 	return NULL;
4089 }
4090 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
4091 
mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch * esw,u8 rep_type)4092 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
4093 {
4094 	return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
4095 }
4096 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
4097 
mlx5_eswitch_vport_rep(struct mlx5_eswitch * esw,u16 vport)4098 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
4099 						u16 vport)
4100 {
4101 	return mlx5_eswitch_get_rep(esw, vport);
4102 }
4103 EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
4104 
mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch * esw)4105 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
4106 {
4107 	return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
4108 }
4109 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
4110 
mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch * esw)4111 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
4112 {
4113 	return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
4114 }
4115 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
4116 
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch * esw,u16 vport_num)4117 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
4118 					      u16 vport_num)
4119 {
4120 	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4121 
4122 	if (WARN_ON_ONCE(IS_ERR(vport)))
4123 		return 0;
4124 
4125 	return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
4126 }
4127 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
4128 
mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch * esw,u16 vport_num,u16 * vhca_id)4129 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
4130 {
4131 	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4132 	void *query_ctx;
4133 	void *hca_caps;
4134 	int err;
4135 
4136 	*vhca_id = 0;
4137 
4138 	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4139 	if (!query_ctx)
4140 		return -ENOMEM;
4141 
4142 	err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx);
4143 	if (err)
4144 		goto out_free;
4145 
4146 	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4147 	*vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
4148 
4149 out_free:
4150 	kfree(query_ctx);
4151 	return err;
4152 }
4153 
mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch * esw,u16 vport_num)4154 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
4155 {
4156 	u16 *old_entry, *vhca_map_entry, vhca_id;
4157 	int err;
4158 
4159 	err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
4160 	if (err) {
4161 		esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
4162 			 vport_num, err);
4163 		return err;
4164 	}
4165 
4166 	vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
4167 	if (!vhca_map_entry)
4168 		return -ENOMEM;
4169 
4170 	*vhca_map_entry = vport_num;
4171 	old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
4172 	if (xa_is_err(old_entry)) {
4173 		kfree(vhca_map_entry);
4174 		return xa_err(old_entry);
4175 	}
4176 	kfree(old_entry);
4177 	return 0;
4178 }
4179 
mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch * esw,u16 vport_num)4180 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
4181 {
4182 	u16 *vhca_map_entry, vhca_id;
4183 	int err;
4184 
4185 	err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
4186 	if (err)
4187 		esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
4188 			 vport_num, err);
4189 
4190 	vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
4191 	kfree(vhca_map_entry);
4192 }
4193 
mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch * esw,u16 vhca_id,u16 * vport_num)4194 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
4195 {
4196 	u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
4197 
4198 	if (!res)
4199 		return -ENOENT;
4200 
4201 	*vport_num = *res;
4202 	return 0;
4203 }
4204 
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch * esw,u16 vport_num)4205 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
4206 					    u16 vport_num)
4207 {
4208 	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4209 
4210 	if (WARN_ON_ONCE(IS_ERR(vport)))
4211 		return 0;
4212 
4213 	return vport->metadata;
4214 }
4215 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
4216 
mlx5_devlink_port_fn_hw_addr_get(struct devlink_port * port,u8 * hw_addr,int * hw_addr_len,struct netlink_ext_ack * extack)4217 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port,
4218 				     u8 *hw_addr, int *hw_addr_len,
4219 				     struct netlink_ext_ack *extack)
4220 {
4221 	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4222 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4223 
4224 	mutex_lock(&esw->state_lock);
4225 	ether_addr_copy(hw_addr, vport->info.mac);
4226 	*hw_addr_len = ETH_ALEN;
4227 	mutex_unlock(&esw->state_lock);
4228 	return 0;
4229 }
4230 
mlx5_devlink_port_fn_hw_addr_set(struct devlink_port * port,const u8 * hw_addr,int hw_addr_len,struct netlink_ext_ack * extack)4231 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port,
4232 				     const u8 *hw_addr, int hw_addr_len,
4233 				     struct netlink_ext_ack *extack)
4234 {
4235 	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4236 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4237 
4238 	return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr);
4239 }
4240 
mlx5_devlink_port_fn_migratable_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4241 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled,
4242 					struct netlink_ext_ack *extack)
4243 {
4244 	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4245 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4246 
4247 	if (!MLX5_CAP_GEN(esw->dev, migration)) {
4248 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4249 		return -EOPNOTSUPP;
4250 	}
4251 
4252 	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4253 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4254 		return -EOPNOTSUPP;
4255 	}
4256 
4257 	mutex_lock(&esw->state_lock);
4258 	*is_enabled = vport->info.mig_enabled;
4259 	mutex_unlock(&esw->state_lock);
4260 	return 0;
4261 }
4262 
mlx5_devlink_port_fn_migratable_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4263 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable,
4264 					struct netlink_ext_ack *extack)
4265 {
4266 	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4267 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4268 	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4269 	void *query_ctx;
4270 	void *hca_caps;
4271 	int err;
4272 
4273 	if (!MLX5_CAP_GEN(esw->dev, migration)) {
4274 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4275 		return -EOPNOTSUPP;
4276 	}
4277 
4278 	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4279 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4280 		return -EOPNOTSUPP;
4281 	}
4282 
4283 	mutex_lock(&esw->state_lock);
4284 
4285 	if (vport->info.mig_enabled == enable) {
4286 		err = 0;
4287 		goto out;
4288 	}
4289 
4290 	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4291 	if (!query_ctx) {
4292 		err = -ENOMEM;
4293 		goto out;
4294 	}
4295 
4296 	err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx,
4297 					    MLX5_CAP_GENERAL_2);
4298 	if (err) {
4299 		NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4300 		goto out_free;
4301 	}
4302 
4303 	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4304 	MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable);
4305 
4306 	err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport,
4307 					    MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
4308 	if (err) {
4309 		NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap");
4310 		goto out_free;
4311 	}
4312 
4313 	vport->info.mig_enabled = enable;
4314 
4315 out_free:
4316 	kfree(query_ctx);
4317 out:
4318 	mutex_unlock(&esw->state_lock);
4319 	return err;
4320 }
4321 
mlx5_devlink_port_fn_roce_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4322 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled,
4323 				  struct netlink_ext_ack *extack)
4324 {
4325 	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4326 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4327 
4328 	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4329 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4330 		return -EOPNOTSUPP;
4331 	}
4332 
4333 	mutex_lock(&esw->state_lock);
4334 	*is_enabled = vport->info.roce_enabled;
4335 	mutex_unlock(&esw->state_lock);
4336 	return 0;
4337 }
4338 
mlx5_devlink_port_fn_roce_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4339 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable,
4340 				  struct netlink_ext_ack *extack)
4341 {
4342 	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4343 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4344 	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4345 	u16 vport_num = vport->vport;
4346 	void *query_ctx;
4347 	void *hca_caps;
4348 	int err;
4349 
4350 	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4351 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4352 		return -EOPNOTSUPP;
4353 	}
4354 
4355 	mutex_lock(&esw->state_lock);
4356 
4357 	if (vport->info.roce_enabled == enable) {
4358 		err = 0;
4359 		goto out;
4360 	}
4361 
4362 	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4363 	if (!query_ctx) {
4364 		err = -ENOMEM;
4365 		goto out;
4366 	}
4367 
4368 	err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
4369 					    MLX5_CAP_GENERAL);
4370 	if (err) {
4371 		NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4372 		goto out_free;
4373 	}
4374 
4375 	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4376 	MLX5_SET(cmd_hca_cap, hca_caps, roce, enable);
4377 
4378 	err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num,
4379 					    MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
4380 	if (err) {
4381 		NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap");
4382 		goto out_free;
4383 	}
4384 
4385 	vport->info.roce_enabled = enable;
4386 
4387 out_free:
4388 	kfree(query_ctx);
4389 out:
4390 	mutex_unlock(&esw->state_lock);
4391 	return err;
4392 }
4393 
4394 int
mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)4395 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule,
4396 				struct mlx5_esw_flow_attr *esw_attr, int attr_idx)
4397 {
4398 	struct mlx5_flow_destination new_dest = {};
4399 	struct mlx5_flow_destination old_dest = {};
4400 
4401 	if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
4402 		return 0;
4403 
4404 	esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4405 	esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4406 
4407 	return mlx5_modify_rule_destination(rule, &new_dest, &old_dest);
4408 }
4409 
4410 #ifdef CONFIG_XFRM_OFFLOAD
mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4411 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled,
4412 					  struct netlink_ext_ack *extack)
4413 {
4414 	struct mlx5_eswitch *esw;
4415 	struct mlx5_vport *vport;
4416 	int err = 0;
4417 
4418 	esw = mlx5_devlink_eswitch_get(port->devlink);
4419 	if (IS_ERR(esw))
4420 		return PTR_ERR(esw);
4421 
4422 	if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4423 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto");
4424 		return -EOPNOTSUPP;
4425 	}
4426 
4427 	vport = mlx5_devlink_port_vport_get(port);
4428 
4429 	mutex_lock(&esw->state_lock);
4430 	if (!vport->enabled) {
4431 		err = -EOPNOTSUPP;
4432 		goto unlock;
4433 	}
4434 
4435 	*is_enabled = vport->info.ipsec_crypto_enabled;
4436 unlock:
4437 	mutex_unlock(&esw->state_lock);
4438 	return err;
4439 }
4440 
mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4441 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable,
4442 					  struct netlink_ext_ack *extack)
4443 {
4444 	struct mlx5_eswitch *esw;
4445 	struct mlx5_vport *vport;
4446 	u16 vport_num;
4447 	int err;
4448 
4449 	esw = mlx5_devlink_eswitch_get(port->devlink);
4450 	if (IS_ERR(esw))
4451 		return PTR_ERR(esw);
4452 
4453 	vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4454 	err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num);
4455 	if (err) {
4456 		NL_SET_ERR_MSG_MOD(extack,
4457 				   "Device doesn't support IPsec crypto");
4458 		return err;
4459 	}
4460 
4461 	vport = mlx5_devlink_port_vport_get(port);
4462 
4463 	mutex_lock(&esw->state_lock);
4464 	if (!vport->enabled) {
4465 		err = -EOPNOTSUPP;
4466 		NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4467 		goto unlock;
4468 	}
4469 
4470 	if (vport->info.ipsec_crypto_enabled == enable)
4471 		goto unlock;
4472 
4473 	if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4474 		err = -EBUSY;
4475 		goto unlock;
4476 	}
4477 
4478 	err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable);
4479 	if (err) {
4480 		NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto");
4481 		goto unlock;
4482 	}
4483 
4484 	vport->info.ipsec_crypto_enabled = enable;
4485 	if (enable)
4486 		esw->enabled_ipsec_vf_count++;
4487 	else
4488 		esw->enabled_ipsec_vf_count--;
4489 unlock:
4490 	mutex_unlock(&esw->state_lock);
4491 	return err;
4492 }
4493 
mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4494 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled,
4495 					  struct netlink_ext_ack *extack)
4496 {
4497 	struct mlx5_eswitch *esw;
4498 	struct mlx5_vport *vport;
4499 	int err = 0;
4500 
4501 	esw = mlx5_devlink_eswitch_get(port->devlink);
4502 	if (IS_ERR(esw))
4503 		return PTR_ERR(esw);
4504 
4505 	if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4506 		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet");
4507 		return -EOPNOTSUPP;
4508 	}
4509 
4510 	vport = mlx5_devlink_port_vport_get(port);
4511 
4512 	mutex_lock(&esw->state_lock);
4513 	if (!vport->enabled) {
4514 		err = -EOPNOTSUPP;
4515 		goto unlock;
4516 	}
4517 
4518 	*is_enabled = vport->info.ipsec_packet_enabled;
4519 unlock:
4520 	mutex_unlock(&esw->state_lock);
4521 	return err;
4522 }
4523 
mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4524 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port,
4525 					  bool enable,
4526 					  struct netlink_ext_ack *extack)
4527 {
4528 	struct mlx5_eswitch *esw;
4529 	struct mlx5_vport *vport;
4530 	u16 vport_num;
4531 	int err;
4532 
4533 	esw = mlx5_devlink_eswitch_get(port->devlink);
4534 	if (IS_ERR(esw))
4535 		return PTR_ERR(esw);
4536 
4537 	vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4538 	err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num);
4539 	if (err) {
4540 		NL_SET_ERR_MSG_MOD(extack,
4541 				   "Device doesn't support IPsec packet mode");
4542 		return err;
4543 	}
4544 
4545 	vport = mlx5_devlink_port_vport_get(port);
4546 	mutex_lock(&esw->state_lock);
4547 	if (!vport->enabled) {
4548 		err = -EOPNOTSUPP;
4549 		NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4550 		goto unlock;
4551 	}
4552 
4553 	if (vport->info.ipsec_packet_enabled == enable)
4554 		goto unlock;
4555 
4556 	if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4557 		err = -EBUSY;
4558 		goto unlock;
4559 	}
4560 
4561 	err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable);
4562 	if (err) {
4563 		NL_SET_ERR_MSG_MOD(extack,
4564 				   "Failed to set IPsec packet mode");
4565 		goto unlock;
4566 	}
4567 
4568 	vport->info.ipsec_packet_enabled = enable;
4569 	if (enable)
4570 		esw->enabled_ipsec_vf_count++;
4571 	else
4572 		esw->enabled_ipsec_vf_count--;
4573 unlock:
4574 	mutex_unlock(&esw->state_lock);
4575 	return err;
4576 }
4577 #endif /* CONFIG_XFRM_OFFLOAD */
4578