1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2020 Mellanox Technologies.
3 
4 #include <linux/mlx5/driver.h>
5 #include <linux/mlx5/mlx5_ifc.h>
6 #include <linux/mlx5/fs.h>
7 
8 #include "lib/fs_chains.h"
9 #include "fs_ft_pool.h"
10 #include "en/mapping.h"
11 #include "fs_core.h"
12 #include "en_tc.h"
13 
14 #define chains_lock(chains) ((chains)->lock)
15 #define chains_ht(chains) ((chains)->chains_ht)
16 #define prios_ht(chains) ((chains)->prios_ht)
17 #define tc_default_ft(chains) ((chains)->tc_default_ft)
18 #define tc_end_ft(chains) ((chains)->tc_end_ft)
19 #define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \
20 				  FDB_TC_OFFLOAD : MLX5E_TC_PRIO)
21 #define FT_TBL_SZ (64 * 1024)
22 
23 struct mlx5_fs_chains {
24 	struct mlx5_core_dev *dev;
25 
26 	struct rhashtable chains_ht;
27 	struct rhashtable prios_ht;
28 	/* Protects above chains_ht and prios_ht */
29 	struct mutex lock;
30 
31 	struct mlx5_flow_table *tc_default_ft;
32 	struct mlx5_flow_table *tc_end_ft;
33 	struct mapping_ctx *chains_mapping;
34 
35 	enum mlx5_flow_namespace_type ns;
36 	u32 group_num;
37 	u32 flags;
38 };
39 
40 struct fs_chain {
41 	struct rhash_head node;
42 
43 	u32 chain;
44 
45 	int ref;
46 	int id;
47 
48 	struct mlx5_fs_chains *chains;
49 	struct list_head prios_list;
50 	struct mlx5_flow_handle *restore_rule;
51 	struct mlx5_modify_hdr *miss_modify_hdr;
52 };
53 
54 struct prio_key {
55 	u32 chain;
56 	u32 prio;
57 	u32 level;
58 };
59 
60 struct prio {
61 	struct rhash_head node;
62 	struct list_head list;
63 
64 	struct prio_key key;
65 
66 	int ref;
67 
68 	struct fs_chain *chain;
69 	struct mlx5_flow_table *ft;
70 	struct mlx5_flow_table *next_ft;
71 	struct mlx5_flow_group *miss_group;
72 	struct mlx5_flow_handle *miss_rule;
73 };
74 
75 static const struct rhashtable_params chain_params = {
76 	.head_offset = offsetof(struct fs_chain, node),
77 	.key_offset = offsetof(struct fs_chain, chain),
78 	.key_len = sizeof_field(struct fs_chain, chain),
79 	.automatic_shrinking = true,
80 };
81 
82 static const struct rhashtable_params prio_params = {
83 	.head_offset = offsetof(struct prio, node),
84 	.key_offset = offsetof(struct prio, key),
85 	.key_len = sizeof_field(struct prio, key),
86 	.automatic_shrinking = true,
87 };
88 
mlx5_chains_prios_supported(struct mlx5_fs_chains * chains)89 bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains)
90 {
91 	return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED;
92 }
93 
mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains * chains)94 bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains)
95 {
96 	return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
97 }
98 
mlx5_chains_backwards_supported(struct mlx5_fs_chains * chains)99 bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains)
100 {
101 	return mlx5_chains_prios_supported(chains) &&
102 	       mlx5_chains_ignore_flow_level_supported(chains);
103 }
104 
mlx5_chains_get_chain_range(struct mlx5_fs_chains * chains)105 u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains)
106 {
107 	if (!mlx5_chains_prios_supported(chains))
108 		return 1;
109 
110 	if (mlx5_chains_ignore_flow_level_supported(chains))
111 		return UINT_MAX - 1;
112 
113 	/* We should get here only for eswitch case */
114 	return FDB_TC_MAX_CHAIN;
115 }
116 
mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains * chains)117 u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains)
118 {
119 	return mlx5_chains_get_chain_range(chains) + 1;
120 }
121 
mlx5_chains_get_prio_range(struct mlx5_fs_chains * chains)122 u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
123 {
124 	if (mlx5_chains_ignore_flow_level_supported(chains))
125 		return UINT_MAX;
126 
127 	if (!chains->dev->priv.eswitch ||
128 	    chains->dev->priv.eswitch->mode != MLX5_ESWITCH_OFFLOADS)
129 		return 1;
130 
131 	/* We should get here only for eswitch case */
132 	return FDB_TC_MAX_PRIO;
133 }
134 
mlx5_chains_get_level_range(struct mlx5_fs_chains * chains)135 static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains)
136 {
137 	if (mlx5_chains_ignore_flow_level_supported(chains))
138 		return UINT_MAX;
139 
140 	/* Same value for FDB and NIC RX tables */
141 	return FDB_TC_LEVELS_PER_PRIO;
142 }
143 
144 void
mlx5_chains_set_end_ft(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)145 mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
146 		       struct mlx5_flow_table *ft)
147 {
148 	tc_end_ft(chains) = ft;
149 }
150 
151 static struct mlx5_flow_table *
mlx5_chains_create_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)152 mlx5_chains_create_table(struct mlx5_fs_chains *chains,
153 			 u32 chain, u32 prio, u32 level)
154 {
155 	struct mlx5_flow_table_attr ft_attr = {};
156 	struct mlx5_flow_namespace *ns;
157 	struct mlx5_flow_table *ft;
158 	int sz;
159 
160 	if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED)
161 		ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
162 				  MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
163 
164 	sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE;
165 	ft_attr.max_fte = sz;
166 
167 	/* We use tc_default_ft(chains) as the table's next_ft till
168 	 * ignore_flow_level is allowed on FT creation and not just for FTEs.
169 	 * Instead caller should add an explicit miss rule if needed.
170 	 */
171 	ft_attr.next_ft = tc_default_ft(chains);
172 
173 	/* The root table(chain 0, prio 1, level 0) is required to be
174 	 * connected to the previous fs_core managed prio.
175 	 * We always create it, as a managed table, in order to align with
176 	 * fs_core logic.
177 	 */
178 	if (!mlx5_chains_ignore_flow_level_supported(chains) ||
179 	    (chain == 0 && prio == 1 && level == 0)) {
180 		ft_attr.level = level;
181 		ft_attr.prio = prio - 1;
182 		ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
183 			mlx5_get_fdb_sub_ns(chains->dev, chain) :
184 			mlx5_get_flow_namespace(chains->dev, chains->ns);
185 	} else {
186 		ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
187 		ft_attr.prio = ns_to_chains_fs_prio(chains->ns);
188 		/* Firmware doesn't allow us to create another level 0 table,
189 		 * so we create all unmanaged tables as level 1.
190 		 *
191 		 * To connect them, we use explicit miss rules with
192 		 * ignore_flow_level. Caller is responsible to create
193 		 * these rules (if needed).
194 		 */
195 		ft_attr.level = 1;
196 		ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
197 	}
198 
199 	ft_attr.autogroup.num_reserved_entries = 2;
200 	ft_attr.autogroup.max_num_groups = chains->group_num;
201 	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
202 	if (IS_ERR(ft)) {
203 		mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
204 			       (int)PTR_ERR(ft), chain, prio, level, sz);
205 		return ft;
206 	}
207 
208 	return ft;
209 }
210 
211 static int
create_chain_restore(struct fs_chain * chain)212 create_chain_restore(struct fs_chain *chain)
213 {
214 	struct mlx5_eswitch *esw = chain->chains->dev->priv.eswitch;
215 	u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
216 	struct mlx5_fs_chains *chains = chain->chains;
217 	enum mlx5e_tc_attr_to_reg chain_to_reg;
218 	struct mlx5_modify_hdr *mod_hdr;
219 	u32 index;
220 	int err;
221 
222 	if (chain->chain == mlx5_chains_get_nf_ft_chain(chains) ||
223 	    !mlx5_chains_prios_supported(chains))
224 		return 0;
225 
226 	err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
227 	if (err)
228 		return err;
229 	if (index == MLX5_FS_DEFAULT_FLOW_TAG) {
230 		/* we got the special default flow tag id, so we won't know
231 		 * if we actually marked the packet with the restore rule
232 		 * we create.
233 		 *
234 		 * This case isn't possible with MLX5_FS_DEFAULT_FLOW_TAG = 0.
235 		 */
236 		err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
237 		mapping_remove(chains->chains_mapping, MLX5_FS_DEFAULT_FLOW_TAG);
238 		if (err)
239 			return err;
240 	}
241 
242 	chain->id = index;
243 
244 	if (chains->ns == MLX5_FLOW_NAMESPACE_FDB) {
245 		chain_to_reg = CHAIN_TO_REG;
246 		chain->restore_rule = esw_add_restore_rule(esw, chain->id);
247 		if (IS_ERR(chain->restore_rule)) {
248 			err = PTR_ERR(chain->restore_rule);
249 			goto err_rule;
250 		}
251 	} else if (chains->ns == MLX5_FLOW_NAMESPACE_KERNEL) {
252 		/* For NIC RX we don't need a restore rule
253 		 * since we write the metadata to reg_b
254 		 * that is passed to SW directly.
255 		 */
256 		chain_to_reg = NIC_CHAIN_TO_REG;
257 	} else {
258 		err = -EINVAL;
259 		goto err_rule;
260 	}
261 
262 	MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
263 	MLX5_SET(set_action_in, modact, field,
264 		 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mfield);
265 	MLX5_SET(set_action_in, modact, offset,
266 		 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset);
267 	MLX5_SET(set_action_in, modact, length,
268 		 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen == 32 ?
269 		 0 : mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen);
270 	MLX5_SET(set_action_in, modact, data, chain->id);
271 	mod_hdr = mlx5_modify_header_alloc(chains->dev, chains->ns,
272 					   1, modact);
273 	if (IS_ERR(mod_hdr)) {
274 		err = PTR_ERR(mod_hdr);
275 		goto err_mod_hdr;
276 	}
277 	chain->miss_modify_hdr = mod_hdr;
278 
279 	return 0;
280 
281 err_mod_hdr:
282 	if (!IS_ERR_OR_NULL(chain->restore_rule))
283 		mlx5_del_flow_rules(chain->restore_rule);
284 err_rule:
285 	/* Datapath can't find this mapping, so we can safely remove it */
286 	mapping_remove(chains->chains_mapping, chain->id);
287 	return err;
288 }
289 
destroy_chain_restore(struct fs_chain * chain)290 static void destroy_chain_restore(struct fs_chain *chain)
291 {
292 	struct mlx5_fs_chains *chains = chain->chains;
293 
294 	if (!chain->miss_modify_hdr)
295 		return;
296 
297 	if (chain->restore_rule)
298 		mlx5_del_flow_rules(chain->restore_rule);
299 
300 	mlx5_modify_header_dealloc(chains->dev, chain->miss_modify_hdr);
301 	mapping_remove(chains->chains_mapping, chain->id);
302 }
303 
304 static struct fs_chain *
mlx5_chains_create_chain(struct mlx5_fs_chains * chains,u32 chain)305 mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain)
306 {
307 	struct fs_chain *chain_s = NULL;
308 	int err;
309 
310 	chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL);
311 	if (!chain_s)
312 		return ERR_PTR(-ENOMEM);
313 
314 	chain_s->chains = chains;
315 	chain_s->chain = chain;
316 	INIT_LIST_HEAD(&chain_s->prios_list);
317 
318 	err = create_chain_restore(chain_s);
319 	if (err)
320 		goto err_restore;
321 
322 	err = rhashtable_insert_fast(&chains_ht(chains), &chain_s->node,
323 				     chain_params);
324 	if (err)
325 		goto err_insert;
326 
327 	return chain_s;
328 
329 err_insert:
330 	destroy_chain_restore(chain_s);
331 err_restore:
332 	kvfree(chain_s);
333 	return ERR_PTR(err);
334 }
335 
336 static void
mlx5_chains_destroy_chain(struct fs_chain * chain)337 mlx5_chains_destroy_chain(struct fs_chain *chain)
338 {
339 	struct mlx5_fs_chains *chains = chain->chains;
340 
341 	rhashtable_remove_fast(&chains_ht(chains), &chain->node,
342 			       chain_params);
343 
344 	destroy_chain_restore(chain);
345 	kvfree(chain);
346 }
347 
348 static struct fs_chain *
mlx5_chains_get_chain(struct mlx5_fs_chains * chains,u32 chain)349 mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain)
350 {
351 	struct fs_chain *chain_s;
352 
353 	chain_s = rhashtable_lookup_fast(&chains_ht(chains), &chain,
354 					 chain_params);
355 	if (!chain_s) {
356 		chain_s = mlx5_chains_create_chain(chains, chain);
357 		if (IS_ERR(chain_s))
358 			return chain_s;
359 	}
360 
361 	chain_s->ref++;
362 
363 	return chain_s;
364 }
365 
366 static struct mlx5_flow_handle *
mlx5_chains_add_miss_rule(struct fs_chain * chain,struct mlx5_flow_table * ft,struct mlx5_flow_table * next_ft)367 mlx5_chains_add_miss_rule(struct fs_chain *chain,
368 			  struct mlx5_flow_table *ft,
369 			  struct mlx5_flow_table *next_ft)
370 {
371 	struct mlx5_fs_chains *chains = chain->chains;
372 	struct mlx5_flow_destination dest = {};
373 	struct mlx5_flow_act act = {};
374 
375 	act.flags  = FLOW_ACT_NO_APPEND;
376 	if (mlx5_chains_ignore_flow_level_supported(chain->chains))
377 		act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
378 
379 	act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
380 	dest.type  = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
381 	dest.ft = next_ft;
382 
383 	if (next_ft == tc_end_ft(chains) &&
384 	    chain->chain != mlx5_chains_get_nf_ft_chain(chains) &&
385 	    mlx5_chains_prios_supported(chains)) {
386 		act.modify_hdr = chain->miss_modify_hdr;
387 		act.action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
388 	}
389 
390 	return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
391 }
392 
393 static int
mlx5_chains_update_prio_prevs(struct prio * prio,struct mlx5_flow_table * next_ft)394 mlx5_chains_update_prio_prevs(struct prio *prio,
395 			      struct mlx5_flow_table *next_ft)
396 {
397 	struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
398 	struct fs_chain *chain = prio->chain;
399 	struct prio *pos;
400 	int n = 0, err;
401 
402 	if (prio->key.level)
403 		return 0;
404 
405 	/* Iterate in reverse order until reaching the level 0 rule of
406 	 * the previous priority, adding all the miss rules first, so we can
407 	 * revert them if any of them fails.
408 	 */
409 	pos = prio;
410 	list_for_each_entry_continue_reverse(pos,
411 					     &chain->prios_list,
412 					     list) {
413 		miss_rules[n] = mlx5_chains_add_miss_rule(chain,
414 							  pos->ft,
415 							  next_ft);
416 		if (IS_ERR(miss_rules[n])) {
417 			err = PTR_ERR(miss_rules[n]);
418 			goto err_prev_rule;
419 		}
420 
421 		n++;
422 		if (!pos->key.level)
423 			break;
424 	}
425 
426 	/* Success, delete old miss rules, and update the pointers. */
427 	n = 0;
428 	pos = prio;
429 	list_for_each_entry_continue_reverse(pos,
430 					     &chain->prios_list,
431 					     list) {
432 		mlx5_del_flow_rules(pos->miss_rule);
433 
434 		pos->miss_rule = miss_rules[n];
435 		pos->next_ft = next_ft;
436 
437 		n++;
438 		if (!pos->key.level)
439 			break;
440 	}
441 
442 	return 0;
443 
444 err_prev_rule:
445 	while (--n >= 0)
446 		mlx5_del_flow_rules(miss_rules[n]);
447 
448 	return err;
449 }
450 
451 static void
mlx5_chains_put_chain(struct fs_chain * chain)452 mlx5_chains_put_chain(struct fs_chain *chain)
453 {
454 	if (--chain->ref == 0)
455 		mlx5_chains_destroy_chain(chain);
456 }
457 
458 static struct prio *
mlx5_chains_create_prio(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)459 mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
460 			u32 chain, u32 prio, u32 level)
461 {
462 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
463 	struct mlx5_flow_handle *miss_rule;
464 	struct mlx5_flow_group *miss_group;
465 	struct mlx5_flow_table *next_ft;
466 	struct mlx5_flow_table *ft;
467 	struct fs_chain *chain_s;
468 	struct list_head *pos;
469 	struct prio *prio_s;
470 	u32 *flow_group_in;
471 	int err;
472 
473 	chain_s = mlx5_chains_get_chain(chains, chain);
474 	if (IS_ERR(chain_s))
475 		return ERR_CAST(chain_s);
476 
477 	prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL);
478 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
479 	if (!prio_s || !flow_group_in) {
480 		err = -ENOMEM;
481 		goto err_alloc;
482 	}
483 
484 	/* Chain's prio list is sorted by prio and level.
485 	 * And all levels of some prio point to the next prio's level 0.
486 	 * Example list (prio, level):
487 	 * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0)
488 	 * In hardware, we will we have the following pointers:
489 	 * (3,0) -> (5,0) -> (7,0) -> Slow path
490 	 * (3,1) -> (5,0)
491 	 * (5,1) -> (7,0)
492 	 * (6,1) -> (7,0)
493 	 */
494 
495 	/* Default miss for each chain: */
496 	next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
497 		  tc_default_ft(chains) :
498 		  tc_end_ft(chains);
499 	list_for_each(pos, &chain_s->prios_list) {
500 		struct prio *p = list_entry(pos, struct prio, list);
501 
502 		/* exit on first pos that is larger */
503 		if (prio < p->key.prio || (prio == p->key.prio &&
504 					   level < p->key.level)) {
505 			/* Get next level 0 table */
506 			next_ft = p->key.level == 0 ? p->ft : p->next_ft;
507 			break;
508 		}
509 	}
510 
511 	ft = mlx5_chains_create_table(chains, chain, prio, level);
512 	if (IS_ERR(ft)) {
513 		err = PTR_ERR(ft);
514 		goto err_create;
515 	}
516 
517 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
518 		 ft->max_fte - 2);
519 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
520 		 ft->max_fte - 1);
521 	miss_group = mlx5_create_flow_group(ft, flow_group_in);
522 	if (IS_ERR(miss_group)) {
523 		err = PTR_ERR(miss_group);
524 		goto err_group;
525 	}
526 
527 	/* Add miss rule to next_ft */
528 	miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft);
529 	if (IS_ERR(miss_rule)) {
530 		err = PTR_ERR(miss_rule);
531 		goto err_miss_rule;
532 	}
533 
534 	prio_s->miss_group = miss_group;
535 	prio_s->miss_rule = miss_rule;
536 	prio_s->next_ft = next_ft;
537 	prio_s->chain = chain_s;
538 	prio_s->key.chain = chain;
539 	prio_s->key.prio = prio;
540 	prio_s->key.level = level;
541 	prio_s->ft = ft;
542 
543 	err = rhashtable_insert_fast(&prios_ht(chains), &prio_s->node,
544 				     prio_params);
545 	if (err)
546 		goto err_insert;
547 
548 	list_add(&prio_s->list, pos->prev);
549 
550 	/* Table is ready, connect it */
551 	err = mlx5_chains_update_prio_prevs(prio_s, ft);
552 	if (err)
553 		goto err_update;
554 
555 	kvfree(flow_group_in);
556 	return prio_s;
557 
558 err_update:
559 	list_del(&prio_s->list);
560 	rhashtable_remove_fast(&prios_ht(chains), &prio_s->node,
561 			       prio_params);
562 err_insert:
563 	mlx5_del_flow_rules(miss_rule);
564 err_miss_rule:
565 	mlx5_destroy_flow_group(miss_group);
566 err_group:
567 	mlx5_destroy_flow_table(ft);
568 err_create:
569 err_alloc:
570 	kvfree(prio_s);
571 	kvfree(flow_group_in);
572 	mlx5_chains_put_chain(chain_s);
573 	return ERR_PTR(err);
574 }
575 
576 static void
mlx5_chains_destroy_prio(struct mlx5_fs_chains * chains,struct prio * prio)577 mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains,
578 			 struct prio *prio)
579 {
580 	struct fs_chain *chain = prio->chain;
581 
582 	WARN_ON(mlx5_chains_update_prio_prevs(prio,
583 					      prio->next_ft));
584 
585 	list_del(&prio->list);
586 	rhashtable_remove_fast(&prios_ht(chains), &prio->node,
587 			       prio_params);
588 	mlx5_del_flow_rules(prio->miss_rule);
589 	mlx5_destroy_flow_group(prio->miss_group);
590 	mlx5_destroy_flow_table(prio->ft);
591 	mlx5_chains_put_chain(chain);
592 	kvfree(prio);
593 }
594 
595 struct mlx5_flow_table *
mlx5_chains_get_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)596 mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
597 		      u32 level)
598 {
599 	struct mlx5_flow_table *prev_fts;
600 	struct prio *prio_s;
601 	struct prio_key key;
602 	int l = 0;
603 
604 	if ((chain > mlx5_chains_get_chain_range(chains) &&
605 	     chain != mlx5_chains_get_nf_ft_chain(chains)) ||
606 	    prio > mlx5_chains_get_prio_range(chains) ||
607 	    level > mlx5_chains_get_level_range(chains))
608 		return ERR_PTR(-EOPNOTSUPP);
609 
610 	/* create earlier levels for correct fs_core lookup when
611 	 * connecting tables.
612 	 */
613 	for (l = 0; l < level; l++) {
614 		prev_fts = mlx5_chains_get_table(chains, chain, prio, l);
615 		if (IS_ERR(prev_fts)) {
616 			prio_s = ERR_CAST(prev_fts);
617 			goto err_get_prevs;
618 		}
619 	}
620 
621 	key.chain = chain;
622 	key.prio = prio;
623 	key.level = level;
624 
625 	mutex_lock(&chains_lock(chains));
626 	prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
627 					prio_params);
628 	if (!prio_s) {
629 		prio_s = mlx5_chains_create_prio(chains, chain,
630 						 prio, level);
631 		if (IS_ERR(prio_s))
632 			goto err_create_prio;
633 	}
634 
635 	++prio_s->ref;
636 	mutex_unlock(&chains_lock(chains));
637 
638 	return prio_s->ft;
639 
640 err_create_prio:
641 	mutex_unlock(&chains_lock(chains));
642 err_get_prevs:
643 	while (--l >= 0)
644 		mlx5_chains_put_table(chains, chain, prio, l);
645 	return ERR_CAST(prio_s);
646 }
647 
648 void
mlx5_chains_put_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)649 mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
650 		      u32 level)
651 {
652 	struct prio *prio_s;
653 	struct prio_key key;
654 
655 	key.chain = chain;
656 	key.prio = prio;
657 	key.level = level;
658 
659 	mutex_lock(&chains_lock(chains));
660 	prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
661 					prio_params);
662 	if (!prio_s)
663 		goto err_get_prio;
664 
665 	if (--prio_s->ref == 0)
666 		mlx5_chains_destroy_prio(chains, prio_s);
667 	mutex_unlock(&chains_lock(chains));
668 
669 	while (level-- > 0)
670 		mlx5_chains_put_table(chains, chain, prio, level);
671 
672 	return;
673 
674 err_get_prio:
675 	mutex_unlock(&chains_lock(chains));
676 	WARN_ONCE(1,
677 		  "Couldn't find table: (chain: %d prio: %d level: %d)",
678 		  chain, prio, level);
679 }
680 
681 struct mlx5_flow_table *
mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains * chains)682 mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
683 {
684 	return tc_end_ft(chains);
685 }
686 
687 struct mlx5_flow_table *
mlx5_chains_create_global_table(struct mlx5_fs_chains * chains)688 mlx5_chains_create_global_table(struct mlx5_fs_chains *chains)
689 {
690 	u32 chain, prio, level;
691 	int err;
692 
693 	if (!mlx5_chains_ignore_flow_level_supported(chains)) {
694 		err = -EOPNOTSUPP;
695 
696 		mlx5_core_warn(chains->dev,
697 			       "Couldn't create global flow table, ignore_flow_level not supported.");
698 		goto err_ignore;
699 	}
700 
701 	chain = mlx5_chains_get_chain_range(chains),
702 	prio = mlx5_chains_get_prio_range(chains);
703 	level = mlx5_chains_get_level_range(chains);
704 
705 	return mlx5_chains_create_table(chains, chain, prio, level);
706 
707 err_ignore:
708 	return ERR_PTR(err);
709 }
710 
711 void
mlx5_chains_destroy_global_table(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)712 mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
713 				 struct mlx5_flow_table *ft)
714 {
715 	mlx5_destroy_flow_table(ft);
716 }
717 
718 static struct mlx5_fs_chains *
mlx5_chains_init(struct mlx5_core_dev * dev,struct mlx5_chains_attr * attr)719 mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
720 {
721 	struct mlx5_fs_chains *chains_priv;
722 	u32 max_flow_counter;
723 	int err;
724 
725 	chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
726 	if (!chains_priv)
727 		return ERR_PTR(-ENOMEM);
728 
729 	max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
730 			    MLX5_CAP_GEN(dev, max_flow_counter_15_0);
731 
732 	mlx5_core_dbg(dev,
733 		      "Init flow table chains, max counters(%d), groups(%d), max flow table size(%d)\n",
734 		      max_flow_counter, attr->max_grp_num, attr->max_ft_sz);
735 
736 	chains_priv->dev = dev;
737 	chains_priv->flags = attr->flags;
738 	chains_priv->ns = attr->ns;
739 	chains_priv->group_num = attr->max_grp_num;
740 	chains_priv->chains_mapping = attr->mapping;
741 	tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft;
742 
743 	mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
744 		       mlx5_chains_get_chain_range(chains_priv),
745 		       mlx5_chains_get_prio_range(chains_priv));
746 
747 	err = rhashtable_init(&chains_ht(chains_priv), &chain_params);
748 	if (err)
749 		goto init_chains_ht_err;
750 
751 	err = rhashtable_init(&prios_ht(chains_priv), &prio_params);
752 	if (err)
753 		goto init_prios_ht_err;
754 
755 	mutex_init(&chains_lock(chains_priv));
756 
757 	return chains_priv;
758 
759 init_prios_ht_err:
760 	rhashtable_destroy(&chains_ht(chains_priv));
761 init_chains_ht_err:
762 	kfree(chains_priv);
763 	return ERR_PTR(err);
764 }
765 
766 static void
mlx5_chains_cleanup(struct mlx5_fs_chains * chains)767 mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
768 {
769 	mutex_destroy(&chains_lock(chains));
770 	rhashtable_destroy(&prios_ht(chains));
771 	rhashtable_destroy(&chains_ht(chains));
772 
773 	kfree(chains);
774 }
775 
776 struct mlx5_fs_chains *
mlx5_chains_create(struct mlx5_core_dev * dev,struct mlx5_chains_attr * attr)777 mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
778 {
779 	struct mlx5_fs_chains *chains;
780 
781 	chains = mlx5_chains_init(dev, attr);
782 
783 	return chains;
784 }
785 
786 void
mlx5_chains_destroy(struct mlx5_fs_chains * chains)787 mlx5_chains_destroy(struct mlx5_fs_chains *chains)
788 {
789 	mlx5_chains_cleanup(chains);
790 }
791 
792 int
mlx5_chains_get_chain_mapping(struct mlx5_fs_chains * chains,u32 chain,u32 * chain_mapping)793 mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
794 			      u32 *chain_mapping)
795 {
796 	struct mapping_ctx *ctx = chains->chains_mapping;
797 	struct mlx5_mapped_obj mapped_obj = {};
798 
799 	mapped_obj.type = MLX5_MAPPED_OBJ_CHAIN;
800 	mapped_obj.chain = chain;
801 	return mapping_add(ctx, &mapped_obj, chain_mapping);
802 }
803 
804 int
mlx5_chains_put_chain_mapping(struct mlx5_fs_chains * chains,u32 chain_mapping)805 mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping)
806 {
807 	struct mapping_ctx *ctx = chains->chains_mapping;
808 
809 	return mapping_remove(ctx, chain_mapping);
810 }
811