1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2019, Mellanox Technologies */
3 
4 #ifndef	_DR_TYPES_
5 #define	_DR_TYPES_
6 
7 #include <linux/mlx5/vport.h>
8 #include <linux/refcount.h>
9 #include "fs_core.h"
10 #include "wq.h"
11 #include "lib/mlx5.h"
12 #include "mlx5_ifc_dr.h"
13 #include "mlx5dr.h"
14 #include "dr_dbg.h"
15 
16 #define DR_RULE_MAX_STES 18
17 #define DR_ACTION_MAX_STES 5
18 #define DR_STE_SVLAN 0x1
19 #define DR_STE_CVLAN 0x2
20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4)
21 #define DR_NUM_OF_FLEX_PARSERS 8
22 #define DR_STE_MAX_FLEX_0_ID 3
23 #define DR_STE_MAX_FLEX_1_ID 7
24 
25 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg)
26 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg)
27 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg)
28 
dr_is_flex_parser_0_id(u8 parser_id)29 static inline bool dr_is_flex_parser_0_id(u8 parser_id)
30 {
31 	return parser_id <= DR_STE_MAX_FLEX_0_ID;
32 }
33 
dr_is_flex_parser_1_id(u8 parser_id)34 static inline bool dr_is_flex_parser_1_id(u8 parser_id)
35 {
36 	return parser_id > DR_STE_MAX_FLEX_0_ID;
37 }
38 
39 enum mlx5dr_icm_chunk_size {
40 	DR_CHUNK_SIZE_1,
41 	DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */
42 	DR_CHUNK_SIZE_2,
43 	DR_CHUNK_SIZE_4,
44 	DR_CHUNK_SIZE_8,
45 	DR_CHUNK_SIZE_16,
46 	DR_CHUNK_SIZE_32,
47 	DR_CHUNK_SIZE_64,
48 	DR_CHUNK_SIZE_128,
49 	DR_CHUNK_SIZE_256,
50 	DR_CHUNK_SIZE_512,
51 	DR_CHUNK_SIZE_1K,
52 	DR_CHUNK_SIZE_2K,
53 	DR_CHUNK_SIZE_4K,
54 	DR_CHUNK_SIZE_8K,
55 	DR_CHUNK_SIZE_16K,
56 	DR_CHUNK_SIZE_32K,
57 	DR_CHUNK_SIZE_64K,
58 	DR_CHUNK_SIZE_128K,
59 	DR_CHUNK_SIZE_256K,
60 	DR_CHUNK_SIZE_512K,
61 	DR_CHUNK_SIZE_1024K,
62 	DR_CHUNK_SIZE_2048K,
63 	DR_CHUNK_SIZE_MAX,
64 };
65 
66 enum mlx5dr_icm_type {
67 	DR_ICM_TYPE_STE,
68 	DR_ICM_TYPE_MODIFY_ACTION,
69 };
70 
71 static inline enum mlx5dr_icm_chunk_size
mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)72 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)
73 {
74 	chunk += 2;
75 	if (chunk < DR_CHUNK_SIZE_MAX)
76 		return chunk;
77 
78 	return DR_CHUNK_SIZE_MAX;
79 }
80 
81 enum {
82 	DR_STE_SIZE = 64,
83 	DR_STE_SIZE_CTRL = 32,
84 	DR_STE_SIZE_TAG = 16,
85 	DR_STE_SIZE_MASK = 16,
86 	DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK,
87 };
88 
89 enum mlx5dr_ste_ctx_action_cap {
90 	DR_STE_CTX_ACTION_CAP_NONE = 0,
91 	DR_STE_CTX_ACTION_CAP_TX_POP   = 1 << 0,
92 	DR_STE_CTX_ACTION_CAP_RX_PUSH  = 1 << 1,
93 	DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2,
94 	DR_STE_CTX_ACTION_CAP_POP_MDFY = 1 << 3,
95 };
96 
97 enum {
98 	DR_MODIFY_ACTION_SIZE = 8,
99 };
100 
101 enum mlx5dr_matcher_criteria {
102 	DR_MATCHER_CRITERIA_EMPTY = 0,
103 	DR_MATCHER_CRITERIA_OUTER = 1 << 0,
104 	DR_MATCHER_CRITERIA_MISC = 1 << 1,
105 	DR_MATCHER_CRITERIA_INNER = 1 << 2,
106 	DR_MATCHER_CRITERIA_MISC2 = 1 << 3,
107 	DR_MATCHER_CRITERIA_MISC3 = 1 << 4,
108 	DR_MATCHER_CRITERIA_MISC4 = 1 << 5,
109 	DR_MATCHER_CRITERIA_MISC5 = 1 << 6,
110 	DR_MATCHER_CRITERIA_MAX = 1 << 7,
111 };
112 
113 enum mlx5dr_action_type {
114 	DR_ACTION_TYP_TNL_L2_TO_L2,
115 	DR_ACTION_TYP_L2_TO_TNL_L2,
116 	DR_ACTION_TYP_TNL_L3_TO_L2,
117 	DR_ACTION_TYP_L2_TO_TNL_L3,
118 	DR_ACTION_TYP_DROP,
119 	DR_ACTION_TYP_QP,
120 	DR_ACTION_TYP_FT,
121 	DR_ACTION_TYP_CTR,
122 	DR_ACTION_TYP_TAG,
123 	DR_ACTION_TYP_MODIFY_HDR,
124 	DR_ACTION_TYP_VPORT,
125 	DR_ACTION_TYP_POP_VLAN,
126 	DR_ACTION_TYP_PUSH_VLAN,
127 	DR_ACTION_TYP_INSERT_HDR,
128 	DR_ACTION_TYP_REMOVE_HDR,
129 	DR_ACTION_TYP_SAMPLER,
130 	DR_ACTION_TYP_ASO_FLOW_METER,
131 	DR_ACTION_TYP_MAX,
132 };
133 
134 enum mlx5dr_ipv {
135 	DR_RULE_IPV4,
136 	DR_RULE_IPV6,
137 	DR_RULE_IPV_MAX,
138 };
139 
140 struct mlx5dr_icm_pool;
141 struct mlx5dr_icm_chunk;
142 struct mlx5dr_icm_buddy_mem;
143 struct mlx5dr_ste_htbl;
144 struct mlx5dr_match_param;
145 struct mlx5dr_cmd_caps;
146 struct mlx5dr_rule_rx_tx;
147 struct mlx5dr_matcher_rx_tx;
148 struct mlx5dr_ste_ctx;
149 
150 struct mlx5dr_ste {
151 	/* refcount: indicates the num of rules that using this ste */
152 	u32 refcount;
153 
154 	/* this ste is part of a rule, located in ste's chain */
155 	u8 ste_chain_location;
156 
157 	/* attached to the miss_list head at each htbl entry */
158 	struct list_head miss_list_node;
159 
160 	/* this ste is member of htbl */
161 	struct mlx5dr_ste_htbl *htbl;
162 
163 	struct mlx5dr_ste_htbl *next_htbl;
164 
165 	/* The rule this STE belongs to */
166 	struct mlx5dr_rule_rx_tx *rule_rx_tx;
167 };
168 
169 struct mlx5dr_ste_htbl_ctrl {
170 	/* total number of valid entries belonging to this hash table. This
171 	 * includes the non collision and collision entries
172 	 */
173 	unsigned int num_of_valid_entries;
174 
175 	/* total number of collisions entries attached to this table */
176 	unsigned int num_of_collisions;
177 };
178 
179 struct mlx5dr_ste_htbl {
180 	u16 lu_type;
181 	u16 byte_mask;
182 	u32 refcount;
183 	struct mlx5dr_icm_chunk *chunk;
184 	struct mlx5dr_ste *pointing_ste;
185 	struct mlx5dr_ste_htbl_ctrl ctrl;
186 };
187 
188 struct mlx5dr_ste_send_info {
189 	struct mlx5dr_ste *ste;
190 	struct list_head send_list;
191 	u16 size;
192 	u16 offset;
193 	u8 data_cont[DR_STE_SIZE];
194 	u8 *data;
195 };
196 
197 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size,
198 					       u16 offset, u8 *data,
199 					       struct mlx5dr_ste_send_info *ste_info,
200 					       struct list_head *send_list,
201 					       bool copy_data);
202 
203 struct mlx5dr_ste_build {
204 	u8 inner:1;
205 	u8 rx:1;
206 	u8 vhca_id_valid:1;
207 	struct mlx5dr_domain *dmn;
208 	struct mlx5dr_cmd_caps *caps;
209 	u16 lu_type;
210 	u16 byte_mask;
211 	u8 bit_mask[DR_STE_SIZE_MASK];
212 	int (*ste_build_tag_func)(struct mlx5dr_match_param *spec,
213 				  struct mlx5dr_ste_build *sb,
214 				  u8 *tag);
215 };
216 
217 struct mlx5dr_ste_htbl *
218 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool,
219 		      enum mlx5dr_icm_chunk_size chunk_size,
220 		      u16 lu_type, u16 byte_mask);
221 
222 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl);
223 
mlx5dr_htbl_put(struct mlx5dr_ste_htbl * htbl)224 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl)
225 {
226 	htbl->refcount--;
227 	if (!htbl->refcount)
228 		mlx5dr_ste_htbl_free(htbl);
229 }
230 
mlx5dr_htbl_get(struct mlx5dr_ste_htbl * htbl)231 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl)
232 {
233 	htbl->refcount++;
234 }
235 
236 /* STE utils */
237 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl);
238 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx,
239 			      u8 *hw_ste, u64 miss_addr);
240 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx,
241 			     u8 *hw_ste, u64 icm_addr, u32 ht_size);
242 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx,
243 					  u8 *hw_ste,
244 					  struct mlx5dr_ste_htbl *next_htbl);
245 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask);
246 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher,
247 				u8 ste_location);
248 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste);
249 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste);
250 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste);
251 
252 #define MLX5DR_MAX_VLANS 2
253 
254 struct mlx5dr_ste_actions_attr {
255 	u32	modify_index;
256 	u16	modify_actions;
257 	u32	decap_index;
258 	u16	decap_actions;
259 	u8	decap_with_vlan:1;
260 	u64	final_icm_addr;
261 	u32	flow_tag;
262 	u32	ctr_id;
263 	u16	gvmi;
264 	u16	hit_gvmi;
265 	struct {
266 		u32	id;
267 		u32	size;
268 		u8	param_0;
269 		u8	param_1;
270 	} reformat;
271 	struct {
272 		int	count;
273 		u32	headers[MLX5DR_MAX_VLANS];
274 	} vlans;
275 
276 	struct {
277 		u32 obj_id;
278 		u32 offset;
279 		u8 dest_reg_id;
280 		u8 init_color;
281 	} aso_flow_meter;
282 };
283 
284 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx,
285 			       struct mlx5dr_domain *dmn,
286 			       u8 *action_type_set,
287 			       u8 *last_ste,
288 			       struct mlx5dr_ste_actions_attr *attr,
289 			       u32 *added_stes);
290 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx,
291 			       struct mlx5dr_domain *dmn,
292 			       u8 *action_type_set,
293 			       u8 *last_ste,
294 			       struct mlx5dr_ste_actions_attr *attr,
295 			       u32 *added_stes);
296 
297 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx,
298 			       __be64 *hw_action,
299 			       u8 hw_field,
300 			       u8 shifter,
301 			       u8 length,
302 			       u32 data);
303 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx,
304 			       __be64 *hw_action,
305 			       u8 hw_field,
306 			       u8 shifter,
307 			       u8 length,
308 			       u32 data);
309 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx,
310 				__be64 *hw_action,
311 				u8 dst_hw_field,
312 				u8 dst_shifter,
313 				u8 dst_len,
314 				u8 src_hw_field,
315 				u8 src_shifter);
316 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
317 					void *data,
318 					u32 data_sz,
319 					u8 *hw_action,
320 					u32 hw_action_sz,
321 					u16 *used_hw_action_num);
322 
323 const struct mlx5dr_ste_action_modify_field *
324 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field);
325 
326 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version);
327 void mlx5dr_ste_free(struct mlx5dr_ste *ste,
328 		     struct mlx5dr_matcher *matcher,
329 		     struct mlx5dr_matcher_rx_tx *nic_matcher);
mlx5dr_ste_put(struct mlx5dr_ste * ste,struct mlx5dr_matcher * matcher,struct mlx5dr_matcher_rx_tx * nic_matcher)330 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste,
331 				  struct mlx5dr_matcher *matcher,
332 				  struct mlx5dr_matcher_rx_tx *nic_matcher)
333 {
334 	ste->refcount--;
335 	if (!ste->refcount)
336 		mlx5dr_ste_free(ste, matcher, nic_matcher);
337 }
338 
339 /* initial as 0, increased only when ste appears in a new rule */
mlx5dr_ste_get(struct mlx5dr_ste * ste)340 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste)
341 {
342 	ste->refcount++;
343 }
344 
mlx5dr_ste_is_not_used(struct mlx5dr_ste * ste)345 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste)
346 {
347 	return !ste->refcount;
348 }
349 
350 bool mlx5dr_ste_equal_tag(void *src, void *dst);
351 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher,
352 				struct mlx5dr_matcher_rx_tx *nic_matcher,
353 				struct mlx5dr_ste *ste,
354 				u8 *cur_hw_ste,
355 				enum mlx5dr_icm_chunk_size log_table_size);
356 
357 /* STE build functions */
358 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn,
359 			       u8 match_criteria,
360 			       struct mlx5dr_match_param *mask,
361 			       struct mlx5dr_match_param *value);
362 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher,
363 			     struct mlx5dr_matcher_rx_tx *nic_matcher,
364 			     struct mlx5dr_match_param *value,
365 			     u8 *ste_arr);
366 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx,
367 				     struct mlx5dr_ste_build *builder,
368 				     struct mlx5dr_match_param *mask,
369 				     bool inner, bool rx);
370 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx,
371 					  struct mlx5dr_ste_build *sb,
372 					  struct mlx5dr_match_param *mask,
373 					  bool inner, bool rx);
374 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx,
375 				       struct mlx5dr_ste_build *sb,
376 				       struct mlx5dr_match_param *mask,
377 				       bool inner, bool rx);
378 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx,
379 				      struct mlx5dr_ste_build *sb,
380 				      struct mlx5dr_match_param *mask,
381 				      bool inner, bool rx);
382 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx,
383 				      struct mlx5dr_ste_build *sb,
384 				      struct mlx5dr_match_param *mask,
385 				      bool inner, bool rx);
386 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx,
387 				 struct mlx5dr_ste_build *sb,
388 				 struct mlx5dr_match_param *mask,
389 				 bool inner, bool rx);
390 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx,
391 				 struct mlx5dr_ste_build *sb,
392 				 struct mlx5dr_match_param *mask,
393 				 bool inner, bool rx);
394 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx,
395 				 struct mlx5dr_ste_build *sb,
396 				 struct mlx5dr_match_param *mask,
397 				 bool inner, bool rx);
398 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx,
399 				     struct mlx5dr_ste_build *sb,
400 				     struct mlx5dr_match_param *mask,
401 				     bool inner, bool rx);
402 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx,
403 				  struct mlx5dr_ste_build *sb,
404 				  struct mlx5dr_match_param *mask,
405 				  bool inner, bool rx);
406 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx,
407 			      struct mlx5dr_ste_build *sb,
408 			      struct mlx5dr_match_param *mask,
409 			      bool inner, bool rx);
410 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx,
411 			   struct mlx5dr_ste_build *sb,
412 			   struct mlx5dr_match_param *mask,
413 			   bool inner, bool rx);
414 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx,
415 			       struct mlx5dr_ste_build *sb,
416 			       struct mlx5dr_match_param *mask,
417 			       bool inner, bool rx);
418 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx,
419 					struct mlx5dr_ste_build *sb,
420 					struct mlx5dr_match_param *mask,
421 					struct mlx5dr_cmd_caps *caps,
422 					bool inner, bool rx);
423 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx,
424 					struct mlx5dr_ste_build *sb,
425 					struct mlx5dr_match_param *mask,
426 					struct mlx5dr_cmd_caps *caps,
427 					bool inner, bool rx);
428 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx,
429 			   struct mlx5dr_ste_build *sb,
430 			   struct mlx5dr_match_param *mask,
431 			   struct mlx5dr_cmd_caps *caps,
432 			   bool inner, bool rx);
433 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx,
434 				    struct mlx5dr_ste_build *sb,
435 				    struct mlx5dr_match_param *mask,
436 				    bool inner, bool rx);
437 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx,
438 				 struct mlx5dr_ste_build *sb,
439 				 struct mlx5dr_match_param *mask,
440 				 bool inner, bool rx);
441 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx,
442 					 struct mlx5dr_ste_build *sb,
443 					 struct mlx5dr_match_param *mask,
444 					 struct mlx5dr_cmd_caps *caps,
445 					 bool inner, bool rx);
446 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx,
447 					       struct mlx5dr_ste_build *sb,
448 					       struct mlx5dr_match_param *mask,
449 					       struct mlx5dr_cmd_caps *caps,
450 					       bool inner, bool rx);
451 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx,
452 			       struct mlx5dr_ste_build *sb,
453 			       struct mlx5dr_match_param *mask,
454 			       bool inner, bool rx);
455 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
456 					     struct mlx5dr_ste_build *sb,
457 					     struct mlx5dr_match_param *mask,
458 					     struct mlx5dr_cmd_caps *caps,
459 					     bool inner, bool rx);
460 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
461 					     struct mlx5dr_ste_build *sb,
462 					     struct mlx5dr_match_param *mask,
463 					     struct mlx5dr_cmd_caps *caps,
464 					     bool inner, bool rx);
465 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx,
466 				     struct mlx5dr_ste_build *sb,
467 				     struct mlx5dr_match_param *mask,
468 				     bool inner, bool rx);
469 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx,
470 				      struct mlx5dr_ste_build *sb,
471 				      struct mlx5dr_match_param *mask,
472 				      bool inner, bool rx);
473 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx,
474 				 struct mlx5dr_ste_build *sb,
475 				 struct mlx5dr_match_param *mask,
476 				 bool inner, bool rx);
477 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx,
478 				 struct mlx5dr_ste_build *sb,
479 				 struct mlx5dr_match_param *mask,
480 				 bool inner, bool rx);
481 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx,
482 				   struct mlx5dr_ste_build *sb,
483 				   struct mlx5dr_match_param *mask,
484 				   struct mlx5dr_domain *dmn,
485 				   bool inner, bool rx);
486 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
487 				    struct mlx5dr_ste_build *sb,
488 				    struct mlx5dr_match_param *mask,
489 				    bool inner, bool rx);
490 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
491 				    struct mlx5dr_ste_build *sb,
492 				    struct mlx5dr_match_param *mask,
493 				    bool inner, bool rx);
494 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx);
495 
496 /* Actions utils */
497 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher,
498 				 struct mlx5dr_matcher_rx_tx *nic_matcher,
499 				 struct mlx5dr_action *actions[],
500 				 u32 num_actions,
501 				 u8 *ste_arr,
502 				 u32 *new_hw_ste_arr_sz);
503 
504 struct mlx5dr_match_spec {
505 	u32 smac_47_16;		/* Source MAC address of incoming packet */
506 	/* Incoming packet Ethertype - this is the Ethertype
507 	 * following the last VLAN tag of the packet
508 	 */
509 	u32 smac_15_0:16;	/* Source MAC address of incoming packet */
510 	u32 ethertype:16;
511 
512 	u32 dmac_47_16;		/* Destination MAC address of incoming packet */
513 
514 	u32 dmac_15_0:16;	/* Destination MAC address of incoming packet */
515 	/* Priority of first VLAN tag in the incoming packet.
516 	 * Valid only when cvlan_tag==1 or svlan_tag==1
517 	 */
518 	u32 first_prio:3;
519 	/* CFI bit of first VLAN tag in the incoming packet.
520 	 * Valid only when cvlan_tag==1 or svlan_tag==1
521 	 */
522 	u32 first_cfi:1;
523 	/* VLAN ID of first VLAN tag in the incoming packet.
524 	 * Valid only when cvlan_tag==1 or svlan_tag==1
525 	 */
526 	u32 first_vid:12;
527 
528 	u32 ip_protocol:8;	/* IP protocol */
529 	/* Differentiated Services Code Point derived from
530 	 * Traffic Class/TOS field of IPv6/v4
531 	 */
532 	u32 ip_dscp:6;
533 	/* Explicit Congestion Notification derived from
534 	 * Traffic Class/TOS field of IPv6/v4
535 	 */
536 	u32 ip_ecn:2;
537 	/* The first vlan in the packet is c-vlan (0x8100).
538 	 * cvlan_tag and svlan_tag cannot be set together
539 	 */
540 	u32 cvlan_tag:1;
541 	/* The first vlan in the packet is s-vlan (0x8a88).
542 	 * cvlan_tag and svlan_tag cannot be set together
543 	 */
544 	u32 svlan_tag:1;
545 	u32 frag:1;		/* Packet is an IP fragment */
546 	u32 ip_version:4;	/* IP version */
547 	/* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK;
548 	 *             Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS
549 	 */
550 	u32 tcp_flags:9;
551 
552 	/* TCP source port.;tcp and udp sport/dport are mutually exclusive */
553 	u32 tcp_sport:16;
554 	/* TCP destination port.
555 	 * tcp and udp sport/dport are mutually exclusive
556 	 */
557 	u32 tcp_dport:16;
558 
559 	u32 reserved_auto1:16;
560 	u32 ipv4_ihl:4;
561 	u32 reserved_auto2:4;
562 	u32 ttl_hoplimit:8;
563 
564 	/* UDP source port.;tcp and udp sport/dport are mutually exclusive */
565 	u32 udp_sport:16;
566 	/* UDP destination port.;tcp and udp sport/dport are mutually exclusive */
567 	u32 udp_dport:16;
568 
569 	/* IPv6 source address of incoming packets
570 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
571 	 * This field should be qualified by an appropriate ethertype
572 	 */
573 	u32 src_ip_127_96;
574 	/* IPv6 source address of incoming packets
575 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
576 	 * This field should be qualified by an appropriate ethertype
577 	 */
578 	u32 src_ip_95_64;
579 	/* IPv6 source address of incoming packets
580 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
581 	 * This field should be qualified by an appropriate ethertype
582 	 */
583 	u32 src_ip_63_32;
584 	/* IPv6 source address of incoming packets
585 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
586 	 * This field should be qualified by an appropriate ethertype
587 	 */
588 	u32 src_ip_31_0;
589 	/* IPv6 destination address of incoming packets
590 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
591 	 * This field should be qualified by an appropriate ethertype
592 	 */
593 	u32 dst_ip_127_96;
594 	/* IPv6 destination address of incoming packets
595 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
596 	 * This field should be qualified by an appropriate ethertype
597 	 */
598 	u32 dst_ip_95_64;
599 	/* IPv6 destination address of incoming packets
600 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
601 	 * This field should be qualified by an appropriate ethertype
602 	 */
603 	u32 dst_ip_63_32;
604 	/* IPv6 destination address of incoming packets
605 	 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
606 	 * This field should be qualified by an appropriate ethertype
607 	 */
608 	u32 dst_ip_31_0;
609 };
610 
611 struct mlx5dr_match_misc {
612 	/* used with GRE, checksum exist when gre_c_present == 1 */
613 	u32 gre_c_present:1;
614 	u32 reserved_auto1:1;
615 	/* used with GRE, key exist when gre_k_present == 1 */
616 	u32 gre_k_present:1;
617 	/* used with GRE, sequence number exist when gre_s_present == 1 */
618 	u32 gre_s_present:1;
619 	u32 source_vhca_port:4;
620 	u32 source_sqn:24;		/* Source SQN */
621 
622 	u32 source_eswitch_owner_vhca_id:16;
623 	/* Source port.;0xffff determines wire port */
624 	u32 source_port:16;
625 
626 	/* Priority of second VLAN tag in the outer header of the incoming packet.
627 	 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
628 	 */
629 	u32 outer_second_prio:3;
630 	/* CFI bit of first VLAN tag in the outer header of the incoming packet.
631 	 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
632 	 */
633 	u32 outer_second_cfi:1;
634 	/* VLAN ID of first VLAN tag the outer header of the incoming packet.
635 	 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
636 	 */
637 	u32 outer_second_vid:12;
638 	/* Priority of second VLAN tag in the inner header of the incoming packet.
639 	 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
640 	 */
641 	u32 inner_second_prio:3;
642 	/* CFI bit of first VLAN tag in the inner header of the incoming packet.
643 	 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
644 	 */
645 	u32 inner_second_cfi:1;
646 	/* VLAN ID of first VLAN tag the inner header of the incoming packet.
647 	 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
648 	 */
649 	u32 inner_second_vid:12;
650 
651 	u32 outer_second_cvlan_tag:1;
652 	u32 inner_second_cvlan_tag:1;
653 	/* The second vlan in the outer header of the packet is c-vlan (0x8100).
654 	 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
655 	 */
656 	u32 outer_second_svlan_tag:1;
657 	/* The second vlan in the inner header of the packet is c-vlan (0x8100).
658 	 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
659 	 */
660 	u32 inner_second_svlan_tag:1;
661 	/* The second vlan in the outer header of the packet is s-vlan (0x8a88).
662 	 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
663 	 */
664 	u32 reserved_auto2:12;
665 	/* The second vlan in the inner header of the packet is s-vlan (0x8a88).
666 	 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
667 	 */
668 	u32 gre_protocol:16;		/* GRE Protocol (outer) */
669 
670 	u32 gre_key_h:24;		/* GRE Key[31:8] (outer) */
671 	u32 gre_key_l:8;		/* GRE Key [7:0] (outer) */
672 
673 	u32 vxlan_vni:24;		/* VXLAN VNI (outer) */
674 	u32 reserved_auto3:8;
675 
676 	u32 geneve_vni:24;		/* GENEVE VNI field (outer) */
677 	u32 reserved_auto4:6;
678 	u32 geneve_tlv_option_0_exist:1;
679 	u32 geneve_oam:1;		/* GENEVE OAM field (outer) */
680 
681 	u32 reserved_auto5:12;
682 	u32 outer_ipv6_flow_label:20;	/* Flow label of incoming IPv6 packet (outer) */
683 
684 	u32 reserved_auto6:12;
685 	u32 inner_ipv6_flow_label:20;	/* Flow label of incoming IPv6 packet (inner) */
686 
687 	u32 reserved_auto7:10;
688 	u32 geneve_opt_len:6;		/* GENEVE OptLen (outer) */
689 	u32 geneve_protocol_type:16;	/* GENEVE protocol type (outer) */
690 
691 	u32 reserved_auto8:8;
692 	u32 bth_dst_qp:24;		/* Destination QP in BTH header */
693 
694 	u32 reserved_auto9;
695 	u32 outer_esp_spi;
696 	u32 reserved_auto10[3];
697 };
698 
699 struct mlx5dr_match_misc2 {
700 	u32 outer_first_mpls_label:20;		/* First MPLS LABEL (outer) */
701 	u32 outer_first_mpls_exp:3;		/* First MPLS EXP (outer) */
702 	u32 outer_first_mpls_s_bos:1;		/* First MPLS S_BOS (outer) */
703 	u32 outer_first_mpls_ttl:8;		/* First MPLS TTL (outer) */
704 
705 	u32 inner_first_mpls_label:20;		/* First MPLS LABEL (inner) */
706 	u32 inner_first_mpls_exp:3;		/* First MPLS EXP (inner) */
707 	u32 inner_first_mpls_s_bos:1;		/* First MPLS S_BOS (inner) */
708 	u32 inner_first_mpls_ttl:8;		/* First MPLS TTL (inner) */
709 
710 	u32 outer_first_mpls_over_gre_label:20;	/* last MPLS LABEL (outer) */
711 	u32 outer_first_mpls_over_gre_exp:3;	/* last MPLS EXP (outer) */
712 	u32 outer_first_mpls_over_gre_s_bos:1;	/* last MPLS S_BOS (outer) */
713 	u32 outer_first_mpls_over_gre_ttl:8;	/* last MPLS TTL (outer) */
714 
715 	u32 outer_first_mpls_over_udp_label:20;	/* last MPLS LABEL (outer) */
716 	u32 outer_first_mpls_over_udp_exp:3;	/* last MPLS EXP (outer) */
717 	u32 outer_first_mpls_over_udp_s_bos:1;	/* last MPLS S_BOS (outer) */
718 	u32 outer_first_mpls_over_udp_ttl:8;	/* last MPLS TTL (outer) */
719 
720 	u32 metadata_reg_c_7;			/* metadata_reg_c_7 */
721 	u32 metadata_reg_c_6;			/* metadata_reg_c_6 */
722 	u32 metadata_reg_c_5;			/* metadata_reg_c_5 */
723 	u32 metadata_reg_c_4;			/* metadata_reg_c_4 */
724 	u32 metadata_reg_c_3;			/* metadata_reg_c_3 */
725 	u32 metadata_reg_c_2;			/* metadata_reg_c_2 */
726 	u32 metadata_reg_c_1;			/* metadata_reg_c_1 */
727 	u32 metadata_reg_c_0;			/* metadata_reg_c_0 */
728 	u32 metadata_reg_a;			/* metadata_reg_a */
729 	u32 reserved_auto1[3];
730 };
731 
732 struct mlx5dr_match_misc3 {
733 	u32 inner_tcp_seq_num;
734 	u32 outer_tcp_seq_num;
735 	u32 inner_tcp_ack_num;
736 	u32 outer_tcp_ack_num;
737 
738 	u32 reserved_auto1:8;
739 	u32 outer_vxlan_gpe_vni:24;
740 
741 	u32 outer_vxlan_gpe_next_protocol:8;
742 	u32 outer_vxlan_gpe_flags:8;
743 	u32 reserved_auto2:16;
744 
745 	u32 icmpv4_header_data;
746 	u32 icmpv6_header_data;
747 
748 	u8 icmpv4_type;
749 	u8 icmpv4_code;
750 	u8 icmpv6_type;
751 	u8 icmpv6_code;
752 
753 	u32 geneve_tlv_option_0_data;
754 
755 	u32 gtpu_teid;
756 
757 	u8 gtpu_msg_type;
758 	u8 gtpu_msg_flags;
759 	u32 reserved_auto3:16;
760 
761 	u32 gtpu_dw_2;
762 	u32 gtpu_first_ext_dw_0;
763 	u32 gtpu_dw_0;
764 	u32 reserved_auto4;
765 };
766 
767 struct mlx5dr_match_misc4 {
768 	u32 prog_sample_field_value_0;
769 	u32 prog_sample_field_id_0;
770 	u32 prog_sample_field_value_1;
771 	u32 prog_sample_field_id_1;
772 	u32 prog_sample_field_value_2;
773 	u32 prog_sample_field_id_2;
774 	u32 prog_sample_field_value_3;
775 	u32 prog_sample_field_id_3;
776 	u32 reserved_auto1[8];
777 };
778 
779 struct mlx5dr_match_misc5 {
780 	u32 macsec_tag_0;
781 	u32 macsec_tag_1;
782 	u32 macsec_tag_2;
783 	u32 macsec_tag_3;
784 	u32 tunnel_header_0;
785 	u32 tunnel_header_1;
786 	u32 tunnel_header_2;
787 	u32 tunnel_header_3;
788 };
789 
790 struct mlx5dr_match_param {
791 	struct mlx5dr_match_spec outer;
792 	struct mlx5dr_match_misc misc;
793 	struct mlx5dr_match_spec inner;
794 	struct mlx5dr_match_misc2 misc2;
795 	struct mlx5dr_match_misc3 misc3;
796 	struct mlx5dr_match_misc4 misc4;
797 	struct mlx5dr_match_misc5 misc5;
798 };
799 
800 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \
801 				       (_misc3)->icmpv4_code || \
802 				       (_misc3)->icmpv4_header_data)
803 
804 #define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \
805 				      (_spec)->src_ip_95_64  || \
806 				      (_spec)->src_ip_63_32  || \
807 				      (_spec)->src_ip_31_0)
808 
809 #define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \
810 				      (_spec)->dst_ip_95_64  || \
811 				      (_spec)->dst_ip_63_32  || \
812 				      (_spec)->dst_ip_31_0)
813 
814 struct mlx5dr_esw_caps {
815 	u64 drop_icm_address_rx;
816 	u64 drop_icm_address_tx;
817 	u64 uplink_icm_address_rx;
818 	u64 uplink_icm_address_tx;
819 	u8 sw_owner:1;
820 	u8 sw_owner_v2:1;
821 };
822 
823 struct mlx5dr_cmd_vport_cap {
824 	u16 vport_gvmi;
825 	u16 vhca_gvmi;
826 	u16 num;
827 	u64 icm_address_rx;
828 	u64 icm_address_tx;
829 };
830 
831 struct mlx5dr_roce_cap {
832 	u8 roce_en:1;
833 	u8 fl_rc_qp_when_roce_disabled:1;
834 	u8 fl_rc_qp_when_roce_enabled:1;
835 };
836 
837 struct mlx5dr_vports {
838 	struct mlx5dr_cmd_vport_cap esw_manager_caps;
839 	struct mlx5dr_cmd_vport_cap uplink_caps;
840 	struct xarray vports_caps_xa;
841 };
842 
843 struct mlx5dr_cmd_caps {
844 	u16 gvmi;
845 	u64 nic_rx_drop_address;
846 	u64 nic_tx_drop_address;
847 	u64 nic_tx_allow_address;
848 	u64 esw_rx_drop_address;
849 	u64 esw_tx_drop_address;
850 	u32 log_icm_size;
851 	u64 hdr_modify_icm_addr;
852 	u32 flex_protocols;
853 	u8 flex_parser_id_icmp_dw0;
854 	u8 flex_parser_id_icmp_dw1;
855 	u8 flex_parser_id_icmpv6_dw0;
856 	u8 flex_parser_id_icmpv6_dw1;
857 	u8 flex_parser_id_geneve_tlv_option_0;
858 	u8 flex_parser_id_mpls_over_gre;
859 	u8 flex_parser_id_mpls_over_udp;
860 	u8 flex_parser_id_gtpu_dw_0;
861 	u8 flex_parser_id_gtpu_teid;
862 	u8 flex_parser_id_gtpu_dw_2;
863 	u8 flex_parser_id_gtpu_first_ext_dw_0;
864 	u8 flex_parser_ok_bits_supp;
865 	u8 max_ft_level;
866 	u16 roce_min_src_udp;
867 	u8 sw_format_ver;
868 	bool eswitch_manager;
869 	bool rx_sw_owner;
870 	bool tx_sw_owner;
871 	bool fdb_sw_owner;
872 	u8 rx_sw_owner_v2:1;
873 	u8 tx_sw_owner_v2:1;
874 	u8 fdb_sw_owner_v2:1;
875 	struct mlx5dr_esw_caps esw_caps;
876 	struct mlx5dr_vports vports;
877 	bool prio_tag_required;
878 	struct mlx5dr_roce_cap roce_caps;
879 	u8 is_ecpf:1;
880 	u8 isolate_vl_tc:1;
881 };
882 
883 enum mlx5dr_domain_nic_type {
884 	DR_DOMAIN_NIC_TYPE_RX,
885 	DR_DOMAIN_NIC_TYPE_TX,
886 };
887 
888 struct mlx5dr_domain_rx_tx {
889 	u64 drop_icm_addr;
890 	u64 default_icm_addr;
891 	enum mlx5dr_domain_nic_type type;
892 	struct mutex mutex; /* protect rx/tx domain */
893 };
894 
895 struct mlx5dr_domain_info {
896 	bool supp_sw_steering;
897 	u32 max_inline_size;
898 	u32 max_send_wr;
899 	u32 max_log_sw_icm_sz;
900 	u32 max_log_action_icm_sz;
901 	struct mlx5dr_domain_rx_tx rx;
902 	struct mlx5dr_domain_rx_tx tx;
903 	struct mlx5dr_cmd_caps caps;
904 };
905 
906 struct mlx5dr_domain {
907 	struct mlx5dr_domain *peer_dmn;
908 	struct mlx5_core_dev *mdev;
909 	u32 pdn;
910 	struct mlx5_uars_page *uar;
911 	enum mlx5dr_domain_type type;
912 	refcount_t refcount;
913 	struct mlx5dr_icm_pool *ste_icm_pool;
914 	struct mlx5dr_icm_pool *action_icm_pool;
915 	struct mlx5dr_send_ring *send_ring;
916 	struct mlx5dr_domain_info info;
917 	struct xarray csum_fts_xa;
918 	struct mlx5dr_ste_ctx *ste_ctx;
919 	struct list_head dbg_tbl_list;
920 	struct mlx5dr_dbg_dump_info dump_info;
921 };
922 
923 struct mlx5dr_table_rx_tx {
924 	struct mlx5dr_ste_htbl *s_anchor;
925 	struct mlx5dr_domain_rx_tx *nic_dmn;
926 	u64 default_icm_addr;
927 	struct list_head nic_matcher_list;
928 };
929 
930 struct mlx5dr_table {
931 	struct mlx5dr_domain *dmn;
932 	struct mlx5dr_table_rx_tx rx;
933 	struct mlx5dr_table_rx_tx tx;
934 	u32 level;
935 	u32 table_type;
936 	u32 table_id;
937 	u32 flags;
938 	struct list_head matcher_list;
939 	struct mlx5dr_action *miss_action;
940 	refcount_t refcount;
941 	struct list_head dbg_node;
942 };
943 
944 struct mlx5dr_matcher_rx_tx {
945 	struct mlx5dr_ste_htbl *s_htbl;
946 	struct mlx5dr_ste_htbl *e_anchor;
947 	struct mlx5dr_ste_build *ste_builder;
948 	struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX]
949 					       [DR_RULE_IPV_MAX]
950 					       [DR_RULE_MAX_STES];
951 	u8 num_of_builders;
952 	u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX];
953 	u64 default_icm_addr;
954 	struct mlx5dr_table_rx_tx *nic_tbl;
955 	u32 prio;
956 	struct list_head list_node;
957 	u32 rules;
958 };
959 
960 struct mlx5dr_matcher {
961 	struct mlx5dr_table *tbl;
962 	struct mlx5dr_matcher_rx_tx rx;
963 	struct mlx5dr_matcher_rx_tx tx;
964 	struct list_head list_node; /* Used for both matchers and dbg managing */
965 	u32 prio;
966 	struct mlx5dr_match_param mask;
967 	u8 match_criteria;
968 	refcount_t refcount;
969 	struct list_head dbg_rule_list;
970 };
971 
972 struct mlx5dr_ste_action_modify_field {
973 	u16 hw_field;
974 	u8 start;
975 	u8 end;
976 	u8 l3_type;
977 	u8 l4_type;
978 };
979 
980 struct mlx5dr_action_rewrite {
981 	struct mlx5dr_domain *dmn;
982 	struct mlx5dr_icm_chunk *chunk;
983 	u8 *data;
984 	u16 num_of_actions;
985 	u32 index;
986 	u8 allow_rx:1;
987 	u8 allow_tx:1;
988 	u8 modify_ttl:1;
989 };
990 
991 struct mlx5dr_action_reformat {
992 	struct mlx5dr_domain *dmn;
993 	u32 id;
994 	u32 size;
995 	u8 param_0;
996 	u8 param_1;
997 };
998 
999 struct mlx5dr_action_sampler {
1000 	struct mlx5dr_domain *dmn;
1001 	u64 rx_icm_addr;
1002 	u64 tx_icm_addr;
1003 	u32 sampler_id;
1004 };
1005 
1006 struct mlx5dr_action_dest_tbl {
1007 	u8 is_fw_tbl:1;
1008 	union {
1009 		struct mlx5dr_table *tbl;
1010 		struct {
1011 			struct mlx5dr_domain *dmn;
1012 			u32 id;
1013 			u32 group_id;
1014 			enum fs_flow_table_type type;
1015 			u64 rx_icm_addr;
1016 			u64 tx_icm_addr;
1017 			struct mlx5dr_action **ref_actions;
1018 			u32 num_of_ref_actions;
1019 		} fw_tbl;
1020 	};
1021 };
1022 
1023 struct mlx5dr_action_ctr {
1024 	u32 ctr_id;
1025 	u32 offset;
1026 };
1027 
1028 struct mlx5dr_action_vport {
1029 	struct mlx5dr_domain *dmn;
1030 	struct mlx5dr_cmd_vport_cap *caps;
1031 };
1032 
1033 struct mlx5dr_action_push_vlan {
1034 	u32 vlan_hdr; /* tpid_pcp_dei_vid */
1035 };
1036 
1037 struct mlx5dr_action_flow_tag {
1038 	u32 flow_tag;
1039 };
1040 
1041 struct mlx5dr_rule_action_member {
1042 	struct mlx5dr_action *action;
1043 	struct list_head list;
1044 };
1045 
1046 struct mlx5dr_action_aso_flow_meter {
1047 	struct mlx5dr_domain *dmn;
1048 	u32 obj_id;
1049 	u32 offset;
1050 	u8 dest_reg_id;
1051 	u8 init_color;
1052 };
1053 
1054 struct mlx5dr_action {
1055 	enum mlx5dr_action_type action_type;
1056 	refcount_t refcount;
1057 
1058 	union {
1059 		void *data;
1060 		struct mlx5dr_action_rewrite *rewrite;
1061 		struct mlx5dr_action_reformat *reformat;
1062 		struct mlx5dr_action_sampler *sampler;
1063 		struct mlx5dr_action_dest_tbl *dest_tbl;
1064 		struct mlx5dr_action_ctr *ctr;
1065 		struct mlx5dr_action_vport *vport;
1066 		struct mlx5dr_action_push_vlan *push_vlan;
1067 		struct mlx5dr_action_flow_tag *flow_tag;
1068 		struct mlx5dr_action_aso_flow_meter *aso;
1069 	};
1070 };
1071 
1072 enum mlx5dr_connect_type {
1073 	CONNECT_HIT	= 1,
1074 	CONNECT_MISS	= 2,
1075 };
1076 
1077 struct mlx5dr_htbl_connect_info {
1078 	enum mlx5dr_connect_type type;
1079 	union {
1080 		struct mlx5dr_ste_htbl *hit_next_htbl;
1081 		u64 miss_icm_addr;
1082 	};
1083 };
1084 
1085 struct mlx5dr_rule_rx_tx {
1086 	struct mlx5dr_matcher_rx_tx *nic_matcher;
1087 	struct mlx5dr_ste *last_rule_ste;
1088 };
1089 
1090 struct mlx5dr_rule {
1091 	struct mlx5dr_matcher *matcher;
1092 	struct mlx5dr_rule_rx_tx rx;
1093 	struct mlx5dr_rule_rx_tx tx;
1094 	struct list_head rule_actions_list;
1095 	struct list_head dbg_node;
1096 	u32 flow_source;
1097 };
1098 
1099 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule,
1100 				 struct mlx5dr_ste *ste,
1101 				 bool force);
1102 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr,
1103 					 struct mlx5dr_ste *curr_ste,
1104 					 int *num_of_stes);
1105 
1106 struct mlx5dr_icm_chunk {
1107 	struct mlx5dr_icm_buddy_mem *buddy_mem;
1108 	struct list_head chunk_list;
1109 
1110 	/* indicates the index of this chunk in the whole memory,
1111 	 * used for deleting the chunk from the buddy
1112 	 */
1113 	unsigned int seg;
1114 	enum mlx5dr_icm_chunk_size size;
1115 
1116 	/* Memory optimisation */
1117 	struct mlx5dr_ste *ste_arr;
1118 	u8 *hw_ste_arr;
1119 	struct list_head *miss_list;
1120 };
1121 
mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx * nic_dmn)1122 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn)
1123 {
1124 	mutex_lock(&nic_dmn->mutex);
1125 }
1126 
mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx * nic_dmn)1127 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn)
1128 {
1129 	mutex_unlock(&nic_dmn->mutex);
1130 }
1131 
mlx5dr_domain_lock(struct mlx5dr_domain * dmn)1132 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn)
1133 {
1134 	mlx5dr_domain_nic_lock(&dmn->info.rx);
1135 	mlx5dr_domain_nic_lock(&dmn->info.tx);
1136 }
1137 
mlx5dr_domain_unlock(struct mlx5dr_domain * dmn)1138 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn)
1139 {
1140 	mlx5dr_domain_nic_unlock(&dmn->info.tx);
1141 	mlx5dr_domain_nic_unlock(&dmn->info.rx);
1142 }
1143 
1144 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn,
1145 				  struct mlx5dr_matcher_rx_tx *nic_matcher);
1146 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn,
1147 				       struct mlx5dr_matcher_rx_tx *nic_matcher);
1148 
1149 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher,
1150 				   struct mlx5dr_matcher_rx_tx *nic_matcher,
1151 				   enum mlx5dr_ipv outer_ipv,
1152 				   enum mlx5dr_ipv inner_ipv);
1153 
1154 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk);
1155 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk);
1156 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk);
1157 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk);
1158 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk);
1159 u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste);
1160 
1161 static inline int
mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)1162 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)
1163 {
1164 	if (icm_type == DR_ICM_TYPE_STE)
1165 		return DR_STE_SIZE;
1166 
1167 	return DR_MODIFY_ACTION_SIZE;
1168 }
1169 
1170 static inline u32
mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)1171 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)
1172 {
1173 	return 1 << chunk_size;
1174 }
1175 
1176 static inline int
mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,enum mlx5dr_icm_type icm_type)1177 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,
1178 				   enum mlx5dr_icm_type icm_type)
1179 {
1180 	int num_of_entries;
1181 	int entry_size;
1182 
1183 	entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type);
1184 	num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
1185 
1186 	return entry_size * num_of_entries;
1187 }
1188 
1189 static inline int
mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl * htbl)1190 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl)
1191 {
1192 	int num_of_entries =
1193 		mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk->size);
1194 
1195 	/* Threshold is 50%, one is added to table of size 1 */
1196 	return (num_of_entries + 1) / 2;
1197 }
1198 
1199 static inline bool
mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl * htbl)1200 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl)
1201 {
1202 	if (htbl->chunk->size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask)
1203 		return false;
1204 
1205 	return true;
1206 }
1207 
1208 struct mlx5dr_cmd_vport_cap *
1209 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport);
1210 
1211 struct mlx5dr_cmd_query_flow_table_details {
1212 	u8 status;
1213 	u8 level;
1214 	u64 sw_owner_icm_root_1;
1215 	u64 sw_owner_icm_root_0;
1216 };
1217 
1218 struct mlx5dr_cmd_create_flow_table_attr {
1219 	u32 table_type;
1220 	u16 uid;
1221 	u64 icm_addr_rx;
1222 	u64 icm_addr_tx;
1223 	u8 level;
1224 	bool sw_owner;
1225 	bool term_tbl;
1226 	bool decap_en;
1227 	bool reformat_en;
1228 };
1229 
1230 /* internal API functions */
1231 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev,
1232 			    struct mlx5dr_cmd_caps *caps);
1233 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev,
1234 				       bool other_vport, u16 vport_number,
1235 				       u64 *icm_address_rx,
1236 				       u64 *icm_address_tx);
1237 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev,
1238 			  bool other_vport, u16 vport_number, u16 *gvmi);
1239 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev,
1240 			      struct mlx5dr_esw_caps *caps);
1241 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev,
1242 				  u32 sampler_id,
1243 				  u64 *rx_icm_addr,
1244 				  u64 *tx_icm_addr);
1245 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev);
1246 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev,
1247 					u32 table_type,
1248 					u32 table_id,
1249 					u32 group_id,
1250 					u32 modify_header_id,
1251 					u16 vport_id);
1252 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev,
1253 				    u32 table_type,
1254 				    u32 table_id);
1255 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev,
1256 				   u32 table_type,
1257 				   u8 num_of_actions,
1258 				   u64 *actions,
1259 				   u32 *modify_header_id);
1260 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev,
1261 				     u32 modify_header_id);
1262 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev,
1263 				       u32 table_type,
1264 				       u32 table_id,
1265 				       u32 *group_id);
1266 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev,
1267 				  u32 table_type,
1268 				  u32 table_id,
1269 				  u32 group_id);
1270 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev,
1271 				 struct mlx5dr_cmd_create_flow_table_attr *attr,
1272 				 u64 *fdb_rx_icm_addr,
1273 				 u32 *table_id);
1274 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev,
1275 				  u32 table_id,
1276 				  u32 table_type);
1277 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev,
1278 				enum fs_flow_table_type type,
1279 				u32 table_id,
1280 				struct mlx5dr_cmd_query_flow_table_details *output);
1281 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev,
1282 				   enum mlx5_reformat_ctx_type rt,
1283 				   u8 reformat_param_0,
1284 				   u8 reformat_param_1,
1285 				   size_t reformat_size,
1286 				   void *reformat_data,
1287 				   u32 *reformat_id);
1288 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev,
1289 				     u32 reformat_id);
1290 
1291 struct mlx5dr_cmd_gid_attr {
1292 	u8 gid[16];
1293 	u8 mac[6];
1294 	u32 roce_ver;
1295 };
1296 
1297 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
1298 			 u16 index, struct mlx5dr_cmd_gid_attr *attr);
1299 
1300 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
1301 					       enum mlx5dr_icm_type icm_type);
1302 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool);
1303 
1304 struct mlx5dr_icm_chunk *
1305 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
1306 		       enum mlx5dr_icm_chunk_size chunk_size);
1307 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk);
1308 
1309 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx,
1310 				     u8 *hw_ste_p, u32 ste_size);
1311 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn,
1312 				      struct mlx5dr_domain_rx_tx *nic_dmn,
1313 				      struct mlx5dr_ste_htbl *htbl,
1314 				      struct mlx5dr_htbl_connect_info *connect_info,
1315 				      bool update_hw_ste);
1316 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx,
1317 				  u16 gvmi,
1318 				  enum mlx5dr_domain_nic_type nic_type,
1319 				  struct mlx5dr_ste_htbl *htbl,
1320 				  u8 *formatted_ste,
1321 				  struct mlx5dr_htbl_connect_info *connect_info);
1322 void mlx5dr_ste_copy_param(u8 match_criteria,
1323 			   struct mlx5dr_match_param *set_param,
1324 			   struct mlx5dr_match_parameters *mask,
1325 			   bool clear);
1326 
1327 struct mlx5dr_qp {
1328 	struct mlx5_core_dev *mdev;
1329 	struct mlx5_wq_qp wq;
1330 	struct mlx5_uars_page *uar;
1331 	struct mlx5_wq_ctrl wq_ctrl;
1332 	u32 qpn;
1333 	struct {
1334 		unsigned int pc;
1335 		unsigned int cc;
1336 		unsigned int size;
1337 		unsigned int *wqe_head;
1338 		unsigned int wqe_cnt;
1339 	} sq;
1340 	struct {
1341 		unsigned int pc;
1342 		unsigned int cc;
1343 		unsigned int size;
1344 		unsigned int wqe_cnt;
1345 	} rq;
1346 	int max_inline_data;
1347 };
1348 
1349 struct mlx5dr_cq {
1350 	struct mlx5_core_dev *mdev;
1351 	struct mlx5_cqwq wq;
1352 	struct mlx5_wq_ctrl wq_ctrl;
1353 	struct mlx5_core_cq mcq;
1354 	struct mlx5dr_qp *qp;
1355 };
1356 
1357 struct mlx5dr_mr {
1358 	struct mlx5_core_dev *mdev;
1359 	u32 mkey;
1360 	dma_addr_t dma_addr;
1361 	void *addr;
1362 	size_t size;
1363 };
1364 
1365 #define MAX_SEND_CQE		64
1366 #define MIN_READ_SYNC		64
1367 
1368 struct mlx5dr_send_ring {
1369 	struct mlx5dr_cq *cq;
1370 	struct mlx5dr_qp *qp;
1371 	struct mlx5dr_mr *mr;
1372 	/* How much wqes are waiting for completion */
1373 	u32 pending_wqe;
1374 	/* Signal request per this trash hold value */
1375 	u16 signal_th;
1376 	/* Each post_send_size less than max_post_send_size */
1377 	u32 max_post_send_size;
1378 	/* manage the send queue */
1379 	u32 tx_head;
1380 	void *buf;
1381 	u32 buf_size;
1382 	u8 sync_buff[MIN_READ_SYNC];
1383 	struct mlx5dr_mr *sync_mr;
1384 	spinlock_t lock; /* Protect the data path of the send ring */
1385 	bool err_state; /* send_ring is not usable in err state */
1386 };
1387 
1388 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn);
1389 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn,
1390 			   struct mlx5dr_send_ring *send_ring);
1391 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn);
1392 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn,
1393 			     struct mlx5dr_ste *ste,
1394 			     u8 *data,
1395 			     u16 size,
1396 			     u16 offset);
1397 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn,
1398 			      struct mlx5dr_ste_htbl *htbl,
1399 			      u8 *formatted_ste, u8 *mask);
1400 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn,
1401 					struct mlx5dr_ste_htbl *htbl,
1402 					u8 *ste_init_data,
1403 					bool update_hw_ste);
1404 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn,
1405 				struct mlx5dr_action *action);
1406 
1407 struct mlx5dr_cmd_ft_info {
1408 	u32 id;
1409 	u16 vport;
1410 	enum fs_flow_table_type type;
1411 };
1412 
1413 struct mlx5dr_cmd_flow_destination_hw_info {
1414 	enum mlx5_flow_destination_type type;
1415 	union {
1416 		u32 tir_num;
1417 		u32 ft_num;
1418 		u32 ft_id;
1419 		u32 counter_id;
1420 		u32 sampler_id;
1421 		struct {
1422 			u16 num;
1423 			u16 vhca_id;
1424 			u32 reformat_id;
1425 			u8 flags;
1426 		} vport;
1427 	};
1428 };
1429 
1430 struct mlx5dr_cmd_fte_info {
1431 	u32 dests_size;
1432 	u32 index;
1433 	struct mlx5_flow_context flow_context;
1434 	u32 *val;
1435 	struct mlx5_flow_act action;
1436 	struct mlx5dr_cmd_flow_destination_hw_info *dest_arr;
1437 	bool ignore_flow_level;
1438 };
1439 
1440 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev,
1441 		       int opmod, int modify_mask,
1442 		       struct mlx5dr_cmd_ft_info *ft,
1443 		       u32 group_id,
1444 		       struct mlx5dr_cmd_fte_info *fte);
1445 
1446 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps);
1447 
1448 struct mlx5dr_fw_recalc_cs_ft {
1449 	u64 rx_icm_addr;
1450 	u32 table_id;
1451 	u32 group_id;
1452 	u32 modify_hdr_id;
1453 };
1454 
1455 struct mlx5dr_fw_recalc_cs_ft *
1456 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num);
1457 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn,
1458 				    struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft);
1459 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn,
1460 					u16 vport_num,
1461 					u64 *rx_icm_addr);
1462 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn,
1463 			    struct mlx5dr_cmd_flow_destination_hw_info *dest,
1464 			    int num_dest,
1465 			    bool reformat_req,
1466 			    u32 *tbl_id,
1467 			    u32 *group_id,
1468 			    bool ignore_flow_level,
1469 			    u32 flow_source);
1470 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id,
1471 			      u32 group_id);
1472 #endif  /* _DR_TYPES_H_ */
1473