1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2019, Mellanox Technologies */
3
4 #ifndef _DR_TYPES_
5 #define _DR_TYPES_
6
7 #include <linux/mlx5/vport.h>
8 #include <linux/refcount.h>
9 #include "fs_core.h"
10 #include "wq.h"
11 #include "lib/mlx5.h"
12 #include "mlx5_ifc_dr.h"
13 #include "mlx5dr.h"
14 #include "dr_dbg.h"
15
16 #define DR_RULE_MAX_STES 18
17 #define DR_ACTION_MAX_STES 5
18 #define DR_STE_SVLAN 0x1
19 #define DR_STE_CVLAN 0x2
20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4)
21 #define DR_NUM_OF_FLEX_PARSERS 8
22 #define DR_STE_MAX_FLEX_0_ID 3
23 #define DR_STE_MAX_FLEX_1_ID 7
24
25 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg)
26 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg)
27 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg)
28
dr_is_flex_parser_0_id(u8 parser_id)29 static inline bool dr_is_flex_parser_0_id(u8 parser_id)
30 {
31 return parser_id <= DR_STE_MAX_FLEX_0_ID;
32 }
33
dr_is_flex_parser_1_id(u8 parser_id)34 static inline bool dr_is_flex_parser_1_id(u8 parser_id)
35 {
36 return parser_id > DR_STE_MAX_FLEX_0_ID;
37 }
38
39 enum mlx5dr_icm_chunk_size {
40 DR_CHUNK_SIZE_1,
41 DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */
42 DR_CHUNK_SIZE_2,
43 DR_CHUNK_SIZE_4,
44 DR_CHUNK_SIZE_8,
45 DR_CHUNK_SIZE_16,
46 DR_CHUNK_SIZE_32,
47 DR_CHUNK_SIZE_64,
48 DR_CHUNK_SIZE_128,
49 DR_CHUNK_SIZE_256,
50 DR_CHUNK_SIZE_512,
51 DR_CHUNK_SIZE_1K,
52 DR_CHUNK_SIZE_2K,
53 DR_CHUNK_SIZE_4K,
54 DR_CHUNK_SIZE_8K,
55 DR_CHUNK_SIZE_16K,
56 DR_CHUNK_SIZE_32K,
57 DR_CHUNK_SIZE_64K,
58 DR_CHUNK_SIZE_128K,
59 DR_CHUNK_SIZE_256K,
60 DR_CHUNK_SIZE_512K,
61 DR_CHUNK_SIZE_1024K,
62 DR_CHUNK_SIZE_2048K,
63 DR_CHUNK_SIZE_MAX,
64 };
65
66 enum mlx5dr_icm_type {
67 DR_ICM_TYPE_STE,
68 DR_ICM_TYPE_MODIFY_ACTION,
69 };
70
71 static inline enum mlx5dr_icm_chunk_size
mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)72 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)
73 {
74 chunk += 2;
75 if (chunk < DR_CHUNK_SIZE_MAX)
76 return chunk;
77
78 return DR_CHUNK_SIZE_MAX;
79 }
80
81 enum {
82 DR_STE_SIZE = 64,
83 DR_STE_SIZE_CTRL = 32,
84 DR_STE_SIZE_TAG = 16,
85 DR_STE_SIZE_MASK = 16,
86 DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK,
87 };
88
89 enum mlx5dr_ste_ctx_action_cap {
90 DR_STE_CTX_ACTION_CAP_NONE = 0,
91 DR_STE_CTX_ACTION_CAP_TX_POP = 1 << 0,
92 DR_STE_CTX_ACTION_CAP_RX_PUSH = 1 << 1,
93 DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2,
94 DR_STE_CTX_ACTION_CAP_POP_MDFY = 1 << 3,
95 };
96
97 enum {
98 DR_MODIFY_ACTION_SIZE = 8,
99 };
100
101 enum mlx5dr_matcher_criteria {
102 DR_MATCHER_CRITERIA_EMPTY = 0,
103 DR_MATCHER_CRITERIA_OUTER = 1 << 0,
104 DR_MATCHER_CRITERIA_MISC = 1 << 1,
105 DR_MATCHER_CRITERIA_INNER = 1 << 2,
106 DR_MATCHER_CRITERIA_MISC2 = 1 << 3,
107 DR_MATCHER_CRITERIA_MISC3 = 1 << 4,
108 DR_MATCHER_CRITERIA_MISC4 = 1 << 5,
109 DR_MATCHER_CRITERIA_MISC5 = 1 << 6,
110 DR_MATCHER_CRITERIA_MAX = 1 << 7,
111 };
112
113 enum mlx5dr_action_type {
114 DR_ACTION_TYP_TNL_L2_TO_L2,
115 DR_ACTION_TYP_L2_TO_TNL_L2,
116 DR_ACTION_TYP_TNL_L3_TO_L2,
117 DR_ACTION_TYP_L2_TO_TNL_L3,
118 DR_ACTION_TYP_DROP,
119 DR_ACTION_TYP_QP,
120 DR_ACTION_TYP_FT,
121 DR_ACTION_TYP_CTR,
122 DR_ACTION_TYP_TAG,
123 DR_ACTION_TYP_MODIFY_HDR,
124 DR_ACTION_TYP_VPORT,
125 DR_ACTION_TYP_POP_VLAN,
126 DR_ACTION_TYP_PUSH_VLAN,
127 DR_ACTION_TYP_INSERT_HDR,
128 DR_ACTION_TYP_REMOVE_HDR,
129 DR_ACTION_TYP_SAMPLER,
130 DR_ACTION_TYP_MAX,
131 };
132
133 enum mlx5dr_ipv {
134 DR_RULE_IPV4,
135 DR_RULE_IPV6,
136 DR_RULE_IPV_MAX,
137 };
138
139 struct mlx5dr_icm_pool;
140 struct mlx5dr_icm_chunk;
141 struct mlx5dr_icm_buddy_mem;
142 struct mlx5dr_ste_htbl;
143 struct mlx5dr_match_param;
144 struct mlx5dr_cmd_caps;
145 struct mlx5dr_rule_rx_tx;
146 struct mlx5dr_matcher_rx_tx;
147 struct mlx5dr_ste_ctx;
148
149 struct mlx5dr_ste {
150 /* refcount: indicates the num of rules that using this ste */
151 u32 refcount;
152
153 /* this ste is part of a rule, located in ste's chain */
154 u8 ste_chain_location;
155
156 /* attached to the miss_list head at each htbl entry */
157 struct list_head miss_list_node;
158
159 /* this ste is member of htbl */
160 struct mlx5dr_ste_htbl *htbl;
161
162 struct mlx5dr_ste_htbl *next_htbl;
163
164 /* The rule this STE belongs to */
165 struct mlx5dr_rule_rx_tx *rule_rx_tx;
166 };
167
168 struct mlx5dr_ste_htbl_ctrl {
169 /* total number of valid entries belonging to this hash table. This
170 * includes the non collision and collision entries
171 */
172 unsigned int num_of_valid_entries;
173
174 /* total number of collisions entries attached to this table */
175 unsigned int num_of_collisions;
176 };
177
178 struct mlx5dr_ste_htbl {
179 u16 lu_type;
180 u16 byte_mask;
181 u32 refcount;
182 struct mlx5dr_icm_chunk *chunk;
183 struct mlx5dr_ste *pointing_ste;
184 struct mlx5dr_ste_htbl_ctrl ctrl;
185 };
186
187 struct mlx5dr_ste_send_info {
188 struct mlx5dr_ste *ste;
189 struct list_head send_list;
190 u16 size;
191 u16 offset;
192 u8 data_cont[DR_STE_SIZE];
193 u8 *data;
194 };
195
196 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size,
197 u16 offset, u8 *data,
198 struct mlx5dr_ste_send_info *ste_info,
199 struct list_head *send_list,
200 bool copy_data);
201
202 struct mlx5dr_ste_build {
203 u8 inner:1;
204 u8 rx:1;
205 u8 vhca_id_valid:1;
206 struct mlx5dr_domain *dmn;
207 struct mlx5dr_cmd_caps *caps;
208 u16 lu_type;
209 u16 byte_mask;
210 u8 bit_mask[DR_STE_SIZE_MASK];
211 int (*ste_build_tag_func)(struct mlx5dr_match_param *spec,
212 struct mlx5dr_ste_build *sb,
213 u8 *tag);
214 };
215
216 struct mlx5dr_ste_htbl *
217 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool,
218 enum mlx5dr_icm_chunk_size chunk_size,
219 u16 lu_type, u16 byte_mask);
220
221 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl);
222
mlx5dr_htbl_put(struct mlx5dr_ste_htbl * htbl)223 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl)
224 {
225 htbl->refcount--;
226 if (!htbl->refcount)
227 mlx5dr_ste_htbl_free(htbl);
228 }
229
mlx5dr_htbl_get(struct mlx5dr_ste_htbl * htbl)230 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl)
231 {
232 htbl->refcount++;
233 }
234
235 /* STE utils */
236 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl);
237 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx,
238 u8 *hw_ste, u64 miss_addr);
239 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx,
240 u8 *hw_ste, u64 icm_addr, u32 ht_size);
241 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx,
242 u8 *hw_ste,
243 struct mlx5dr_ste_htbl *next_htbl);
244 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask);
245 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher,
246 u8 ste_location);
247 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste);
248 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste);
249 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste);
250
251 #define MLX5DR_MAX_VLANS 2
252
253 struct mlx5dr_ste_actions_attr {
254 u32 modify_index;
255 u16 modify_actions;
256 u32 decap_index;
257 u16 decap_actions;
258 u8 decap_with_vlan:1;
259 u64 final_icm_addr;
260 u32 flow_tag;
261 u32 ctr_id;
262 u16 gvmi;
263 u16 hit_gvmi;
264 struct {
265 u32 id;
266 u32 size;
267 u8 param_0;
268 u8 param_1;
269 } reformat;
270 struct {
271 int count;
272 u32 headers[MLX5DR_MAX_VLANS];
273 } vlans;
274 };
275
276 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx,
277 struct mlx5dr_domain *dmn,
278 u8 *action_type_set,
279 u8 *last_ste,
280 struct mlx5dr_ste_actions_attr *attr,
281 u32 *added_stes);
282 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx,
283 struct mlx5dr_domain *dmn,
284 u8 *action_type_set,
285 u8 *last_ste,
286 struct mlx5dr_ste_actions_attr *attr,
287 u32 *added_stes);
288
289 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx,
290 __be64 *hw_action,
291 u8 hw_field,
292 u8 shifter,
293 u8 length,
294 u32 data);
295 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx,
296 __be64 *hw_action,
297 u8 hw_field,
298 u8 shifter,
299 u8 length,
300 u32 data);
301 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx,
302 __be64 *hw_action,
303 u8 dst_hw_field,
304 u8 dst_shifter,
305 u8 dst_len,
306 u8 src_hw_field,
307 u8 src_shifter);
308 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
309 void *data,
310 u32 data_sz,
311 u8 *hw_action,
312 u32 hw_action_sz,
313 u16 *used_hw_action_num);
314
315 const struct mlx5dr_ste_action_modify_field *
316 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field);
317
318 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version);
319 void mlx5dr_ste_free(struct mlx5dr_ste *ste,
320 struct mlx5dr_matcher *matcher,
321 struct mlx5dr_matcher_rx_tx *nic_matcher);
mlx5dr_ste_put(struct mlx5dr_ste * ste,struct mlx5dr_matcher * matcher,struct mlx5dr_matcher_rx_tx * nic_matcher)322 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste,
323 struct mlx5dr_matcher *matcher,
324 struct mlx5dr_matcher_rx_tx *nic_matcher)
325 {
326 ste->refcount--;
327 if (!ste->refcount)
328 mlx5dr_ste_free(ste, matcher, nic_matcher);
329 }
330
331 /* initial as 0, increased only when ste appears in a new rule */
mlx5dr_ste_get(struct mlx5dr_ste * ste)332 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste)
333 {
334 ste->refcount++;
335 }
336
mlx5dr_ste_is_not_used(struct mlx5dr_ste * ste)337 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste)
338 {
339 return !ste->refcount;
340 }
341
342 bool mlx5dr_ste_equal_tag(void *src, void *dst);
343 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher,
344 struct mlx5dr_matcher_rx_tx *nic_matcher,
345 struct mlx5dr_ste *ste,
346 u8 *cur_hw_ste,
347 enum mlx5dr_icm_chunk_size log_table_size);
348
349 /* STE build functions */
350 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn,
351 u8 match_criteria,
352 struct mlx5dr_match_param *mask,
353 struct mlx5dr_match_param *value);
354 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher,
355 struct mlx5dr_matcher_rx_tx *nic_matcher,
356 struct mlx5dr_match_param *value,
357 u8 *ste_arr);
358 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx,
359 struct mlx5dr_ste_build *builder,
360 struct mlx5dr_match_param *mask,
361 bool inner, bool rx);
362 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx,
363 struct mlx5dr_ste_build *sb,
364 struct mlx5dr_match_param *mask,
365 bool inner, bool rx);
366 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx,
367 struct mlx5dr_ste_build *sb,
368 struct mlx5dr_match_param *mask,
369 bool inner, bool rx);
370 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx,
371 struct mlx5dr_ste_build *sb,
372 struct mlx5dr_match_param *mask,
373 bool inner, bool rx);
374 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx,
375 struct mlx5dr_ste_build *sb,
376 struct mlx5dr_match_param *mask,
377 bool inner, bool rx);
378 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx,
379 struct mlx5dr_ste_build *sb,
380 struct mlx5dr_match_param *mask,
381 bool inner, bool rx);
382 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx,
383 struct mlx5dr_ste_build *sb,
384 struct mlx5dr_match_param *mask,
385 bool inner, bool rx);
386 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx,
387 struct mlx5dr_ste_build *sb,
388 struct mlx5dr_match_param *mask,
389 bool inner, bool rx);
390 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx,
391 struct mlx5dr_ste_build *sb,
392 struct mlx5dr_match_param *mask,
393 bool inner, bool rx);
394 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx,
395 struct mlx5dr_ste_build *sb,
396 struct mlx5dr_match_param *mask,
397 bool inner, bool rx);
398 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx,
399 struct mlx5dr_ste_build *sb,
400 struct mlx5dr_match_param *mask,
401 bool inner, bool rx);
402 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx,
403 struct mlx5dr_ste_build *sb,
404 struct mlx5dr_match_param *mask,
405 bool inner, bool rx);
406 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx,
407 struct mlx5dr_ste_build *sb,
408 struct mlx5dr_match_param *mask,
409 bool inner, bool rx);
410 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx,
411 struct mlx5dr_ste_build *sb,
412 struct mlx5dr_match_param *mask,
413 struct mlx5dr_cmd_caps *caps,
414 bool inner, bool rx);
415 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx,
416 struct mlx5dr_ste_build *sb,
417 struct mlx5dr_match_param *mask,
418 struct mlx5dr_cmd_caps *caps,
419 bool inner, bool rx);
420 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx,
421 struct mlx5dr_ste_build *sb,
422 struct mlx5dr_match_param *mask,
423 struct mlx5dr_cmd_caps *caps,
424 bool inner, bool rx);
425 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx,
426 struct mlx5dr_ste_build *sb,
427 struct mlx5dr_match_param *mask,
428 bool inner, bool rx);
429 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx,
430 struct mlx5dr_ste_build *sb,
431 struct mlx5dr_match_param *mask,
432 bool inner, bool rx);
433 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx,
434 struct mlx5dr_ste_build *sb,
435 struct mlx5dr_match_param *mask,
436 struct mlx5dr_cmd_caps *caps,
437 bool inner, bool rx);
438 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx,
439 struct mlx5dr_ste_build *sb,
440 struct mlx5dr_match_param *mask,
441 struct mlx5dr_cmd_caps *caps,
442 bool inner, bool rx);
443 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx,
444 struct mlx5dr_ste_build *sb,
445 struct mlx5dr_match_param *mask,
446 bool inner, bool rx);
447 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
448 struct mlx5dr_ste_build *sb,
449 struct mlx5dr_match_param *mask,
450 struct mlx5dr_cmd_caps *caps,
451 bool inner, bool rx);
452 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
453 struct mlx5dr_ste_build *sb,
454 struct mlx5dr_match_param *mask,
455 struct mlx5dr_cmd_caps *caps,
456 bool inner, bool rx);
457 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx,
458 struct mlx5dr_ste_build *sb,
459 struct mlx5dr_match_param *mask,
460 bool inner, bool rx);
461 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx,
462 struct mlx5dr_ste_build *sb,
463 struct mlx5dr_match_param *mask,
464 bool inner, bool rx);
465 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx,
466 struct mlx5dr_ste_build *sb,
467 struct mlx5dr_match_param *mask,
468 bool inner, bool rx);
469 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx,
470 struct mlx5dr_ste_build *sb,
471 struct mlx5dr_match_param *mask,
472 bool inner, bool rx);
473 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx,
474 struct mlx5dr_ste_build *sb,
475 struct mlx5dr_match_param *mask,
476 struct mlx5dr_domain *dmn,
477 bool inner, bool rx);
478 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
479 struct mlx5dr_ste_build *sb,
480 struct mlx5dr_match_param *mask,
481 bool inner, bool rx);
482 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
483 struct mlx5dr_ste_build *sb,
484 struct mlx5dr_match_param *mask,
485 bool inner, bool rx);
486 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx);
487
488 /* Actions utils */
489 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher,
490 struct mlx5dr_matcher_rx_tx *nic_matcher,
491 struct mlx5dr_action *actions[],
492 u32 num_actions,
493 u8 *ste_arr,
494 u32 *new_hw_ste_arr_sz);
495
496 struct mlx5dr_match_spec {
497 u32 smac_47_16; /* Source MAC address of incoming packet */
498 /* Incoming packet Ethertype - this is the Ethertype
499 * following the last VLAN tag of the packet
500 */
501 u32 smac_15_0:16; /* Source MAC address of incoming packet */
502 u32 ethertype:16;
503
504 u32 dmac_47_16; /* Destination MAC address of incoming packet */
505
506 u32 dmac_15_0:16; /* Destination MAC address of incoming packet */
507 /* Priority of first VLAN tag in the incoming packet.
508 * Valid only when cvlan_tag==1 or svlan_tag==1
509 */
510 u32 first_prio:3;
511 /* CFI bit of first VLAN tag in the incoming packet.
512 * Valid only when cvlan_tag==1 or svlan_tag==1
513 */
514 u32 first_cfi:1;
515 /* VLAN ID of first VLAN tag in the incoming packet.
516 * Valid only when cvlan_tag==1 or svlan_tag==1
517 */
518 u32 first_vid:12;
519
520 u32 ip_protocol:8; /* IP protocol */
521 /* Differentiated Services Code Point derived from
522 * Traffic Class/TOS field of IPv6/v4
523 */
524 u32 ip_dscp:6;
525 /* Explicit Congestion Notification derived from
526 * Traffic Class/TOS field of IPv6/v4
527 */
528 u32 ip_ecn:2;
529 /* The first vlan in the packet is c-vlan (0x8100).
530 * cvlan_tag and svlan_tag cannot be set together
531 */
532 u32 cvlan_tag:1;
533 /* The first vlan in the packet is s-vlan (0x8a88).
534 * cvlan_tag and svlan_tag cannot be set together
535 */
536 u32 svlan_tag:1;
537 u32 frag:1; /* Packet is an IP fragment */
538 u32 ip_version:4; /* IP version */
539 /* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK;
540 * Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS
541 */
542 u32 tcp_flags:9;
543
544 /* TCP source port.;tcp and udp sport/dport are mutually exclusive */
545 u32 tcp_sport:16;
546 /* TCP destination port.
547 * tcp and udp sport/dport are mutually exclusive
548 */
549 u32 tcp_dport:16;
550
551 u32 reserved_auto1:16;
552 u32 ipv4_ihl:4;
553 u32 reserved_auto2:4;
554 u32 ttl_hoplimit:8;
555
556 /* UDP source port.;tcp and udp sport/dport are mutually exclusive */
557 u32 udp_sport:16;
558 /* UDP destination port.;tcp and udp sport/dport are mutually exclusive */
559 u32 udp_dport:16;
560
561 /* IPv6 source address of incoming packets
562 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
563 * This field should be qualified by an appropriate ethertype
564 */
565 u32 src_ip_127_96;
566 /* IPv6 source address of incoming packets
567 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
568 * This field should be qualified by an appropriate ethertype
569 */
570 u32 src_ip_95_64;
571 /* IPv6 source address of incoming packets
572 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
573 * This field should be qualified by an appropriate ethertype
574 */
575 u32 src_ip_63_32;
576 /* IPv6 source address of incoming packets
577 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
578 * This field should be qualified by an appropriate ethertype
579 */
580 u32 src_ip_31_0;
581 /* IPv6 destination address of incoming packets
582 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
583 * This field should be qualified by an appropriate ethertype
584 */
585 u32 dst_ip_127_96;
586 /* IPv6 destination address of incoming packets
587 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
588 * This field should be qualified by an appropriate ethertype
589 */
590 u32 dst_ip_95_64;
591 /* IPv6 destination address of incoming packets
592 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
593 * This field should be qualified by an appropriate ethertype
594 */
595 u32 dst_ip_63_32;
596 /* IPv6 destination address of incoming packets
597 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
598 * This field should be qualified by an appropriate ethertype
599 */
600 u32 dst_ip_31_0;
601 };
602
603 struct mlx5dr_match_misc {
604 /* used with GRE, checksum exist when gre_c_present == 1 */
605 u32 gre_c_present:1;
606 u32 reserved_auto1:1;
607 /* used with GRE, key exist when gre_k_present == 1 */
608 u32 gre_k_present:1;
609 /* used with GRE, sequence number exist when gre_s_present == 1 */
610 u32 gre_s_present:1;
611 u32 source_vhca_port:4;
612 u32 source_sqn:24; /* Source SQN */
613
614 u32 source_eswitch_owner_vhca_id:16;
615 /* Source port.;0xffff determines wire port */
616 u32 source_port:16;
617
618 /* Priority of second VLAN tag in the outer header of the incoming packet.
619 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
620 */
621 u32 outer_second_prio:3;
622 /* CFI bit of first VLAN tag in the outer header of the incoming packet.
623 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
624 */
625 u32 outer_second_cfi:1;
626 /* VLAN ID of first VLAN tag the outer header of the incoming packet.
627 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
628 */
629 u32 outer_second_vid:12;
630 /* Priority of second VLAN tag in the inner header of the incoming packet.
631 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
632 */
633 u32 inner_second_prio:3;
634 /* CFI bit of first VLAN tag in the inner header of the incoming packet.
635 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
636 */
637 u32 inner_second_cfi:1;
638 /* VLAN ID of first VLAN tag the inner header of the incoming packet.
639 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
640 */
641 u32 inner_second_vid:12;
642
643 u32 outer_second_cvlan_tag:1;
644 u32 inner_second_cvlan_tag:1;
645 /* The second vlan in the outer header of the packet is c-vlan (0x8100).
646 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
647 */
648 u32 outer_second_svlan_tag:1;
649 /* The second vlan in the inner header of the packet is c-vlan (0x8100).
650 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
651 */
652 u32 inner_second_svlan_tag:1;
653 /* The second vlan in the outer header of the packet is s-vlan (0x8a88).
654 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
655 */
656 u32 reserved_auto2:12;
657 /* The second vlan in the inner header of the packet is s-vlan (0x8a88).
658 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
659 */
660 u32 gre_protocol:16; /* GRE Protocol (outer) */
661
662 u32 gre_key_h:24; /* GRE Key[31:8] (outer) */
663 u32 gre_key_l:8; /* GRE Key [7:0] (outer) */
664
665 u32 vxlan_vni:24; /* VXLAN VNI (outer) */
666 u32 reserved_auto3:8;
667
668 u32 geneve_vni:24; /* GENEVE VNI field (outer) */
669 u32 reserved_auto4:6;
670 u32 geneve_tlv_option_0_exist:1;
671 u32 geneve_oam:1; /* GENEVE OAM field (outer) */
672
673 u32 reserved_auto5:12;
674 u32 outer_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (outer) */
675
676 u32 reserved_auto6:12;
677 u32 inner_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (inner) */
678
679 u32 reserved_auto7:10;
680 u32 geneve_opt_len:6; /* GENEVE OptLen (outer) */
681 u32 geneve_protocol_type:16; /* GENEVE protocol type (outer) */
682
683 u32 reserved_auto8:8;
684 u32 bth_dst_qp:24; /* Destination QP in BTH header */
685
686 u32 reserved_auto9;
687 u32 outer_esp_spi;
688 u32 reserved_auto10[3];
689 };
690
691 struct mlx5dr_match_misc2 {
692 u32 outer_first_mpls_label:20; /* First MPLS LABEL (outer) */
693 u32 outer_first_mpls_exp:3; /* First MPLS EXP (outer) */
694 u32 outer_first_mpls_s_bos:1; /* First MPLS S_BOS (outer) */
695 u32 outer_first_mpls_ttl:8; /* First MPLS TTL (outer) */
696
697 u32 inner_first_mpls_label:20; /* First MPLS LABEL (inner) */
698 u32 inner_first_mpls_exp:3; /* First MPLS EXP (inner) */
699 u32 inner_first_mpls_s_bos:1; /* First MPLS S_BOS (inner) */
700 u32 inner_first_mpls_ttl:8; /* First MPLS TTL (inner) */
701
702 u32 outer_first_mpls_over_gre_label:20; /* last MPLS LABEL (outer) */
703 u32 outer_first_mpls_over_gre_exp:3; /* last MPLS EXP (outer) */
704 u32 outer_first_mpls_over_gre_s_bos:1; /* last MPLS S_BOS (outer) */
705 u32 outer_first_mpls_over_gre_ttl:8; /* last MPLS TTL (outer) */
706
707 u32 outer_first_mpls_over_udp_label:20; /* last MPLS LABEL (outer) */
708 u32 outer_first_mpls_over_udp_exp:3; /* last MPLS EXP (outer) */
709 u32 outer_first_mpls_over_udp_s_bos:1; /* last MPLS S_BOS (outer) */
710 u32 outer_first_mpls_over_udp_ttl:8; /* last MPLS TTL (outer) */
711
712 u32 metadata_reg_c_7; /* metadata_reg_c_7 */
713 u32 metadata_reg_c_6; /* metadata_reg_c_6 */
714 u32 metadata_reg_c_5; /* metadata_reg_c_5 */
715 u32 metadata_reg_c_4; /* metadata_reg_c_4 */
716 u32 metadata_reg_c_3; /* metadata_reg_c_3 */
717 u32 metadata_reg_c_2; /* metadata_reg_c_2 */
718 u32 metadata_reg_c_1; /* metadata_reg_c_1 */
719 u32 metadata_reg_c_0; /* metadata_reg_c_0 */
720 u32 metadata_reg_a; /* metadata_reg_a */
721 u32 reserved_auto1[3];
722 };
723
724 struct mlx5dr_match_misc3 {
725 u32 inner_tcp_seq_num;
726 u32 outer_tcp_seq_num;
727 u32 inner_tcp_ack_num;
728 u32 outer_tcp_ack_num;
729
730 u32 reserved_auto1:8;
731 u32 outer_vxlan_gpe_vni:24;
732
733 u32 outer_vxlan_gpe_next_protocol:8;
734 u32 outer_vxlan_gpe_flags:8;
735 u32 reserved_auto2:16;
736
737 u32 icmpv4_header_data;
738 u32 icmpv6_header_data;
739
740 u8 icmpv4_type;
741 u8 icmpv4_code;
742 u8 icmpv6_type;
743 u8 icmpv6_code;
744
745 u32 geneve_tlv_option_0_data;
746
747 u32 gtpu_teid;
748
749 u8 gtpu_msg_type;
750 u8 gtpu_msg_flags;
751 u32 reserved_auto3:16;
752
753 u32 gtpu_dw_2;
754 u32 gtpu_first_ext_dw_0;
755 u32 gtpu_dw_0;
756 u32 reserved_auto4;
757 };
758
759 struct mlx5dr_match_misc4 {
760 u32 prog_sample_field_value_0;
761 u32 prog_sample_field_id_0;
762 u32 prog_sample_field_value_1;
763 u32 prog_sample_field_id_1;
764 u32 prog_sample_field_value_2;
765 u32 prog_sample_field_id_2;
766 u32 prog_sample_field_value_3;
767 u32 prog_sample_field_id_3;
768 u32 reserved_auto1[8];
769 };
770
771 struct mlx5dr_match_misc5 {
772 u32 macsec_tag_0;
773 u32 macsec_tag_1;
774 u32 macsec_tag_2;
775 u32 macsec_tag_3;
776 u32 tunnel_header_0;
777 u32 tunnel_header_1;
778 u32 tunnel_header_2;
779 u32 tunnel_header_3;
780 };
781
782 struct mlx5dr_match_param {
783 struct mlx5dr_match_spec outer;
784 struct mlx5dr_match_misc misc;
785 struct mlx5dr_match_spec inner;
786 struct mlx5dr_match_misc2 misc2;
787 struct mlx5dr_match_misc3 misc3;
788 struct mlx5dr_match_misc4 misc4;
789 struct mlx5dr_match_misc5 misc5;
790 };
791
792 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \
793 (_misc3)->icmpv4_code || \
794 (_misc3)->icmpv4_header_data)
795
796 #define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \
797 (_spec)->src_ip_95_64 || \
798 (_spec)->src_ip_63_32 || \
799 (_spec)->src_ip_31_0)
800
801 #define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \
802 (_spec)->dst_ip_95_64 || \
803 (_spec)->dst_ip_63_32 || \
804 (_spec)->dst_ip_31_0)
805
806 struct mlx5dr_esw_caps {
807 u64 drop_icm_address_rx;
808 u64 drop_icm_address_tx;
809 u64 uplink_icm_address_rx;
810 u64 uplink_icm_address_tx;
811 u8 sw_owner:1;
812 u8 sw_owner_v2:1;
813 };
814
815 struct mlx5dr_cmd_vport_cap {
816 u16 vport_gvmi;
817 u16 vhca_gvmi;
818 u16 num;
819 u64 icm_address_rx;
820 u64 icm_address_tx;
821 };
822
823 struct mlx5dr_roce_cap {
824 u8 roce_en:1;
825 u8 fl_rc_qp_when_roce_disabled:1;
826 u8 fl_rc_qp_when_roce_enabled:1;
827 };
828
829 struct mlx5dr_vports {
830 struct mlx5dr_cmd_vport_cap esw_manager_caps;
831 struct mlx5dr_cmd_vport_cap uplink_caps;
832 struct xarray vports_caps_xa;
833 };
834
835 struct mlx5dr_cmd_caps {
836 u16 gvmi;
837 u64 nic_rx_drop_address;
838 u64 nic_tx_drop_address;
839 u64 nic_tx_allow_address;
840 u64 esw_rx_drop_address;
841 u64 esw_tx_drop_address;
842 u32 log_icm_size;
843 u64 hdr_modify_icm_addr;
844 u32 flex_protocols;
845 u8 flex_parser_id_icmp_dw0;
846 u8 flex_parser_id_icmp_dw1;
847 u8 flex_parser_id_icmpv6_dw0;
848 u8 flex_parser_id_icmpv6_dw1;
849 u8 flex_parser_id_geneve_tlv_option_0;
850 u8 flex_parser_id_mpls_over_gre;
851 u8 flex_parser_id_mpls_over_udp;
852 u8 flex_parser_id_gtpu_dw_0;
853 u8 flex_parser_id_gtpu_teid;
854 u8 flex_parser_id_gtpu_dw_2;
855 u8 flex_parser_id_gtpu_first_ext_dw_0;
856 u8 flex_parser_ok_bits_supp;
857 u8 max_ft_level;
858 u16 roce_min_src_udp;
859 u8 sw_format_ver;
860 bool eswitch_manager;
861 bool rx_sw_owner;
862 bool tx_sw_owner;
863 bool fdb_sw_owner;
864 u8 rx_sw_owner_v2:1;
865 u8 tx_sw_owner_v2:1;
866 u8 fdb_sw_owner_v2:1;
867 struct mlx5dr_esw_caps esw_caps;
868 struct mlx5dr_vports vports;
869 bool prio_tag_required;
870 struct mlx5dr_roce_cap roce_caps;
871 u8 is_ecpf:1;
872 u8 isolate_vl_tc:1;
873 };
874
875 enum mlx5dr_domain_nic_type {
876 DR_DOMAIN_NIC_TYPE_RX,
877 DR_DOMAIN_NIC_TYPE_TX,
878 };
879
880 struct mlx5dr_domain_rx_tx {
881 u64 drop_icm_addr;
882 u64 default_icm_addr;
883 enum mlx5dr_domain_nic_type type;
884 struct mutex mutex; /* protect rx/tx domain */
885 };
886
887 struct mlx5dr_domain_info {
888 bool supp_sw_steering;
889 u32 max_inline_size;
890 u32 max_send_wr;
891 u32 max_log_sw_icm_sz;
892 u32 max_log_action_icm_sz;
893 struct mlx5dr_domain_rx_tx rx;
894 struct mlx5dr_domain_rx_tx tx;
895 struct mlx5dr_cmd_caps caps;
896 };
897
898 struct mlx5dr_domain {
899 struct mlx5dr_domain *peer_dmn;
900 struct mlx5_core_dev *mdev;
901 u32 pdn;
902 struct mlx5_uars_page *uar;
903 enum mlx5dr_domain_type type;
904 refcount_t refcount;
905 struct mlx5dr_icm_pool *ste_icm_pool;
906 struct mlx5dr_icm_pool *action_icm_pool;
907 struct mlx5dr_send_ring *send_ring;
908 struct mlx5dr_domain_info info;
909 struct xarray csum_fts_xa;
910 struct mlx5dr_ste_ctx *ste_ctx;
911 struct list_head dbg_tbl_list;
912 struct mlx5dr_dbg_dump_info dump_info;
913 };
914
915 struct mlx5dr_table_rx_tx {
916 struct mlx5dr_ste_htbl *s_anchor;
917 struct mlx5dr_domain_rx_tx *nic_dmn;
918 u64 default_icm_addr;
919 struct list_head nic_matcher_list;
920 };
921
922 struct mlx5dr_table {
923 struct mlx5dr_domain *dmn;
924 struct mlx5dr_table_rx_tx rx;
925 struct mlx5dr_table_rx_tx tx;
926 u32 level;
927 u32 table_type;
928 u32 table_id;
929 u32 flags;
930 struct list_head matcher_list;
931 struct mlx5dr_action *miss_action;
932 refcount_t refcount;
933 struct list_head dbg_node;
934 };
935
936 struct mlx5dr_matcher_rx_tx {
937 struct mlx5dr_ste_htbl *s_htbl;
938 struct mlx5dr_ste_htbl *e_anchor;
939 struct mlx5dr_ste_build *ste_builder;
940 struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX]
941 [DR_RULE_IPV_MAX]
942 [DR_RULE_MAX_STES];
943 u8 num_of_builders;
944 u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX];
945 u64 default_icm_addr;
946 struct mlx5dr_table_rx_tx *nic_tbl;
947 u32 prio;
948 struct list_head list_node;
949 u32 rules;
950 };
951
952 struct mlx5dr_matcher {
953 struct mlx5dr_table *tbl;
954 struct mlx5dr_matcher_rx_tx rx;
955 struct mlx5dr_matcher_rx_tx tx;
956 struct list_head list_node; /* Used for both matchers and dbg managing */
957 u32 prio;
958 struct mlx5dr_match_param mask;
959 u8 match_criteria;
960 refcount_t refcount;
961 struct list_head dbg_rule_list;
962 };
963
964 struct mlx5dr_ste_action_modify_field {
965 u16 hw_field;
966 u8 start;
967 u8 end;
968 u8 l3_type;
969 u8 l4_type;
970 };
971
972 struct mlx5dr_action_rewrite {
973 struct mlx5dr_domain *dmn;
974 struct mlx5dr_icm_chunk *chunk;
975 u8 *data;
976 u16 num_of_actions;
977 u32 index;
978 u8 allow_rx:1;
979 u8 allow_tx:1;
980 u8 modify_ttl:1;
981 };
982
983 struct mlx5dr_action_reformat {
984 struct mlx5dr_domain *dmn;
985 u32 id;
986 u32 size;
987 u8 param_0;
988 u8 param_1;
989 };
990
991 struct mlx5dr_action_sampler {
992 struct mlx5dr_domain *dmn;
993 u64 rx_icm_addr;
994 u64 tx_icm_addr;
995 u32 sampler_id;
996 };
997
998 struct mlx5dr_action_dest_tbl {
999 u8 is_fw_tbl:1;
1000 union {
1001 struct mlx5dr_table *tbl;
1002 struct {
1003 struct mlx5dr_domain *dmn;
1004 u32 id;
1005 u32 group_id;
1006 enum fs_flow_table_type type;
1007 u64 rx_icm_addr;
1008 u64 tx_icm_addr;
1009 struct mlx5dr_action **ref_actions;
1010 u32 num_of_ref_actions;
1011 } fw_tbl;
1012 };
1013 };
1014
1015 struct mlx5dr_action_ctr {
1016 u32 ctr_id;
1017 u32 offset;
1018 };
1019
1020 struct mlx5dr_action_vport {
1021 struct mlx5dr_domain *dmn;
1022 struct mlx5dr_cmd_vport_cap *caps;
1023 };
1024
1025 struct mlx5dr_action_push_vlan {
1026 u32 vlan_hdr; /* tpid_pcp_dei_vid */
1027 };
1028
1029 struct mlx5dr_action_flow_tag {
1030 u32 flow_tag;
1031 };
1032
1033 struct mlx5dr_rule_action_member {
1034 struct mlx5dr_action *action;
1035 struct list_head list;
1036 };
1037
1038 struct mlx5dr_action {
1039 enum mlx5dr_action_type action_type;
1040 refcount_t refcount;
1041
1042 union {
1043 void *data;
1044 struct mlx5dr_action_rewrite *rewrite;
1045 struct mlx5dr_action_reformat *reformat;
1046 struct mlx5dr_action_sampler *sampler;
1047 struct mlx5dr_action_dest_tbl *dest_tbl;
1048 struct mlx5dr_action_ctr *ctr;
1049 struct mlx5dr_action_vport *vport;
1050 struct mlx5dr_action_push_vlan *push_vlan;
1051 struct mlx5dr_action_flow_tag *flow_tag;
1052 };
1053 };
1054
1055 enum mlx5dr_connect_type {
1056 CONNECT_HIT = 1,
1057 CONNECT_MISS = 2,
1058 };
1059
1060 struct mlx5dr_htbl_connect_info {
1061 enum mlx5dr_connect_type type;
1062 union {
1063 struct mlx5dr_ste_htbl *hit_next_htbl;
1064 u64 miss_icm_addr;
1065 };
1066 };
1067
1068 struct mlx5dr_rule_rx_tx {
1069 struct mlx5dr_matcher_rx_tx *nic_matcher;
1070 struct mlx5dr_ste *last_rule_ste;
1071 };
1072
1073 struct mlx5dr_rule {
1074 struct mlx5dr_matcher *matcher;
1075 struct mlx5dr_rule_rx_tx rx;
1076 struct mlx5dr_rule_rx_tx tx;
1077 struct list_head rule_actions_list;
1078 struct list_head dbg_node;
1079 u32 flow_source;
1080 };
1081
1082 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule,
1083 struct mlx5dr_ste *ste,
1084 bool force);
1085 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr,
1086 struct mlx5dr_ste *curr_ste,
1087 int *num_of_stes);
1088
1089 struct mlx5dr_icm_chunk {
1090 struct mlx5dr_icm_buddy_mem *buddy_mem;
1091 struct list_head chunk_list;
1092
1093 /* indicates the index of this chunk in the whole memory,
1094 * used for deleting the chunk from the buddy
1095 */
1096 unsigned int seg;
1097 enum mlx5dr_icm_chunk_size size;
1098
1099 /* Memory optimisation */
1100 struct mlx5dr_ste *ste_arr;
1101 u8 *hw_ste_arr;
1102 struct list_head *miss_list;
1103 };
1104
mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx * nic_dmn)1105 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn)
1106 {
1107 mutex_lock(&nic_dmn->mutex);
1108 }
1109
mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx * nic_dmn)1110 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn)
1111 {
1112 mutex_unlock(&nic_dmn->mutex);
1113 }
1114
mlx5dr_domain_lock(struct mlx5dr_domain * dmn)1115 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn)
1116 {
1117 mlx5dr_domain_nic_lock(&dmn->info.rx);
1118 mlx5dr_domain_nic_lock(&dmn->info.tx);
1119 }
1120
mlx5dr_domain_unlock(struct mlx5dr_domain * dmn)1121 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn)
1122 {
1123 mlx5dr_domain_nic_unlock(&dmn->info.tx);
1124 mlx5dr_domain_nic_unlock(&dmn->info.rx);
1125 }
1126
1127 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn,
1128 struct mlx5dr_matcher_rx_tx *nic_matcher);
1129 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn,
1130 struct mlx5dr_matcher_rx_tx *nic_matcher);
1131
1132 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher,
1133 struct mlx5dr_matcher_rx_tx *nic_matcher,
1134 enum mlx5dr_ipv outer_ipv,
1135 enum mlx5dr_ipv inner_ipv);
1136
1137 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk);
1138 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk);
1139 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk);
1140 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk);
1141 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk);
1142 u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste);
1143
1144 static inline int
mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)1145 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)
1146 {
1147 if (icm_type == DR_ICM_TYPE_STE)
1148 return DR_STE_SIZE;
1149
1150 return DR_MODIFY_ACTION_SIZE;
1151 }
1152
1153 static inline u32
mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)1154 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)
1155 {
1156 return 1 << chunk_size;
1157 }
1158
1159 static inline int
mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,enum mlx5dr_icm_type icm_type)1160 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,
1161 enum mlx5dr_icm_type icm_type)
1162 {
1163 int num_of_entries;
1164 int entry_size;
1165
1166 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type);
1167 num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
1168
1169 return entry_size * num_of_entries;
1170 }
1171
1172 static inline int
mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl * htbl)1173 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl)
1174 {
1175 int num_of_entries =
1176 mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk->size);
1177
1178 /* Threshold is 50%, one is added to table of size 1 */
1179 return (num_of_entries + 1) / 2;
1180 }
1181
1182 static inline bool
mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl * htbl)1183 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl)
1184 {
1185 if (htbl->chunk->size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask)
1186 return false;
1187
1188 return true;
1189 }
1190
1191 struct mlx5dr_cmd_vport_cap *
1192 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport);
1193
1194 struct mlx5dr_cmd_query_flow_table_details {
1195 u8 status;
1196 u8 level;
1197 u64 sw_owner_icm_root_1;
1198 u64 sw_owner_icm_root_0;
1199 };
1200
1201 struct mlx5dr_cmd_create_flow_table_attr {
1202 u32 table_type;
1203 u64 icm_addr_rx;
1204 u64 icm_addr_tx;
1205 u8 level;
1206 bool sw_owner;
1207 bool term_tbl;
1208 bool decap_en;
1209 bool reformat_en;
1210 };
1211
1212 /* internal API functions */
1213 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev,
1214 struct mlx5dr_cmd_caps *caps);
1215 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev,
1216 bool other_vport, u16 vport_number,
1217 u64 *icm_address_rx,
1218 u64 *icm_address_tx);
1219 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev,
1220 bool other_vport, u16 vport_number, u16 *gvmi);
1221 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev,
1222 struct mlx5dr_esw_caps *caps);
1223 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev,
1224 u32 sampler_id,
1225 u64 *rx_icm_addr,
1226 u64 *tx_icm_addr);
1227 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev);
1228 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev,
1229 u32 table_type,
1230 u32 table_id,
1231 u32 group_id,
1232 u32 modify_header_id,
1233 u16 vport_id);
1234 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev,
1235 u32 table_type,
1236 u32 table_id);
1237 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev,
1238 u32 table_type,
1239 u8 num_of_actions,
1240 u64 *actions,
1241 u32 *modify_header_id);
1242 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev,
1243 u32 modify_header_id);
1244 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev,
1245 u32 table_type,
1246 u32 table_id,
1247 u32 *group_id);
1248 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev,
1249 u32 table_type,
1250 u32 table_id,
1251 u32 group_id);
1252 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev,
1253 struct mlx5dr_cmd_create_flow_table_attr *attr,
1254 u64 *fdb_rx_icm_addr,
1255 u32 *table_id);
1256 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev,
1257 u32 table_id,
1258 u32 table_type);
1259 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev,
1260 enum fs_flow_table_type type,
1261 u32 table_id,
1262 struct mlx5dr_cmd_query_flow_table_details *output);
1263 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev,
1264 enum mlx5_reformat_ctx_type rt,
1265 u8 reformat_param_0,
1266 u8 reformat_param_1,
1267 size_t reformat_size,
1268 void *reformat_data,
1269 u32 *reformat_id);
1270 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev,
1271 u32 reformat_id);
1272
1273 struct mlx5dr_cmd_gid_attr {
1274 u8 gid[16];
1275 u8 mac[6];
1276 u32 roce_ver;
1277 };
1278
1279 struct mlx5dr_cmd_qp_create_attr {
1280 u32 page_id;
1281 u32 pdn;
1282 u32 cqn;
1283 u32 pm_state;
1284 u32 service_type;
1285 u32 buff_umem_id;
1286 u32 db_umem_id;
1287 u32 sq_wqe_cnt;
1288 u32 rq_wqe_cnt;
1289 u32 rq_wqe_shift;
1290 u8 isolate_vl_tc:1;
1291 };
1292
1293 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
1294 u16 index, struct mlx5dr_cmd_gid_attr *attr);
1295
1296 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
1297 enum mlx5dr_icm_type icm_type);
1298 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool);
1299
1300 struct mlx5dr_icm_chunk *
1301 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
1302 enum mlx5dr_icm_chunk_size chunk_size);
1303 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk);
1304
1305 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx,
1306 u8 *hw_ste_p, u32 ste_size);
1307 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn,
1308 struct mlx5dr_domain_rx_tx *nic_dmn,
1309 struct mlx5dr_ste_htbl *htbl,
1310 struct mlx5dr_htbl_connect_info *connect_info,
1311 bool update_hw_ste);
1312 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx,
1313 u16 gvmi,
1314 enum mlx5dr_domain_nic_type nic_type,
1315 struct mlx5dr_ste_htbl *htbl,
1316 u8 *formatted_ste,
1317 struct mlx5dr_htbl_connect_info *connect_info);
1318 void mlx5dr_ste_copy_param(u8 match_criteria,
1319 struct mlx5dr_match_param *set_param,
1320 struct mlx5dr_match_parameters *mask,
1321 bool clear);
1322
1323 struct mlx5dr_qp {
1324 struct mlx5_core_dev *mdev;
1325 struct mlx5_wq_qp wq;
1326 struct mlx5_uars_page *uar;
1327 struct mlx5_wq_ctrl wq_ctrl;
1328 u32 qpn;
1329 struct {
1330 unsigned int pc;
1331 unsigned int cc;
1332 unsigned int size;
1333 unsigned int *wqe_head;
1334 unsigned int wqe_cnt;
1335 } sq;
1336 struct {
1337 unsigned int pc;
1338 unsigned int cc;
1339 unsigned int size;
1340 unsigned int wqe_cnt;
1341 } rq;
1342 int max_inline_data;
1343 };
1344
1345 struct mlx5dr_cq {
1346 struct mlx5_core_dev *mdev;
1347 struct mlx5_cqwq wq;
1348 struct mlx5_wq_ctrl wq_ctrl;
1349 struct mlx5_core_cq mcq;
1350 struct mlx5dr_qp *qp;
1351 };
1352
1353 struct mlx5dr_mr {
1354 struct mlx5_core_dev *mdev;
1355 u32 mkey;
1356 dma_addr_t dma_addr;
1357 void *addr;
1358 size_t size;
1359 };
1360
1361 #define MAX_SEND_CQE 64
1362 #define MIN_READ_SYNC 64
1363
1364 struct mlx5dr_send_ring {
1365 struct mlx5dr_cq *cq;
1366 struct mlx5dr_qp *qp;
1367 struct mlx5dr_mr *mr;
1368 /* How much wqes are waiting for completion */
1369 u32 pending_wqe;
1370 /* Signal request per this trash hold value */
1371 u16 signal_th;
1372 /* Each post_send_size less than max_post_send_size */
1373 u32 max_post_send_size;
1374 /* manage the send queue */
1375 u32 tx_head;
1376 void *buf;
1377 u32 buf_size;
1378 u8 sync_buff[MIN_READ_SYNC];
1379 struct mlx5dr_mr *sync_mr;
1380 spinlock_t lock; /* Protect the data path of the send ring */
1381 bool err_state; /* send_ring is not usable in err state */
1382 };
1383
1384 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn);
1385 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn,
1386 struct mlx5dr_send_ring *send_ring);
1387 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn);
1388 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn,
1389 struct mlx5dr_ste *ste,
1390 u8 *data,
1391 u16 size,
1392 u16 offset);
1393 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn,
1394 struct mlx5dr_ste_htbl *htbl,
1395 u8 *formatted_ste, u8 *mask);
1396 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn,
1397 struct mlx5dr_ste_htbl *htbl,
1398 u8 *ste_init_data,
1399 bool update_hw_ste);
1400 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn,
1401 struct mlx5dr_action *action);
1402
1403 struct mlx5dr_cmd_ft_info {
1404 u32 id;
1405 u16 vport;
1406 enum fs_flow_table_type type;
1407 };
1408
1409 struct mlx5dr_cmd_flow_destination_hw_info {
1410 enum mlx5_flow_destination_type type;
1411 union {
1412 u32 tir_num;
1413 u32 ft_num;
1414 u32 ft_id;
1415 u32 counter_id;
1416 u32 sampler_id;
1417 struct {
1418 u16 num;
1419 u16 vhca_id;
1420 u32 reformat_id;
1421 u8 flags;
1422 } vport;
1423 };
1424 };
1425
1426 struct mlx5dr_cmd_fte_info {
1427 u32 dests_size;
1428 u32 index;
1429 struct mlx5_flow_context flow_context;
1430 u32 *val;
1431 struct mlx5_flow_act action;
1432 struct mlx5dr_cmd_flow_destination_hw_info *dest_arr;
1433 bool ignore_flow_level;
1434 };
1435
1436 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev,
1437 int opmod, int modify_mask,
1438 struct mlx5dr_cmd_ft_info *ft,
1439 u32 group_id,
1440 struct mlx5dr_cmd_fte_info *fte);
1441
1442 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps);
1443
1444 struct mlx5dr_fw_recalc_cs_ft {
1445 u64 rx_icm_addr;
1446 u32 table_id;
1447 u32 group_id;
1448 u32 modify_hdr_id;
1449 };
1450
1451 struct mlx5dr_fw_recalc_cs_ft *
1452 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num);
1453 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn,
1454 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft);
1455 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn,
1456 u16 vport_num,
1457 u64 *rx_icm_addr);
1458 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn,
1459 struct mlx5dr_cmd_flow_destination_hw_info *dest,
1460 int num_dest,
1461 bool reformat_req,
1462 u32 *tbl_id,
1463 u32 *group_id,
1464 bool ignore_flow_level,
1465 u32 flow_source);
1466 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id,
1467 u32 group_id);
1468 #endif /* _DR_TYPES_H_ */
1469