1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2021 Corigine, Inc. */
3
4 #include "conntrack.h"
5 #include "../nfp_port.h"
6
7 const struct rhashtable_params nfp_tc_ct_merge_params = {
8 .head_offset = offsetof(struct nfp_fl_ct_tc_merge,
9 hash_node),
10 .key_len = sizeof(unsigned long) * 2,
11 .key_offset = offsetof(struct nfp_fl_ct_tc_merge, cookie),
12 .automatic_shrinking = true,
13 };
14
15 const struct rhashtable_params nfp_nft_ct_merge_params = {
16 .head_offset = offsetof(struct nfp_fl_nft_tc_merge,
17 hash_node),
18 .key_len = sizeof(unsigned long) * 3,
19 .key_offset = offsetof(struct nfp_fl_nft_tc_merge, cookie),
20 .automatic_shrinking = true,
21 };
22
23 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
24 enum flow_action_id act_id);
25
26 /**
27 * get_hashentry() - Wrapper around hashtable lookup.
28 * @ht: hashtable where entry could be found
29 * @key: key to lookup
30 * @params: hashtable params
31 * @size: size of entry to allocate if not in table
32 *
33 * Returns an entry from a hashtable. If entry does not exist
34 * yet allocate the memory for it and return the new entry.
35 */
get_hashentry(struct rhashtable * ht,void * key,const struct rhashtable_params params,size_t size)36 static void *get_hashentry(struct rhashtable *ht, void *key,
37 const struct rhashtable_params params, size_t size)
38 {
39 void *result;
40
41 result = rhashtable_lookup_fast(ht, key, params);
42
43 if (result)
44 return result;
45
46 result = kzalloc(size, GFP_KERNEL);
47 if (!result)
48 return ERR_PTR(-ENOMEM);
49
50 return result;
51 }
52
is_pre_ct_flow(struct flow_cls_offload * flow)53 bool is_pre_ct_flow(struct flow_cls_offload *flow)
54 {
55 struct flow_action_entry *act;
56 int i;
57
58 flow_action_for_each(i, act, &flow->rule->action) {
59 if (act->id == FLOW_ACTION_CT && !act->ct.action)
60 return true;
61 }
62 return false;
63 }
64
is_post_ct_flow(struct flow_cls_offload * flow)65 bool is_post_ct_flow(struct flow_cls_offload *flow)
66 {
67 struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
68 struct flow_dissector *dissector = rule->match.dissector;
69 struct flow_match_ct ct;
70
71 if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
72 flow_rule_match_ct(rule, &ct);
73 if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
74 return true;
75 }
76 return false;
77 }
78
79 /**
80 * get_mangled_key() - Mangle the key if mangle act exists
81 * @rule: rule that carries the actions
82 * @buf: pointer to key to be mangled
83 * @offset: used to adjust mangled offset in L2/L3/L4 header
84 * @key_sz: key size
85 * @htype: mangling type
86 *
87 * Returns buf where the mangled key stores.
88 */
get_mangled_key(struct flow_rule * rule,void * buf,u32 offset,size_t key_sz,enum flow_action_mangle_base htype)89 static void *get_mangled_key(struct flow_rule *rule, void *buf,
90 u32 offset, size_t key_sz,
91 enum flow_action_mangle_base htype)
92 {
93 struct flow_action_entry *act;
94 u32 *val = (u32 *)buf;
95 u32 off, msk, key;
96 int i;
97
98 flow_action_for_each(i, act, &rule->action) {
99 if (act->id == FLOW_ACTION_MANGLE &&
100 act->mangle.htype == htype) {
101 off = act->mangle.offset - offset;
102 msk = act->mangle.mask;
103 key = act->mangle.val;
104
105 /* Mangling is supposed to be u32 aligned */
106 if (off % 4 || off >= key_sz)
107 continue;
108
109 val[off >> 2] &= msk;
110 val[off >> 2] |= key;
111 }
112 }
113
114 return buf;
115 }
116
117 /* Only tos and ttl are involved in flow_match_ip structure, which
118 * doesn't conform to the layout of ip/ipv6 header definition. So
119 * they need particular process here: fill them into the ip/ipv6
120 * header, so that mangling actions can work directly.
121 */
122 #define NFP_IPV4_TOS_MASK GENMASK(23, 16)
123 #define NFP_IPV4_TTL_MASK GENMASK(31, 24)
124 #define NFP_IPV6_TCLASS_MASK GENMASK(27, 20)
125 #define NFP_IPV6_HLIMIT_MASK GENMASK(7, 0)
get_mangled_tos_ttl(struct flow_rule * rule,void * buf,bool is_v6)126 static void *get_mangled_tos_ttl(struct flow_rule *rule, void *buf,
127 bool is_v6)
128 {
129 struct flow_match_ip match;
130 /* IPv4's ttl field is in third dword. */
131 __be32 ip_hdr[3];
132 u32 tmp, hdr_len;
133
134 flow_rule_match_ip(rule, &match);
135
136 if (is_v6) {
137 tmp = FIELD_PREP(NFP_IPV6_TCLASS_MASK, match.key->tos);
138 ip_hdr[0] = cpu_to_be32(tmp);
139 tmp = FIELD_PREP(NFP_IPV6_HLIMIT_MASK, match.key->ttl);
140 ip_hdr[1] = cpu_to_be32(tmp);
141 hdr_len = 2 * sizeof(__be32);
142 } else {
143 tmp = FIELD_PREP(NFP_IPV4_TOS_MASK, match.key->tos);
144 ip_hdr[0] = cpu_to_be32(tmp);
145 tmp = FIELD_PREP(NFP_IPV4_TTL_MASK, match.key->ttl);
146 ip_hdr[2] = cpu_to_be32(tmp);
147 hdr_len = 3 * sizeof(__be32);
148 }
149
150 get_mangled_key(rule, ip_hdr, 0, hdr_len,
151 is_v6 ? FLOW_ACT_MANGLE_HDR_TYPE_IP6 :
152 FLOW_ACT_MANGLE_HDR_TYPE_IP4);
153
154 match.key = buf;
155
156 if (is_v6) {
157 tmp = be32_to_cpu(ip_hdr[0]);
158 match.key->tos = FIELD_GET(NFP_IPV6_TCLASS_MASK, tmp);
159 tmp = be32_to_cpu(ip_hdr[1]);
160 match.key->ttl = FIELD_GET(NFP_IPV6_HLIMIT_MASK, tmp);
161 } else {
162 tmp = be32_to_cpu(ip_hdr[0]);
163 match.key->tos = FIELD_GET(NFP_IPV4_TOS_MASK, tmp);
164 tmp = be32_to_cpu(ip_hdr[2]);
165 match.key->ttl = FIELD_GET(NFP_IPV4_TTL_MASK, tmp);
166 }
167
168 return buf;
169 }
170
171 /* Note entry1 and entry2 are not swappable, entry1 should be
172 * the former flow whose mangle action need be taken into account
173 * if existed, and entry2 should be the latter flow whose action
174 * we don't care.
175 */
nfp_ct_merge_check(struct nfp_fl_ct_flow_entry * entry1,struct nfp_fl_ct_flow_entry * entry2)176 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
177 struct nfp_fl_ct_flow_entry *entry2)
178 {
179 unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys &
180 entry2->rule->match.dissector->used_keys;
181 bool out, is_v6 = false;
182 u8 ip_proto = 0;
183 /* Temporary buffer for mangling keys, 64 is enough to cover max
184 * struct size of key in various fields that may be mangled.
185 * Supported fileds to mangle:
186 * mac_src/mac_dst(struct flow_match_eth_addrs, 12B)
187 * nw_tos/nw_ttl(struct flow_match_ip, 2B)
188 * nw_src/nw_dst(struct flow_match_ipv4/6_addrs, 32B)
189 * tp_src/tp_dst(struct flow_match_ports, 4B)
190 */
191 char buf[64];
192
193 if (entry1->netdev && entry2->netdev &&
194 entry1->netdev != entry2->netdev)
195 return -EINVAL;
196
197 /* check the overlapped fields one by one, the unmasked part
198 * should not conflict with each other.
199 */
200 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) {
201 struct flow_match_control match1, match2;
202
203 flow_rule_match_control(entry1->rule, &match1);
204 flow_rule_match_control(entry2->rule, &match2);
205 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
206 if (out)
207 goto check_failed;
208 }
209
210 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) {
211 struct flow_match_basic match1, match2;
212
213 flow_rule_match_basic(entry1->rule, &match1);
214 flow_rule_match_basic(entry2->rule, &match2);
215
216 /* n_proto field is a must in ct-related flows,
217 * it should be either ipv4 or ipv6.
218 */
219 is_v6 = match1.key->n_proto == htons(ETH_P_IPV6);
220 /* ip_proto field is a must when port field is cared */
221 ip_proto = match1.key->ip_proto;
222
223 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
224 if (out)
225 goto check_failed;
226 }
227
228 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
229 struct flow_match_ipv4_addrs match1, match2;
230
231 flow_rule_match_ipv4_addrs(entry1->rule, &match1);
232 flow_rule_match_ipv4_addrs(entry2->rule, &match2);
233
234 memcpy(buf, match1.key, sizeof(*match1.key));
235 match1.key = get_mangled_key(entry1->rule, buf,
236 offsetof(struct iphdr, saddr),
237 sizeof(*match1.key),
238 FLOW_ACT_MANGLE_HDR_TYPE_IP4);
239
240 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
241 if (out)
242 goto check_failed;
243 }
244
245 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
246 struct flow_match_ipv6_addrs match1, match2;
247
248 flow_rule_match_ipv6_addrs(entry1->rule, &match1);
249 flow_rule_match_ipv6_addrs(entry2->rule, &match2);
250
251 memcpy(buf, match1.key, sizeof(*match1.key));
252 match1.key = get_mangled_key(entry1->rule, buf,
253 offsetof(struct ipv6hdr, saddr),
254 sizeof(*match1.key),
255 FLOW_ACT_MANGLE_HDR_TYPE_IP6);
256
257 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
258 if (out)
259 goto check_failed;
260 }
261
262 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) {
263 enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC;
264 struct flow_match_ports match1, match2;
265
266 flow_rule_match_ports(entry1->rule, &match1);
267 flow_rule_match_ports(entry2->rule, &match2);
268
269 if (ip_proto == IPPROTO_UDP)
270 htype = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
271 else if (ip_proto == IPPROTO_TCP)
272 htype = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
273
274 memcpy(buf, match1.key, sizeof(*match1.key));
275 match1.key = get_mangled_key(entry1->rule, buf, 0,
276 sizeof(*match1.key), htype);
277
278 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
279 if (out)
280 goto check_failed;
281 }
282
283 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
284 struct flow_match_eth_addrs match1, match2;
285
286 flow_rule_match_eth_addrs(entry1->rule, &match1);
287 flow_rule_match_eth_addrs(entry2->rule, &match2);
288
289 memcpy(buf, match1.key, sizeof(*match1.key));
290 match1.key = get_mangled_key(entry1->rule, buf, 0,
291 sizeof(*match1.key),
292 FLOW_ACT_MANGLE_HDR_TYPE_ETH);
293
294 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
295 if (out)
296 goto check_failed;
297 }
298
299 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) {
300 struct flow_match_vlan match1, match2;
301
302 flow_rule_match_vlan(entry1->rule, &match1);
303 flow_rule_match_vlan(entry2->rule, &match2);
304 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
305 if (out)
306 goto check_failed;
307 }
308
309 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) {
310 struct flow_match_mpls match1, match2;
311
312 flow_rule_match_mpls(entry1->rule, &match1);
313 flow_rule_match_mpls(entry2->rule, &match2);
314 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
315 if (out)
316 goto check_failed;
317 }
318
319 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) {
320 struct flow_match_tcp match1, match2;
321
322 flow_rule_match_tcp(entry1->rule, &match1);
323 flow_rule_match_tcp(entry2->rule, &match2);
324 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
325 if (out)
326 goto check_failed;
327 }
328
329 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) {
330 struct flow_match_ip match1, match2;
331
332 flow_rule_match_ip(entry1->rule, &match1);
333 flow_rule_match_ip(entry2->rule, &match2);
334
335 match1.key = get_mangled_tos_ttl(entry1->rule, buf, is_v6);
336 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
337 if (out)
338 goto check_failed;
339 }
340
341 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
342 struct flow_match_enc_keyid match1, match2;
343
344 flow_rule_match_enc_keyid(entry1->rule, &match1);
345 flow_rule_match_enc_keyid(entry2->rule, &match2);
346 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
347 if (out)
348 goto check_failed;
349 }
350
351 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
352 struct flow_match_ipv4_addrs match1, match2;
353
354 flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1);
355 flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2);
356 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
357 if (out)
358 goto check_failed;
359 }
360
361 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
362 struct flow_match_ipv6_addrs match1, match2;
363
364 flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1);
365 flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2);
366 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
367 if (out)
368 goto check_failed;
369 }
370
371 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
372 struct flow_match_control match1, match2;
373
374 flow_rule_match_enc_control(entry1->rule, &match1);
375 flow_rule_match_enc_control(entry2->rule, &match2);
376 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
377 if (out)
378 goto check_failed;
379 }
380
381 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) {
382 struct flow_match_ip match1, match2;
383
384 flow_rule_match_enc_ip(entry1->rule, &match1);
385 flow_rule_match_enc_ip(entry2->rule, &match2);
386 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
387 if (out)
388 goto check_failed;
389 }
390
391 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
392 struct flow_match_enc_opts match1, match2;
393
394 flow_rule_match_enc_opts(entry1->rule, &match1);
395 flow_rule_match_enc_opts(entry2->rule, &match2);
396 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
397 if (out)
398 goto check_failed;
399 }
400
401 return 0;
402
403 check_failed:
404 return -EINVAL;
405 }
406
nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry * pre_ct_entry,struct nfp_fl_ct_flow_entry * post_ct_entry,struct nfp_fl_ct_flow_entry * nft_entry)407 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
408 struct nfp_fl_ct_flow_entry *post_ct_entry,
409 struct nfp_fl_ct_flow_entry *nft_entry)
410 {
411 struct flow_action_entry *act;
412 int i;
413
414 /* Check for pre_ct->action conflicts */
415 flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
416 switch (act->id) {
417 case FLOW_ACTION_VLAN_PUSH:
418 case FLOW_ACTION_VLAN_POP:
419 case FLOW_ACTION_VLAN_MANGLE:
420 case FLOW_ACTION_MPLS_PUSH:
421 case FLOW_ACTION_MPLS_POP:
422 case FLOW_ACTION_MPLS_MANGLE:
423 return -EOPNOTSUPP;
424 default:
425 break;
426 }
427 }
428
429 /* Check for nft->action conflicts */
430 flow_action_for_each(i, act, &nft_entry->rule->action) {
431 switch (act->id) {
432 case FLOW_ACTION_VLAN_PUSH:
433 case FLOW_ACTION_VLAN_POP:
434 case FLOW_ACTION_VLAN_MANGLE:
435 case FLOW_ACTION_MPLS_PUSH:
436 case FLOW_ACTION_MPLS_POP:
437 case FLOW_ACTION_MPLS_MANGLE:
438 return -EOPNOTSUPP;
439 default:
440 break;
441 }
442 }
443 return 0;
444 }
445
nfp_ct_check_meta(struct nfp_fl_ct_flow_entry * post_ct_entry,struct nfp_fl_ct_flow_entry * nft_entry)446 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
447 struct nfp_fl_ct_flow_entry *nft_entry)
448 {
449 struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
450 struct flow_action_entry *ct_met;
451 struct flow_match_ct ct;
452 int i;
453
454 ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA);
455 if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) {
456 u32 *act_lbl;
457
458 act_lbl = ct_met->ct_metadata.labels;
459 flow_rule_match_ct(post_ct_entry->rule, &ct);
460 for (i = 0; i < 4; i++) {
461 if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
462 (act_lbl[i] & ct.mask->ct_labels[i]))
463 return -EINVAL;
464 }
465
466 if ((ct.key->ct_mark & ct.mask->ct_mark) ^
467 (ct_met->ct_metadata.mark & ct.mask->ct_mark))
468 return -EINVAL;
469
470 return 0;
471 }
472
473 return -EINVAL;
474 }
475
476 static int
nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls,uint16_t * map)477 nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls, uint16_t *map)
478 {
479 int key_size;
480
481 /* This field must always be present */
482 key_size = sizeof(struct nfp_flower_meta_tci);
483 map[FLOW_PAY_META_TCI] = 0;
484
485 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_EXT_META) {
486 map[FLOW_PAY_EXT_META] = key_size;
487 key_size += sizeof(struct nfp_flower_ext_meta);
488 }
489 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_PORT) {
490 map[FLOW_PAY_INPORT] = key_size;
491 key_size += sizeof(struct nfp_flower_in_port);
492 }
493 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_MAC) {
494 map[FLOW_PAY_MAC_MPLS] = key_size;
495 key_size += sizeof(struct nfp_flower_mac_mpls);
496 }
497 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_TP) {
498 map[FLOW_PAY_L4] = key_size;
499 key_size += sizeof(struct nfp_flower_tp_ports);
500 }
501 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV4) {
502 map[FLOW_PAY_IPV4] = key_size;
503 key_size += sizeof(struct nfp_flower_ipv4);
504 }
505 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV6) {
506 map[FLOW_PAY_IPV6] = key_size;
507 key_size += sizeof(struct nfp_flower_ipv6);
508 }
509
510 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_QINQ) {
511 map[FLOW_PAY_QINQ] = key_size;
512 key_size += sizeof(struct nfp_flower_vlan);
513 }
514
515 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
516 map[FLOW_PAY_GRE] = key_size;
517 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
518 key_size += sizeof(struct nfp_flower_ipv6_gre_tun);
519 else
520 key_size += sizeof(struct nfp_flower_ipv4_gre_tun);
521 }
522
523 if ((in_key_ls.key_layer & NFP_FLOWER_LAYER_VXLAN) ||
524 (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE)) {
525 map[FLOW_PAY_UDP_TUN] = key_size;
526 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
527 key_size += sizeof(struct nfp_flower_ipv6_udp_tun);
528 else
529 key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
530 }
531
532 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
533 map[FLOW_PAY_GENEVE_OPT] = key_size;
534 key_size += sizeof(struct nfp_flower_geneve_options);
535 }
536
537 return key_size;
538 }
539
nfp_fl_merge_actions_offload(struct flow_rule ** rules,struct nfp_flower_priv * priv,struct net_device * netdev,struct nfp_fl_payload * flow_pay)540 static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
541 struct nfp_flower_priv *priv,
542 struct net_device *netdev,
543 struct nfp_fl_payload *flow_pay)
544 {
545 struct flow_action_entry *a_in;
546 int i, j, num_actions, id;
547 struct flow_rule *a_rule;
548 int err = 0, offset = 0;
549
550 num_actions = rules[CT_TYPE_PRE_CT]->action.num_entries +
551 rules[CT_TYPE_NFT]->action.num_entries +
552 rules[CT_TYPE_POST_CT]->action.num_entries;
553
554 a_rule = flow_rule_alloc(num_actions);
555 if (!a_rule)
556 return -ENOMEM;
557
558 /* Actions need a BASIC dissector. */
559 a_rule->match = rules[CT_TYPE_PRE_CT]->match;
560
561 /* Copy actions */
562 for (j = 0; j < _CT_TYPE_MAX; j++) {
563 if (flow_rule_match_key(rules[j], FLOW_DISSECTOR_KEY_BASIC)) {
564 struct flow_match_basic match;
565
566 /* ip_proto is the only field that needed in later compile_action,
567 * needed to set the correct checksum flags. It doesn't really matter
568 * which input rule's ip_proto field we take as the earlier merge checks
569 * would have made sure that they don't conflict. We do not know which
570 * of the subflows would have the ip_proto filled in, so we need to iterate
571 * through the subflows and assign the proper subflow to a_rule
572 */
573 flow_rule_match_basic(rules[j], &match);
574 if (match.mask->ip_proto)
575 a_rule->match = rules[j]->match;
576 }
577
578 for (i = 0; i < rules[j]->action.num_entries; i++) {
579 a_in = &rules[j]->action.entries[i];
580 id = a_in->id;
581
582 /* Ignore CT related actions as these would already have
583 * been taken care of by previous checks, and we do not send
584 * any CT actions to the firmware.
585 */
586 switch (id) {
587 case FLOW_ACTION_CT:
588 case FLOW_ACTION_GOTO:
589 case FLOW_ACTION_CT_METADATA:
590 continue;
591 default:
592 memcpy(&a_rule->action.entries[offset++],
593 a_in, sizeof(struct flow_action_entry));
594 break;
595 }
596 }
597 }
598
599 /* Some actions would have been ignored, so update the num_entries field */
600 a_rule->action.num_entries = offset;
601 err = nfp_flower_compile_action(priv->app, a_rule, netdev, flow_pay, NULL);
602 kfree(a_rule);
603
604 return err;
605 }
606
nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge * m_entry)607 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
608 {
609 enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
610 struct nfp_fl_ct_zone_entry *zt = m_entry->zt;
611 struct nfp_fl_key_ls key_layer, tmp_layer;
612 struct nfp_flower_priv *priv = zt->priv;
613 u16 key_map[_FLOW_PAY_LAYERS_MAX];
614 struct nfp_fl_payload *flow_pay;
615
616 struct flow_rule *rules[_CT_TYPE_MAX];
617 u8 *key, *msk, *kdata, *mdata;
618 struct nfp_port *port = NULL;
619 struct net_device *netdev;
620 bool qinq_sup;
621 u32 port_id;
622 u16 offset;
623 int i, err;
624
625 netdev = m_entry->netdev;
626 qinq_sup = !!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_QINQ);
627
628 rules[CT_TYPE_PRE_CT] = m_entry->tc_m_parent->pre_ct_parent->rule;
629 rules[CT_TYPE_NFT] = m_entry->nft_parent->rule;
630 rules[CT_TYPE_POST_CT] = m_entry->tc_m_parent->post_ct_parent->rule;
631
632 memset(&key_layer, 0, sizeof(struct nfp_fl_key_ls));
633 memset(&key_map, 0, sizeof(key_map));
634
635 /* Calculate the resultant key layer and size for offload */
636 for (i = 0; i < _CT_TYPE_MAX; i++) {
637 err = nfp_flower_calculate_key_layers(priv->app,
638 m_entry->netdev,
639 &tmp_layer, rules[i],
640 &tun_type, NULL);
641 if (err)
642 return err;
643
644 key_layer.key_layer |= tmp_layer.key_layer;
645 key_layer.key_layer_two |= tmp_layer.key_layer_two;
646 }
647 key_layer.key_size = nfp_fl_calc_key_layers_sz(key_layer, key_map);
648
649 flow_pay = nfp_flower_allocate_new(&key_layer);
650 if (!flow_pay)
651 return -ENOMEM;
652
653 memset(flow_pay->unmasked_data, 0, key_layer.key_size);
654 memset(flow_pay->mask_data, 0, key_layer.key_size);
655
656 kdata = flow_pay->unmasked_data;
657 mdata = flow_pay->mask_data;
658
659 offset = key_map[FLOW_PAY_META_TCI];
660 key = kdata + offset;
661 msk = mdata + offset;
662 nfp_flower_compile_meta((struct nfp_flower_meta_tci *)key,
663 (struct nfp_flower_meta_tci *)msk,
664 key_layer.key_layer);
665
666 if (NFP_FLOWER_LAYER_EXT_META & key_layer.key_layer) {
667 offset = key_map[FLOW_PAY_EXT_META];
668 key = kdata + offset;
669 msk = mdata + offset;
670 nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)key,
671 key_layer.key_layer_two);
672 nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)msk,
673 key_layer.key_layer_two);
674 }
675
676 /* Using in_port from the -trk rule. The tc merge checks should already
677 * be checking that the ingress netdevs are the same
678 */
679 port_id = nfp_flower_get_port_id_from_netdev(priv->app, netdev);
680 offset = key_map[FLOW_PAY_INPORT];
681 key = kdata + offset;
682 msk = mdata + offset;
683 err = nfp_flower_compile_port((struct nfp_flower_in_port *)key,
684 port_id, false, tun_type, NULL);
685 if (err)
686 goto ct_offload_err;
687 err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk,
688 port_id, true, tun_type, NULL);
689 if (err)
690 goto ct_offload_err;
691
692 /* This following part works on the assumption that previous checks has
693 * already filtered out flows that has different values for the different
694 * layers. Here we iterate through all three rules and merge their respective
695 * masked value(cared bits), basic method is:
696 * final_key = (r1_key & r1_mask) | (r2_key & r2_mask) | (r3_key & r3_mask)
697 * final_mask = r1_mask | r2_mask | r3_mask
698 * If none of the rules contains a match that is also fine, that simply means
699 * that the layer is not present.
700 */
701 if (!qinq_sup) {
702 for (i = 0; i < _CT_TYPE_MAX; i++) {
703 offset = key_map[FLOW_PAY_META_TCI];
704 key = kdata + offset;
705 msk = mdata + offset;
706 nfp_flower_compile_tci((struct nfp_flower_meta_tci *)key,
707 (struct nfp_flower_meta_tci *)msk,
708 rules[i]);
709 }
710 }
711
712 if (NFP_FLOWER_LAYER_MAC & key_layer.key_layer) {
713 offset = key_map[FLOW_PAY_MAC_MPLS];
714 key = kdata + offset;
715 msk = mdata + offset;
716 for (i = 0; i < _CT_TYPE_MAX; i++) {
717 nfp_flower_compile_mac((struct nfp_flower_mac_mpls *)key,
718 (struct nfp_flower_mac_mpls *)msk,
719 rules[i]);
720 err = nfp_flower_compile_mpls((struct nfp_flower_mac_mpls *)key,
721 (struct nfp_flower_mac_mpls *)msk,
722 rules[i], NULL);
723 if (err)
724 goto ct_offload_err;
725 }
726 }
727
728 if (NFP_FLOWER_LAYER_IPV4 & key_layer.key_layer) {
729 offset = key_map[FLOW_PAY_IPV4];
730 key = kdata + offset;
731 msk = mdata + offset;
732 for (i = 0; i < _CT_TYPE_MAX; i++) {
733 nfp_flower_compile_ipv4((struct nfp_flower_ipv4 *)key,
734 (struct nfp_flower_ipv4 *)msk,
735 rules[i]);
736 }
737 }
738
739 if (NFP_FLOWER_LAYER_IPV6 & key_layer.key_layer) {
740 offset = key_map[FLOW_PAY_IPV6];
741 key = kdata + offset;
742 msk = mdata + offset;
743 for (i = 0; i < _CT_TYPE_MAX; i++) {
744 nfp_flower_compile_ipv6((struct nfp_flower_ipv6 *)key,
745 (struct nfp_flower_ipv6 *)msk,
746 rules[i]);
747 }
748 }
749
750 if (NFP_FLOWER_LAYER_TP & key_layer.key_layer) {
751 offset = key_map[FLOW_PAY_L4];
752 key = kdata + offset;
753 msk = mdata + offset;
754 for (i = 0; i < _CT_TYPE_MAX; i++) {
755 nfp_flower_compile_tport((struct nfp_flower_tp_ports *)key,
756 (struct nfp_flower_tp_ports *)msk,
757 rules[i]);
758 }
759 }
760
761 if (NFP_FLOWER_LAYER2_QINQ & key_layer.key_layer_two) {
762 offset = key_map[FLOW_PAY_QINQ];
763 key = kdata + offset;
764 msk = mdata + offset;
765 for (i = 0; i < _CT_TYPE_MAX; i++) {
766 nfp_flower_compile_vlan((struct nfp_flower_vlan *)key,
767 (struct nfp_flower_vlan *)msk,
768 rules[i]);
769 }
770 }
771
772 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
773 offset = key_map[FLOW_PAY_GRE];
774 key = kdata + offset;
775 msk = mdata + offset;
776 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
777 struct nfp_flower_ipv6_gre_tun *gre_match;
778 struct nfp_ipv6_addr_entry *entry;
779 struct in6_addr *dst;
780
781 for (i = 0; i < _CT_TYPE_MAX; i++) {
782 nfp_flower_compile_ipv6_gre_tun((void *)key,
783 (void *)msk, rules[i]);
784 }
785 gre_match = (struct nfp_flower_ipv6_gre_tun *)key;
786 dst = &gre_match->ipv6.dst;
787
788 entry = nfp_tunnel_add_ipv6_off(priv->app, dst);
789 if (!entry) {
790 err = -ENOMEM;
791 goto ct_offload_err;
792 }
793
794 flow_pay->nfp_tun_ipv6 = entry;
795 } else {
796 __be32 dst;
797
798 for (i = 0; i < _CT_TYPE_MAX; i++) {
799 nfp_flower_compile_ipv4_gre_tun((void *)key,
800 (void *)msk, rules[i]);
801 }
802 dst = ((struct nfp_flower_ipv4_gre_tun *)key)->ipv4.dst;
803
804 /* Store the tunnel destination in the rule data.
805 * This must be present and be an exact match.
806 */
807 flow_pay->nfp_tun_ipv4_addr = dst;
808 nfp_tunnel_add_ipv4_off(priv->app, dst);
809 }
810 }
811
812 if (key_layer.key_layer & NFP_FLOWER_LAYER_VXLAN ||
813 key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE) {
814 offset = key_map[FLOW_PAY_UDP_TUN];
815 key = kdata + offset;
816 msk = mdata + offset;
817 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
818 struct nfp_flower_ipv6_udp_tun *udp_match;
819 struct nfp_ipv6_addr_entry *entry;
820 struct in6_addr *dst;
821
822 for (i = 0; i < _CT_TYPE_MAX; i++) {
823 nfp_flower_compile_ipv6_udp_tun((void *)key,
824 (void *)msk, rules[i]);
825 }
826 udp_match = (struct nfp_flower_ipv6_udp_tun *)key;
827 dst = &udp_match->ipv6.dst;
828
829 entry = nfp_tunnel_add_ipv6_off(priv->app, dst);
830 if (!entry) {
831 err = -ENOMEM;
832 goto ct_offload_err;
833 }
834
835 flow_pay->nfp_tun_ipv6 = entry;
836 } else {
837 __be32 dst;
838
839 for (i = 0; i < _CT_TYPE_MAX; i++) {
840 nfp_flower_compile_ipv4_udp_tun((void *)key,
841 (void *)msk, rules[i]);
842 }
843 dst = ((struct nfp_flower_ipv4_udp_tun *)key)->ipv4.dst;
844
845 /* Store the tunnel destination in the rule data.
846 * This must be present and be an exact match.
847 */
848 flow_pay->nfp_tun_ipv4_addr = dst;
849 nfp_tunnel_add_ipv4_off(priv->app, dst);
850 }
851
852 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
853 offset = key_map[FLOW_PAY_GENEVE_OPT];
854 key = kdata + offset;
855 msk = mdata + offset;
856 for (i = 0; i < _CT_TYPE_MAX; i++)
857 nfp_flower_compile_geneve_opt(key, msk, rules[i]);
858 }
859 }
860
861 /* Merge actions into flow_pay */
862 err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay);
863 if (err)
864 goto ct_offload_err;
865
866 /* Use the pointer address as the cookie, but set the last bit to 1.
867 * This is to avoid the 'is_merge_flow' check from detecting this as
868 * an already merged flow. This works since address alignment means
869 * that the last bit for pointer addresses will be 0.
870 */
871 flow_pay->tc_flower_cookie = ((unsigned long)flow_pay) | 0x1;
872 err = nfp_compile_flow_metadata(priv->app, flow_pay->tc_flower_cookie,
873 flow_pay, netdev, NULL);
874 if (err)
875 goto ct_offload_err;
876
877 if (nfp_netdev_is_nfp_repr(netdev))
878 port = nfp_port_from_netdev(netdev);
879
880 err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node,
881 nfp_flower_table_params);
882 if (err)
883 goto ct_release_offload_meta_err;
884
885 err = nfp_flower_xmit_flow(priv->app, flow_pay,
886 NFP_FLOWER_CMSG_TYPE_FLOW_ADD);
887 if (err)
888 goto ct_remove_rhash_err;
889
890 m_entry->tc_flower_cookie = flow_pay->tc_flower_cookie;
891 m_entry->flow_pay = flow_pay;
892
893 if (port)
894 port->tc_offload_cnt++;
895
896 return err;
897
898 ct_remove_rhash_err:
899 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
900 &flow_pay->fl_node,
901 nfp_flower_table_params));
902 ct_release_offload_meta_err:
903 nfp_modify_flow_metadata(priv->app, flow_pay);
904 ct_offload_err:
905 if (flow_pay->nfp_tun_ipv4_addr)
906 nfp_tunnel_del_ipv4_off(priv->app, flow_pay->nfp_tun_ipv4_addr);
907 if (flow_pay->nfp_tun_ipv6)
908 nfp_tunnel_put_ipv6_off(priv->app, flow_pay->nfp_tun_ipv6);
909 kfree(flow_pay->action_data);
910 kfree(flow_pay->mask_data);
911 kfree(flow_pay->unmasked_data);
912 kfree(flow_pay);
913 return err;
914 }
915
nfp_fl_ct_del_offload(struct nfp_app * app,unsigned long cookie,struct net_device * netdev)916 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
917 struct net_device *netdev)
918 {
919 struct nfp_flower_priv *priv = app->priv;
920 struct nfp_fl_payload *flow_pay;
921 struct nfp_port *port = NULL;
922 int err = 0;
923
924 if (nfp_netdev_is_nfp_repr(netdev))
925 port = nfp_port_from_netdev(netdev);
926
927 flow_pay = nfp_flower_search_fl_table(app, cookie, netdev);
928 if (!flow_pay)
929 return -ENOENT;
930
931 err = nfp_modify_flow_metadata(app, flow_pay);
932 if (err)
933 goto err_free_merge_flow;
934
935 if (flow_pay->nfp_tun_ipv4_addr)
936 nfp_tunnel_del_ipv4_off(app, flow_pay->nfp_tun_ipv4_addr);
937
938 if (flow_pay->nfp_tun_ipv6)
939 nfp_tunnel_put_ipv6_off(app, flow_pay->nfp_tun_ipv6);
940
941 if (!flow_pay->in_hw) {
942 err = 0;
943 goto err_free_merge_flow;
944 }
945
946 err = nfp_flower_xmit_flow(app, flow_pay,
947 NFP_FLOWER_CMSG_TYPE_FLOW_DEL);
948
949 err_free_merge_flow:
950 nfp_flower_del_linked_merge_flows(app, flow_pay);
951 if (port)
952 port->tc_offload_cnt--;
953 kfree(flow_pay->action_data);
954 kfree(flow_pay->mask_data);
955 kfree(flow_pay->unmasked_data);
956 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
957 &flow_pay->fl_node,
958 nfp_flower_table_params));
959 kfree_rcu(flow_pay, rcu);
960 return err;
961 }
962
nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry * zt,struct nfp_fl_ct_flow_entry * nft_entry,struct nfp_fl_ct_tc_merge * tc_m_entry)963 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
964 struct nfp_fl_ct_flow_entry *nft_entry,
965 struct nfp_fl_ct_tc_merge *tc_m_entry)
966 {
967 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
968 struct nfp_fl_nft_tc_merge *nft_m_entry;
969 unsigned long new_cookie[3];
970 int err;
971
972 pre_ct_entry = tc_m_entry->pre_ct_parent;
973 post_ct_entry = tc_m_entry->post_ct_parent;
974
975 err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
976 if (err)
977 return err;
978
979 /* Check that the two tc flows are also compatible with
980 * the nft entry. No need to check the pre_ct and post_ct
981 * entries as that was already done during pre_merge.
982 * The nft entry does not have a chain populated, so
983 * skip this check.
984 */
985 err = nfp_ct_merge_check(pre_ct_entry, nft_entry);
986 if (err)
987 return err;
988 err = nfp_ct_merge_check(nft_entry, post_ct_entry);
989 if (err)
990 return err;
991 err = nfp_ct_check_meta(post_ct_entry, nft_entry);
992 if (err)
993 return err;
994
995 /* Combine tc_merge and nft cookies for this cookie. */
996 new_cookie[0] = tc_m_entry->cookie[0];
997 new_cookie[1] = tc_m_entry->cookie[1];
998 new_cookie[2] = nft_entry->cookie;
999 nft_m_entry = get_hashentry(&zt->nft_merge_tb,
1000 &new_cookie,
1001 nfp_nft_ct_merge_params,
1002 sizeof(*nft_m_entry));
1003
1004 if (IS_ERR(nft_m_entry))
1005 return PTR_ERR(nft_m_entry);
1006
1007 /* nft_m_entry already present, not merging again */
1008 if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie)))
1009 return 0;
1010
1011 memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
1012 nft_m_entry->zt = zt;
1013 nft_m_entry->tc_m_parent = tc_m_entry;
1014 nft_m_entry->nft_parent = nft_entry;
1015 nft_m_entry->tc_flower_cookie = 0;
1016 /* Copy the netdev from one the pre_ct entry. When the tc_m_entry was created
1017 * it only combined them if the netdevs were the same, so can use any of them.
1018 */
1019 nft_m_entry->netdev = pre_ct_entry->netdev;
1020
1021 /* Add this entry to the tc_m_list and nft_flow lists */
1022 list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
1023 list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);
1024
1025 /* Generate offload structure and send to nfp */
1026 err = nfp_fl_ct_add_offload(nft_m_entry);
1027 if (err)
1028 goto err_nft_ct_offload;
1029
1030 err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
1031 nfp_nft_ct_merge_params);
1032 if (err)
1033 goto err_nft_ct_merge_insert;
1034
1035 zt->nft_merge_count++;
1036
1037 return err;
1038
1039 err_nft_ct_merge_insert:
1040 nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
1041 nft_m_entry->netdev);
1042 err_nft_ct_offload:
1043 list_del(&nft_m_entry->tc_merge_list);
1044 list_del(&nft_m_entry->nft_flow_list);
1045 kfree(nft_m_entry);
1046 return err;
1047 }
1048
nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry * zt,struct nfp_fl_ct_flow_entry * ct_entry1,struct nfp_fl_ct_flow_entry * ct_entry2)1049 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
1050 struct nfp_fl_ct_flow_entry *ct_entry1,
1051 struct nfp_fl_ct_flow_entry *ct_entry2)
1052 {
1053 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
1054 struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
1055 struct nfp_fl_ct_tc_merge *m_entry;
1056 unsigned long new_cookie[2];
1057 int err;
1058
1059 if (ct_entry1->type == CT_TYPE_PRE_CT) {
1060 pre_ct_entry = ct_entry1;
1061 post_ct_entry = ct_entry2;
1062 } else {
1063 post_ct_entry = ct_entry1;
1064 pre_ct_entry = ct_entry2;
1065 }
1066
1067 /* Checks that the chain_index of the filter matches the
1068 * chain_index of the GOTO action.
1069 */
1070 if (post_ct_entry->chain_index != pre_ct_entry->chain_index)
1071 return -EINVAL;
1072
1073 err = nfp_ct_merge_check(pre_ct_entry, post_ct_entry);
1074 if (err)
1075 return err;
1076
1077 new_cookie[0] = pre_ct_entry->cookie;
1078 new_cookie[1] = post_ct_entry->cookie;
1079 m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,
1080 nfp_tc_ct_merge_params, sizeof(*m_entry));
1081 if (IS_ERR(m_entry))
1082 return PTR_ERR(m_entry);
1083
1084 /* m_entry already present, not merging again */
1085 if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie)))
1086 return 0;
1087
1088 memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
1089 m_entry->zt = zt;
1090 m_entry->post_ct_parent = post_ct_entry;
1091 m_entry->pre_ct_parent = pre_ct_entry;
1092
1093 /* Add this entry to the pre_ct and post_ct lists */
1094 list_add(&m_entry->post_ct_list, &post_ct_entry->children);
1095 list_add(&m_entry->pre_ct_list, &pre_ct_entry->children);
1096 INIT_LIST_HEAD(&m_entry->children);
1097
1098 err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,
1099 nfp_tc_ct_merge_params);
1100 if (err)
1101 goto err_ct_tc_merge_insert;
1102 zt->tc_merge_count++;
1103
1104 /* Merge with existing nft flows */
1105 list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
1106 list_node) {
1107 nfp_ct_do_nft_merge(zt, nft_entry, m_entry);
1108 }
1109
1110 return 0;
1111
1112 err_ct_tc_merge_insert:
1113 list_del(&m_entry->post_ct_list);
1114 list_del(&m_entry->pre_ct_list);
1115 kfree(m_entry);
1116 return err;
1117 }
1118
1119 static struct
get_nfp_zone_entry(struct nfp_flower_priv * priv,u16 zone,bool wildcarded)1120 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
1121 u16 zone, bool wildcarded)
1122 {
1123 struct nfp_fl_ct_zone_entry *zt;
1124 int err;
1125
1126 if (wildcarded && priv->ct_zone_wc)
1127 return priv->ct_zone_wc;
1128
1129 if (!wildcarded) {
1130 zt = get_hashentry(&priv->ct_zone_table, &zone,
1131 nfp_zone_table_params, sizeof(*zt));
1132
1133 /* If priv is set this is an existing entry, just return it */
1134 if (IS_ERR(zt) || zt->priv)
1135 return zt;
1136 } else {
1137 zt = kzalloc(sizeof(*zt), GFP_KERNEL);
1138 if (!zt)
1139 return ERR_PTR(-ENOMEM);
1140 }
1141
1142 zt->zone = zone;
1143 zt->priv = priv;
1144 zt->nft = NULL;
1145
1146 /* init the various hash tables and lists*/
1147 INIT_LIST_HEAD(&zt->pre_ct_list);
1148 INIT_LIST_HEAD(&zt->post_ct_list);
1149 INIT_LIST_HEAD(&zt->nft_flows_list);
1150
1151 err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);
1152 if (err)
1153 goto err_tc_merge_tb_init;
1154
1155 err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);
1156 if (err)
1157 goto err_nft_merge_tb_init;
1158
1159 if (wildcarded) {
1160 priv->ct_zone_wc = zt;
1161 } else {
1162 err = rhashtable_insert_fast(&priv->ct_zone_table,
1163 &zt->hash_node,
1164 nfp_zone_table_params);
1165 if (err)
1166 goto err_zone_insert;
1167 }
1168
1169 return zt;
1170
1171 err_zone_insert:
1172 rhashtable_destroy(&zt->nft_merge_tb);
1173 err_nft_merge_tb_init:
1174 rhashtable_destroy(&zt->tc_merge_tb);
1175 err_tc_merge_tb_init:
1176 kfree(zt);
1177 return ERR_PTR(err);
1178 }
1179
get_netdev_from_rule(struct flow_rule * rule)1180 static struct net_device *get_netdev_from_rule(struct flow_rule *rule)
1181 {
1182 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) {
1183 struct flow_match_meta match;
1184
1185 flow_rule_match_meta(rule, &match);
1186 if (match.key->ingress_ifindex & match.mask->ingress_ifindex)
1187 return __dev_get_by_index(&init_net,
1188 match.key->ingress_ifindex);
1189 }
1190
1191 return NULL;
1192 }
1193
1194 static struct
nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry * zt,struct net_device * netdev,struct flow_cls_offload * flow,bool is_nft,struct netlink_ext_ack * extack)1195 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
1196 struct net_device *netdev,
1197 struct flow_cls_offload *flow,
1198 bool is_nft, struct netlink_ext_ack *extack)
1199 {
1200 struct nf_flow_match *nft_match = NULL;
1201 struct nfp_fl_ct_flow_entry *entry;
1202 struct nfp_fl_ct_map_entry *map;
1203 struct flow_action_entry *act;
1204 int err, i;
1205
1206 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1207 if (!entry)
1208 return ERR_PTR(-ENOMEM);
1209
1210 entry->rule = flow_rule_alloc(flow->rule->action.num_entries);
1211 if (!entry->rule) {
1212 err = -ENOMEM;
1213 goto err_pre_ct_rule;
1214 }
1215
1216 /* nft flows gets destroyed after callback return, so need
1217 * to do a full copy instead of just a reference.
1218 */
1219 if (is_nft) {
1220 nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL);
1221 if (!nft_match) {
1222 err = -ENOMEM;
1223 goto err_pre_ct_act;
1224 }
1225 memcpy(&nft_match->dissector, flow->rule->match.dissector,
1226 sizeof(nft_match->dissector));
1227 memcpy(&nft_match->mask, flow->rule->match.mask,
1228 sizeof(nft_match->mask));
1229 memcpy(&nft_match->key, flow->rule->match.key,
1230 sizeof(nft_match->key));
1231 entry->rule->match.dissector = &nft_match->dissector;
1232 entry->rule->match.mask = &nft_match->mask;
1233 entry->rule->match.key = &nft_match->key;
1234
1235 if (!netdev)
1236 netdev = get_netdev_from_rule(entry->rule);
1237 } else {
1238 entry->rule->match.dissector = flow->rule->match.dissector;
1239 entry->rule->match.mask = flow->rule->match.mask;
1240 entry->rule->match.key = flow->rule->match.key;
1241 }
1242
1243 entry->zt = zt;
1244 entry->netdev = netdev;
1245 entry->cookie = flow->cookie;
1246 entry->chain_index = flow->common.chain_index;
1247 entry->tun_offset = NFP_FL_CT_NO_TUN;
1248
1249 /* Copy over action data. Unfortunately we do not get a handle to the
1250 * original tcf_action data, and the flow objects gets destroyed, so we
1251 * cannot just save a pointer to this either, so need to copy over the
1252 * data unfortunately.
1253 */
1254 entry->rule->action.num_entries = flow->rule->action.num_entries;
1255 flow_action_for_each(i, act, &flow->rule->action) {
1256 struct flow_action_entry *new_act;
1257
1258 new_act = &entry->rule->action.entries[i];
1259 memcpy(new_act, act, sizeof(struct flow_action_entry));
1260 /* Entunnel is a special case, need to allocate and copy
1261 * tunnel info.
1262 */
1263 if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
1264 struct ip_tunnel_info *tun = act->tunnel;
1265 size_t tun_size = sizeof(*tun) + tun->options_len;
1266
1267 new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC);
1268 if (!new_act->tunnel) {
1269 err = -ENOMEM;
1270 goto err_pre_ct_tun_cp;
1271 }
1272 entry->tun_offset = i;
1273 }
1274 }
1275
1276 INIT_LIST_HEAD(&entry->children);
1277
1278 /* Now add a ct map entry to flower-priv */
1279 map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
1280 nfp_ct_map_params, sizeof(*map));
1281 if (IS_ERR(map)) {
1282 NL_SET_ERR_MSG_MOD(extack,
1283 "offload error: ct map entry creation failed");
1284 err = -ENOMEM;
1285 goto err_ct_flow_insert;
1286 }
1287 map->cookie = flow->cookie;
1288 map->ct_entry = entry;
1289 err = rhashtable_insert_fast(&zt->priv->ct_map_table,
1290 &map->hash_node,
1291 nfp_ct_map_params);
1292 if (err) {
1293 NL_SET_ERR_MSG_MOD(extack,
1294 "offload error: ct map entry table add failed");
1295 goto err_map_insert;
1296 }
1297
1298 return entry;
1299
1300 err_map_insert:
1301 kfree(map);
1302 err_ct_flow_insert:
1303 if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1304 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
1305 err_pre_ct_tun_cp:
1306 kfree(nft_match);
1307 err_pre_ct_act:
1308 kfree(entry->rule);
1309 err_pre_ct_rule:
1310 kfree(entry);
1311 return ERR_PTR(err);
1312 }
1313
cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge * m_entry)1314 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
1315 {
1316 struct nfp_fl_ct_zone_entry *zt;
1317 int err;
1318
1319 zt = m_entry->zt;
1320
1321 /* Flow is in HW, need to delete */
1322 if (m_entry->tc_flower_cookie) {
1323 err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,
1324 m_entry->netdev);
1325 if (err)
1326 return;
1327 }
1328
1329 WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
1330 &m_entry->hash_node,
1331 nfp_nft_ct_merge_params));
1332 zt->nft_merge_count--;
1333 list_del(&m_entry->tc_merge_list);
1334 list_del(&m_entry->nft_flow_list);
1335
1336 kfree(m_entry);
1337 }
1338
nfp_free_nft_merge_children(void * entry,bool is_nft_flow)1339 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
1340 {
1341 struct nfp_fl_nft_tc_merge *m_entry, *tmp;
1342
1343 /* These post entries are parts of two lists, one is a list of nft_entries
1344 * and the other is of from a list of tc_merge structures. Iterate
1345 * through the relevant list and cleanup the entries.
1346 */
1347
1348 if (is_nft_flow) {
1349 /* Need to iterate through list of nft_flow entries*/
1350 struct nfp_fl_ct_flow_entry *ct_entry = entry;
1351
1352 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1353 nft_flow_list) {
1354 cleanup_nft_merge_entry(m_entry);
1355 }
1356 } else {
1357 /* Need to iterate through list of tc_merged_flow entries*/
1358 struct nfp_fl_ct_tc_merge *ct_entry = entry;
1359
1360 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1361 tc_merge_list) {
1362 cleanup_nft_merge_entry(m_entry);
1363 }
1364 }
1365 }
1366
nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge * m_ent)1367 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
1368 {
1369 struct nfp_fl_ct_zone_entry *zt;
1370 int err;
1371
1372 zt = m_ent->zt;
1373 err = rhashtable_remove_fast(&zt->tc_merge_tb,
1374 &m_ent->hash_node,
1375 nfp_tc_ct_merge_params);
1376 if (err)
1377 pr_warn("WARNING: could not remove merge_entry from hashtable\n");
1378 zt->tc_merge_count--;
1379 list_del(&m_ent->post_ct_list);
1380 list_del(&m_ent->pre_ct_list);
1381
1382 if (!list_empty(&m_ent->children))
1383 nfp_free_nft_merge_children(m_ent, false);
1384 kfree(m_ent);
1385 }
1386
nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry * entry)1387 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
1388 {
1389 struct nfp_fl_ct_tc_merge *m_ent, *tmp;
1390
1391 switch (entry->type) {
1392 case CT_TYPE_PRE_CT:
1393 list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
1394 nfp_del_tc_merge_entry(m_ent);
1395 }
1396 break;
1397 case CT_TYPE_POST_CT:
1398 list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
1399 nfp_del_tc_merge_entry(m_ent);
1400 }
1401 break;
1402 default:
1403 break;
1404 }
1405 }
1406
nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry * entry)1407 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
1408 {
1409 list_del(&entry->list_node);
1410
1411 if (!list_empty(&entry->children)) {
1412 if (entry->type == CT_TYPE_NFT)
1413 nfp_free_nft_merge_children(entry, true);
1414 else
1415 nfp_free_tc_merge_children(entry);
1416 }
1417
1418 if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1419 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
1420
1421 if (entry->type == CT_TYPE_NFT) {
1422 struct nf_flow_match *nft_match;
1423
1424 nft_match = container_of(entry->rule->match.dissector,
1425 struct nf_flow_match, dissector);
1426 kfree(nft_match);
1427 }
1428
1429 kfree(entry->rule);
1430 kfree(entry);
1431 }
1432
get_flow_act(struct flow_rule * rule,enum flow_action_id act_id)1433 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
1434 enum flow_action_id act_id)
1435 {
1436 struct flow_action_entry *act = NULL;
1437 int i;
1438
1439 flow_action_for_each(i, act, &rule->action) {
1440 if (act->id == act_id)
1441 return act;
1442 }
1443 return NULL;
1444 }
1445
1446 static void
nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry * ct_entry1,struct nfp_fl_ct_zone_entry * zt_src,struct nfp_fl_ct_zone_entry * zt_dst)1447 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
1448 struct nfp_fl_ct_zone_entry *zt_src,
1449 struct nfp_fl_ct_zone_entry *zt_dst)
1450 {
1451 struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
1452 struct list_head *ct_list;
1453
1454 if (ct_entry1->type == CT_TYPE_PRE_CT)
1455 ct_list = &zt_src->post_ct_list;
1456 else if (ct_entry1->type == CT_TYPE_POST_CT)
1457 ct_list = &zt_src->pre_ct_list;
1458 else
1459 return;
1460
1461 list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
1462 list_node) {
1463 nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1);
1464 }
1465 }
1466
1467 static void
nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry * nft_entry,struct nfp_fl_ct_zone_entry * zt)1468 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
1469 struct nfp_fl_ct_zone_entry *zt)
1470 {
1471 struct nfp_fl_ct_tc_merge *tc_merge_entry;
1472 struct rhashtable_iter iter;
1473
1474 rhashtable_walk_enter(&zt->tc_merge_tb, &iter);
1475 rhashtable_walk_start(&iter);
1476 while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) {
1477 if (IS_ERR(tc_merge_entry))
1478 continue;
1479 rhashtable_walk_stop(&iter);
1480 nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);
1481 rhashtable_walk_start(&iter);
1482 }
1483 rhashtable_walk_stop(&iter);
1484 rhashtable_walk_exit(&iter);
1485 }
1486
nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv * priv,struct net_device * netdev,struct flow_cls_offload * flow,struct netlink_ext_ack * extack)1487 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
1488 struct net_device *netdev,
1489 struct flow_cls_offload *flow,
1490 struct netlink_ext_ack *extack)
1491 {
1492 struct flow_action_entry *ct_act, *ct_goto;
1493 struct nfp_fl_ct_flow_entry *ct_entry;
1494 struct nfp_fl_ct_zone_entry *zt;
1495 int err;
1496
1497 ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT);
1498 if (!ct_act) {
1499 NL_SET_ERR_MSG_MOD(extack,
1500 "unsupported offload: Conntrack action empty in conntrack offload");
1501 return -EOPNOTSUPP;
1502 }
1503
1504 ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
1505 if (!ct_goto) {
1506 NL_SET_ERR_MSG_MOD(extack,
1507 "unsupported offload: Conntrack requires ACTION_GOTO");
1508 return -EOPNOTSUPP;
1509 }
1510
1511 zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);
1512 if (IS_ERR(zt)) {
1513 NL_SET_ERR_MSG_MOD(extack,
1514 "offload error: Could not create zone table entry");
1515 return PTR_ERR(zt);
1516 }
1517
1518 if (!zt->nft) {
1519 zt->nft = ct_act->ct.flow_table;
1520 err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);
1521 if (err) {
1522 NL_SET_ERR_MSG_MOD(extack,
1523 "offload error: Could not register nft_callback");
1524 return err;
1525 }
1526 }
1527
1528 /* Add entry to pre_ct_list */
1529 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1530 if (IS_ERR(ct_entry))
1531 return PTR_ERR(ct_entry);
1532 ct_entry->type = CT_TYPE_PRE_CT;
1533 ct_entry->chain_index = ct_goto->chain_index;
1534 list_add(&ct_entry->list_node, &zt->pre_ct_list);
1535 zt->pre_ct_count++;
1536
1537 nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1538
1539 /* Need to check and merge with tables in the wc_zone as well */
1540 if (priv->ct_zone_wc)
1541 nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);
1542
1543 return 0;
1544 }
1545
nfp_fl_ct_handle_post_ct(struct nfp_flower_priv * priv,struct net_device * netdev,struct flow_cls_offload * flow,struct netlink_ext_ack * extack)1546 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
1547 struct net_device *netdev,
1548 struct flow_cls_offload *flow,
1549 struct netlink_ext_ack *extack)
1550 {
1551 struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
1552 struct nfp_fl_ct_flow_entry *ct_entry;
1553 struct nfp_fl_ct_zone_entry *zt;
1554 bool wildcarded = false;
1555 struct flow_match_ct ct;
1556
1557 flow_rule_match_ct(rule, &ct);
1558 if (!ct.mask->ct_zone) {
1559 wildcarded = true;
1560 } else if (ct.mask->ct_zone != U16_MAX) {
1561 NL_SET_ERR_MSG_MOD(extack,
1562 "unsupported offload: partially wildcarded ct_zone is not supported");
1563 return -EOPNOTSUPP;
1564 }
1565
1566 zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);
1567 if (IS_ERR(zt)) {
1568 NL_SET_ERR_MSG_MOD(extack,
1569 "offload error: Could not create zone table entry");
1570 return PTR_ERR(zt);
1571 }
1572
1573 /* Add entry to post_ct_list */
1574 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1575 if (IS_ERR(ct_entry))
1576 return PTR_ERR(ct_entry);
1577
1578 ct_entry->type = CT_TYPE_POST_CT;
1579 ct_entry->chain_index = flow->common.chain_index;
1580 list_add(&ct_entry->list_node, &zt->post_ct_list);
1581 zt->post_ct_count++;
1582
1583 if (wildcarded) {
1584 /* Iterate through all zone tables if not empty, look for merges with
1585 * pre_ct entries and merge them.
1586 */
1587 struct rhashtable_iter iter;
1588 struct nfp_fl_ct_zone_entry *zone_table;
1589
1590 rhashtable_walk_enter(&priv->ct_zone_table, &iter);
1591 rhashtable_walk_start(&iter);
1592 while ((zone_table = rhashtable_walk_next(&iter)) != NULL) {
1593 if (IS_ERR(zone_table))
1594 continue;
1595 rhashtable_walk_stop(&iter);
1596 nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table);
1597 rhashtable_walk_start(&iter);
1598 }
1599 rhashtable_walk_stop(&iter);
1600 rhashtable_walk_exit(&iter);
1601 } else {
1602 nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1603 }
1604
1605 return 0;
1606 }
1607
1608 static void
nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge * nft_merge,enum ct_entry_type type,u64 * m_pkts,u64 * m_bytes,u64 * m_used)1609 nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge *nft_merge,
1610 enum ct_entry_type type, u64 *m_pkts,
1611 u64 *m_bytes, u64 *m_used)
1612 {
1613 struct nfp_flower_priv *priv = nft_merge->zt->priv;
1614 struct nfp_fl_payload *nfp_flow;
1615 u32 ctx_id;
1616
1617 nfp_flow = nft_merge->flow_pay;
1618 if (!nfp_flow)
1619 return;
1620
1621 ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
1622 *m_pkts += priv->stats[ctx_id].pkts;
1623 *m_bytes += priv->stats[ctx_id].bytes;
1624 *m_used = max_t(u64, *m_used, priv->stats[ctx_id].used);
1625
1626 /* If request is for a sub_flow which is part of a tunnel merged
1627 * flow then update stats from tunnel merged flows first.
1628 */
1629 if (!list_empty(&nfp_flow->linked_flows))
1630 nfp_flower_update_merge_stats(priv->app, nfp_flow);
1631
1632 if (type != CT_TYPE_NFT) {
1633 /* Update nft cached stats */
1634 flow_stats_update(&nft_merge->nft_parent->stats,
1635 priv->stats[ctx_id].bytes,
1636 priv->stats[ctx_id].pkts,
1637 0, priv->stats[ctx_id].used,
1638 FLOW_ACTION_HW_STATS_DELAYED);
1639 } else {
1640 /* Update pre_ct cached stats */
1641 flow_stats_update(&nft_merge->tc_m_parent->pre_ct_parent->stats,
1642 priv->stats[ctx_id].bytes,
1643 priv->stats[ctx_id].pkts,
1644 0, priv->stats[ctx_id].used,
1645 FLOW_ACTION_HW_STATS_DELAYED);
1646 /* Update post_ct cached stats */
1647 flow_stats_update(&nft_merge->tc_m_parent->post_ct_parent->stats,
1648 priv->stats[ctx_id].bytes,
1649 priv->stats[ctx_id].pkts,
1650 0, priv->stats[ctx_id].used,
1651 FLOW_ACTION_HW_STATS_DELAYED);
1652 }
1653 /* Reset stats from the nfp */
1654 priv->stats[ctx_id].pkts = 0;
1655 priv->stats[ctx_id].bytes = 0;
1656 }
1657
nfp_fl_ct_stats(struct flow_cls_offload * flow,struct nfp_fl_ct_map_entry * ct_map_ent)1658 int nfp_fl_ct_stats(struct flow_cls_offload *flow,
1659 struct nfp_fl_ct_map_entry *ct_map_ent)
1660 {
1661 struct nfp_fl_ct_flow_entry *ct_entry = ct_map_ent->ct_entry;
1662 struct nfp_fl_nft_tc_merge *nft_merge, *nft_m_tmp;
1663 struct nfp_fl_ct_tc_merge *tc_merge, *tc_m_tmp;
1664
1665 u64 pkts = 0, bytes = 0, used = 0;
1666 u64 m_pkts, m_bytes, m_used;
1667
1668 spin_lock_bh(&ct_entry->zt->priv->stats_lock);
1669
1670 if (ct_entry->type == CT_TYPE_PRE_CT) {
1671 /* Iterate tc_merge entries associated with this flow */
1672 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
1673 pre_ct_list) {
1674 m_pkts = 0;
1675 m_bytes = 0;
1676 m_used = 0;
1677 /* Iterate nft_merge entries associated with this tc_merge flow */
1678 list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
1679 tc_merge_list) {
1680 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_PRE_CT,
1681 &m_pkts, &m_bytes, &m_used);
1682 }
1683 pkts += m_pkts;
1684 bytes += m_bytes;
1685 used = max_t(u64, used, m_used);
1686 /* Update post_ct partner */
1687 flow_stats_update(&tc_merge->post_ct_parent->stats,
1688 m_bytes, m_pkts, 0, m_used,
1689 FLOW_ACTION_HW_STATS_DELAYED);
1690 }
1691 } else if (ct_entry->type == CT_TYPE_POST_CT) {
1692 /* Iterate tc_merge entries associated with this flow */
1693 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
1694 post_ct_list) {
1695 m_pkts = 0;
1696 m_bytes = 0;
1697 m_used = 0;
1698 /* Iterate nft_merge entries associated with this tc_merge flow */
1699 list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
1700 tc_merge_list) {
1701 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_POST_CT,
1702 &m_pkts, &m_bytes, &m_used);
1703 }
1704 pkts += m_pkts;
1705 bytes += m_bytes;
1706 used = max_t(u64, used, m_used);
1707 /* Update pre_ct partner */
1708 flow_stats_update(&tc_merge->pre_ct_parent->stats,
1709 m_bytes, m_pkts, 0, m_used,
1710 FLOW_ACTION_HW_STATS_DELAYED);
1711 }
1712 } else {
1713 /* Iterate nft_merge entries associated with this nft flow */
1714 list_for_each_entry_safe(nft_merge, nft_m_tmp, &ct_entry->children,
1715 nft_flow_list) {
1716 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_NFT,
1717 &pkts, &bytes, &used);
1718 }
1719 }
1720
1721 /* Add stats from this request to stats potentially cached by
1722 * previous requests.
1723 */
1724 flow_stats_update(&ct_entry->stats, bytes, pkts, 0, used,
1725 FLOW_ACTION_HW_STATS_DELAYED);
1726 /* Finally update the flow stats from the original stats request */
1727 flow_stats_update(&flow->stats, ct_entry->stats.bytes,
1728 ct_entry->stats.pkts, 0,
1729 ct_entry->stats.lastused,
1730 FLOW_ACTION_HW_STATS_DELAYED);
1731 /* Stats has been synced to original flow, can now clear
1732 * the cache.
1733 */
1734 ct_entry->stats.pkts = 0;
1735 ct_entry->stats.bytes = 0;
1736 spin_unlock_bh(&ct_entry->zt->priv->stats_lock);
1737
1738 return 0;
1739 }
1740
1741 static int
nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry * zt,struct flow_cls_offload * flow)1742 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
1743 {
1744 struct nfp_fl_ct_map_entry *ct_map_ent;
1745 struct nfp_fl_ct_flow_entry *ct_entry;
1746 struct netlink_ext_ack *extack = NULL;
1747
1748 ASSERT_RTNL();
1749
1750 extack = flow->common.extack;
1751 switch (flow->command) {
1752 case FLOW_CLS_REPLACE:
1753 /* Netfilter can request offload multiple times for the same
1754 * flow - protect against adding duplicates.
1755 */
1756 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1757 nfp_ct_map_params);
1758 if (!ct_map_ent) {
1759 ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);
1760 if (IS_ERR(ct_entry))
1761 return PTR_ERR(ct_entry);
1762 ct_entry->type = CT_TYPE_NFT;
1763 list_add(&ct_entry->list_node, &zt->nft_flows_list);
1764 zt->nft_flows_count++;
1765 nfp_ct_merge_nft_with_tc(ct_entry, zt);
1766 }
1767 return 0;
1768 case FLOW_CLS_DESTROY:
1769 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1770 nfp_ct_map_params);
1771 return nfp_fl_ct_del_flow(ct_map_ent);
1772 case FLOW_CLS_STATS:
1773 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1774 nfp_ct_map_params);
1775 if (ct_map_ent)
1776 return nfp_fl_ct_stats(flow, ct_map_ent);
1777 break;
1778 default:
1779 break;
1780 }
1781 return -EINVAL;
1782 }
1783
nfp_fl_ct_handle_nft_flow(enum tc_setup_type type,void * type_data,void * cb_priv)1784 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
1785 {
1786 struct flow_cls_offload *flow = type_data;
1787 struct nfp_fl_ct_zone_entry *zt = cb_priv;
1788 int err = -EOPNOTSUPP;
1789
1790 switch (type) {
1791 case TC_SETUP_CLSFLOWER:
1792 rtnl_lock();
1793 err = nfp_fl_ct_offload_nft_flow(zt, flow);
1794 rtnl_unlock();
1795 break;
1796 default:
1797 return -EOPNOTSUPP;
1798 }
1799 return err;
1800 }
1801
1802 static void
nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry * zt)1803 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
1804 {
1805 struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
1806 struct nfp_fl_ct_map_entry *ct_map_ent;
1807
1808 list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
1809 list_node) {
1810 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,
1811 &nft_entry->cookie,
1812 nfp_ct_map_params);
1813 nfp_fl_ct_del_flow(ct_map_ent);
1814 }
1815 }
1816
nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry * ct_map_ent)1817 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
1818 {
1819 struct nfp_fl_ct_flow_entry *ct_entry;
1820 struct nfp_fl_ct_zone_entry *zt;
1821 struct rhashtable *m_table;
1822
1823 if (!ct_map_ent)
1824 return -ENOENT;
1825
1826 zt = ct_map_ent->ct_entry->zt;
1827 ct_entry = ct_map_ent->ct_entry;
1828 m_table = &zt->priv->ct_map_table;
1829
1830 switch (ct_entry->type) {
1831 case CT_TYPE_PRE_CT:
1832 zt->pre_ct_count--;
1833 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1834 nfp_ct_map_params);
1835 nfp_fl_ct_clean_flow_entry(ct_entry);
1836 kfree(ct_map_ent);
1837
1838 if (!zt->pre_ct_count) {
1839 zt->nft = NULL;
1840 nfp_fl_ct_clean_nft_entries(zt);
1841 }
1842 break;
1843 case CT_TYPE_POST_CT:
1844 zt->post_ct_count--;
1845 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1846 nfp_ct_map_params);
1847 nfp_fl_ct_clean_flow_entry(ct_entry);
1848 kfree(ct_map_ent);
1849 break;
1850 case CT_TYPE_NFT:
1851 zt->nft_flows_count--;
1852 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1853 nfp_ct_map_params);
1854 nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry);
1855 kfree(ct_map_ent);
1856 break;
1857 default:
1858 break;
1859 }
1860
1861 return 0;
1862 }
1863