1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/hash.h>
34 #include <linux/mlx5/fs.h>
35 #include <linux/ip.h>
36 #include <linux/ipv6.h>
37 #include "en.h"
38
39 #define ARFS_HASH_SHIFT BITS_PER_BYTE
40 #define ARFS_HASH_SIZE BIT(BITS_PER_BYTE)
41
42 struct arfs_table {
43 struct mlx5e_flow_table ft;
44 struct mlx5_flow_handle *default_rule;
45 struct hlist_head rules_hash[ARFS_HASH_SIZE];
46 };
47
48 enum arfs_type {
49 ARFS_IPV4_TCP,
50 ARFS_IPV6_TCP,
51 ARFS_IPV4_UDP,
52 ARFS_IPV6_UDP,
53 ARFS_NUM_TYPES,
54 };
55
56 struct mlx5e_arfs_tables {
57 struct arfs_table arfs_tables[ARFS_NUM_TYPES];
58 /* Protect aRFS rules list */
59 spinlock_t arfs_lock;
60 int last_filter_id;
61 struct workqueue_struct *wq;
62 };
63
64 struct arfs_tuple {
65 __be16 etype;
66 u8 ip_proto;
67 union {
68 __be32 src_ipv4;
69 struct in6_addr src_ipv6;
70 };
71 union {
72 __be32 dst_ipv4;
73 struct in6_addr dst_ipv6;
74 };
75 __be16 src_port;
76 __be16 dst_port;
77 };
78
79 struct arfs_rule {
80 struct mlx5e_priv *priv;
81 struct work_struct arfs_work;
82 struct mlx5_flow_handle *rule;
83 struct hlist_node hlist;
84 int rxq;
85 /* Flow ID passed to ndo_rx_flow_steer */
86 int flow_id;
87 /* Filter ID returned by ndo_rx_flow_steer */
88 int filter_id;
89 struct arfs_tuple tuple;
90 };
91
92 #define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \
93 for (i = 0; i < ARFS_NUM_TYPES; i++) \
94 mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j)
95
96 #define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \
97 for (j = 0; j < ARFS_HASH_SIZE; j++) \
98 hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist)
99
arfs_get_tt(enum arfs_type type)100 static enum mlx5_traffic_types arfs_get_tt(enum arfs_type type)
101 {
102 switch (type) {
103 case ARFS_IPV4_TCP:
104 return MLX5_TT_IPV4_TCP;
105 case ARFS_IPV4_UDP:
106 return MLX5_TT_IPV4_UDP;
107 case ARFS_IPV6_TCP:
108 return MLX5_TT_IPV6_TCP;
109 case ARFS_IPV6_UDP:
110 return MLX5_TT_IPV6_UDP;
111 default:
112 return -EINVAL;
113 }
114 }
115
arfs_disable(struct mlx5e_flow_steering * fs)116 static int arfs_disable(struct mlx5e_flow_steering *fs)
117 {
118 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
119 int err, i;
120
121 for (i = 0; i < ARFS_NUM_TYPES; i++) {
122 /* Modify ttc rules destination back to their default */
123 err = mlx5_ttc_fwd_default_dest(ttc, arfs_get_tt(i));
124 if (err) {
125 fs_err(fs,
126 "%s: modify ttc[%d] default destination failed, err(%d)\n",
127 __func__, arfs_get_tt(i), err);
128 return err;
129 }
130 }
131 return 0;
132 }
133
134 static void arfs_del_rules(struct mlx5e_flow_steering *fs);
135
mlx5e_arfs_disable(struct mlx5e_flow_steering * fs)136 int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs)
137 {
138 /* Moving to switchdev mode, fs->arfs is freed by mlx5e_nic_profile
139 * cleanup_rx callback and it is not recreated when
140 * mlx5e_uplink_rep_profile is loaded as mlx5e_create_flow_steering()
141 * is not called by the uplink_rep profile init_rx callback. Thus, if
142 * ntuple is set, moving to switchdev flow will enter this function
143 * with fs->arfs nullified.
144 */
145 if (!mlx5e_fs_get_arfs(fs))
146 return 0;
147
148 arfs_del_rules(fs);
149
150 return arfs_disable(fs);
151 }
152
mlx5e_arfs_enable(struct mlx5e_flow_steering * fs)153 int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs)
154 {
155 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
156 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
157 struct mlx5_flow_destination dest = {};
158 int err, i;
159
160 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
161 for (i = 0; i < ARFS_NUM_TYPES; i++) {
162 dest.ft = arfs->arfs_tables[i].ft.t;
163 /* Modify ttc rules destination to point on the aRFS FTs */
164 err = mlx5_ttc_fwd_dest(ttc, arfs_get_tt(i), &dest);
165 if (err) {
166 fs_err(fs, "%s: modify ttc[%d] dest to arfs, failed err(%d)\n",
167 __func__, arfs_get_tt(i), err);
168 arfs_disable(fs);
169 return err;
170 }
171 }
172 return 0;
173 }
174
arfs_destroy_table(struct arfs_table * arfs_t)175 static void arfs_destroy_table(struct arfs_table *arfs_t)
176 {
177 mlx5_del_flow_rules(arfs_t->default_rule);
178 mlx5e_destroy_flow_table(&arfs_t->ft);
179 }
180
_mlx5e_cleanup_tables(struct mlx5e_flow_steering * fs)181 static void _mlx5e_cleanup_tables(struct mlx5e_flow_steering *fs)
182 {
183 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
184 int i;
185
186 arfs_del_rules(fs);
187 destroy_workqueue(arfs->wq);
188 for (i = 0; i < ARFS_NUM_TYPES; i++) {
189 if (!IS_ERR_OR_NULL(arfs->arfs_tables[i].ft.t))
190 arfs_destroy_table(&arfs->arfs_tables[i]);
191 }
192 }
193
mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering * fs,bool ntuple)194 void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple)
195 {
196 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
197
198 if (!ntuple)
199 return;
200
201 _mlx5e_cleanup_tables(fs);
202 mlx5e_fs_set_arfs(fs, NULL);
203 kvfree(arfs);
204 }
205
arfs_add_default_rule(struct mlx5e_flow_steering * fs,struct mlx5e_rx_res * rx_res,enum arfs_type type)206 static int arfs_add_default_rule(struct mlx5e_flow_steering *fs,
207 struct mlx5e_rx_res *rx_res,
208 enum arfs_type type)
209 {
210 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
211 struct arfs_table *arfs_t = &arfs->arfs_tables[type];
212 struct mlx5_flow_destination dest = {};
213 MLX5_DECLARE_FLOW_ACT(flow_act);
214 enum mlx5_traffic_types tt;
215 int err = 0;
216
217 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
218 tt = arfs_get_tt(type);
219 if (tt == -EINVAL) {
220 fs_err(fs, "%s: bad arfs_type: %d\n", __func__, type);
221 return -EINVAL;
222 }
223
224 /* FIXME: Must use mlx5_ttc_get_default_dest(),
225 * but can't since TTC default is not setup yet !
226 */
227 dest.tir_num = mlx5e_rx_res_get_tirn_rss(rx_res, tt);
228 arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL,
229 &flow_act,
230 &dest, 1);
231 if (IS_ERR(arfs_t->default_rule)) {
232 err = PTR_ERR(arfs_t->default_rule);
233 arfs_t->default_rule = NULL;
234 fs_err(fs, "%s: add rule failed, arfs type=%d\n", __func__, type);
235 }
236
237 return err;
238 }
239
240 #define MLX5E_ARFS_NUM_GROUPS 2
241 #define MLX5E_ARFS_GROUP1_SIZE (BIT(16) - 1)
242 #define MLX5E_ARFS_GROUP2_SIZE BIT(0)
243 #define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\
244 MLX5E_ARFS_GROUP2_SIZE)
arfs_create_groups(struct mlx5e_flow_table * ft,enum arfs_type type)245 static int arfs_create_groups(struct mlx5e_flow_table *ft,
246 enum arfs_type type)
247 {
248 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
249 void *outer_headers_c;
250 int ix = 0;
251 u32 *in;
252 int err;
253 u8 *mc;
254
255 ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS,
256 sizeof(*ft->g), GFP_KERNEL);
257 if (!ft->g)
258 return -ENOMEM;
259
260 in = kvzalloc(inlen, GFP_KERNEL);
261 if (!in) {
262 err = -ENOMEM;
263 goto err_free_g;
264 }
265
266 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
267 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc,
268 outer_headers);
269 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype);
270 switch (type) {
271 case ARFS_IPV4_TCP:
272 case ARFS_IPV6_TCP:
273 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
274 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
275 break;
276 case ARFS_IPV4_UDP:
277 case ARFS_IPV6_UDP:
278 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
279 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport);
280 break;
281 default:
282 err = -EINVAL;
283 goto err_free_in;
284 }
285
286 switch (type) {
287 case ARFS_IPV4_TCP:
288 case ARFS_IPV4_UDP:
289 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
290 src_ipv4_src_ipv6.ipv4_layout.ipv4);
291 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
292 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
293 break;
294 case ARFS_IPV6_TCP:
295 case ARFS_IPV6_UDP:
296 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
297 src_ipv4_src_ipv6.ipv6_layout.ipv6),
298 0xff, 16);
299 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
300 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
301 0xff, 16);
302 break;
303 default:
304 err = -EINVAL;
305 goto err_free_in;
306 }
307
308 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
309 MLX5_SET_CFG(in, start_flow_index, ix);
310 ix += MLX5E_ARFS_GROUP1_SIZE;
311 MLX5_SET_CFG(in, end_flow_index, ix - 1);
312 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
313 if (IS_ERR(ft->g[ft->num_groups]))
314 goto err_clean_group;
315 ft->num_groups++;
316
317 memset(in, 0, inlen);
318 MLX5_SET_CFG(in, start_flow_index, ix);
319 ix += MLX5E_ARFS_GROUP2_SIZE;
320 MLX5_SET_CFG(in, end_flow_index, ix - 1);
321 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
322 if (IS_ERR(ft->g[ft->num_groups]))
323 goto err_clean_group;
324 ft->num_groups++;
325
326 kvfree(in);
327 return 0;
328
329 err_clean_group:
330 err = PTR_ERR(ft->g[ft->num_groups]);
331 ft->g[ft->num_groups] = NULL;
332 err_free_in:
333 kvfree(in);
334 err_free_g:
335 kfree(ft->g);
336 ft->g = NULL;
337 return err;
338 }
339
arfs_create_table(struct mlx5e_flow_steering * fs,struct mlx5e_rx_res * rx_res,enum arfs_type type)340 static int arfs_create_table(struct mlx5e_flow_steering *fs,
341 struct mlx5e_rx_res *rx_res,
342 enum arfs_type type)
343 {
344 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false);
345 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
346 struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft;
347 struct mlx5_flow_table_attr ft_attr = {};
348 int err;
349
350 ft->num_groups = 0;
351
352 ft_attr.max_fte = MLX5E_ARFS_TABLE_SIZE;
353 ft_attr.level = MLX5E_ARFS_FT_LEVEL;
354 ft_attr.prio = MLX5E_NIC_PRIO;
355
356 ft->t = mlx5_create_flow_table(ns, &ft_attr);
357 if (IS_ERR(ft->t)) {
358 err = PTR_ERR(ft->t);
359 ft->t = NULL;
360 return err;
361 }
362
363 err = arfs_create_groups(ft, type);
364 if (err)
365 goto err;
366
367 err = arfs_add_default_rule(fs, rx_res, type);
368 if (err)
369 goto err;
370
371 return 0;
372 err:
373 mlx5e_destroy_flow_table(ft);
374 return err;
375 }
376
mlx5e_arfs_create_tables(struct mlx5e_flow_steering * fs,struct mlx5e_rx_res * rx_res,bool ntuple)377 int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs,
378 struct mlx5e_rx_res *rx_res, bool ntuple)
379 {
380 struct mlx5e_arfs_tables *arfs;
381 int err = -ENOMEM;
382 int i;
383
384 if (!ntuple)
385 return 0;
386
387 arfs = kvzalloc(sizeof(*arfs), GFP_KERNEL);
388 if (!arfs)
389 return -ENOMEM;
390
391 spin_lock_init(&arfs->arfs_lock);
392 arfs->wq = create_singlethread_workqueue("mlx5e_arfs");
393 if (!arfs->wq)
394 goto err;
395
396 mlx5e_fs_set_arfs(fs, arfs);
397
398 for (i = 0; i < ARFS_NUM_TYPES; i++) {
399 err = arfs_create_table(fs, rx_res, i);
400 if (err)
401 goto err_des;
402 }
403 return 0;
404
405 err_des:
406 _mlx5e_cleanup_tables(fs);
407 err:
408 mlx5e_fs_set_arfs(fs, NULL);
409 kvfree(arfs);
410 return err;
411 }
412
413 #define MLX5E_ARFS_EXPIRY_QUOTA 60
414
arfs_may_expire_flow(struct mlx5e_priv * priv)415 static void arfs_may_expire_flow(struct mlx5e_priv *priv)
416 {
417 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs);
418 struct arfs_rule *arfs_rule;
419 struct hlist_node *htmp;
420 HLIST_HEAD(del_list);
421 int quota = 0;
422 int i;
423 int j;
424
425 spin_lock_bh(&arfs->arfs_lock);
426 mlx5e_for_each_arfs_rule(arfs_rule, htmp, arfs->arfs_tables, i, j) {
427 if (!work_pending(&arfs_rule->arfs_work) &&
428 rps_may_expire_flow(priv->netdev,
429 arfs_rule->rxq, arfs_rule->flow_id,
430 arfs_rule->filter_id)) {
431 hlist_del_init(&arfs_rule->hlist);
432 hlist_add_head(&arfs_rule->hlist, &del_list);
433 if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA)
434 break;
435 }
436 }
437 spin_unlock_bh(&arfs->arfs_lock);
438 hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) {
439 if (arfs_rule->rule) {
440 mlx5_del_flow_rules(arfs_rule->rule);
441 priv->channel_stats[arfs_rule->rxq]->rq.arfs_expired++;
442 }
443 hlist_del(&arfs_rule->hlist);
444 kfree(arfs_rule);
445 }
446 }
447
arfs_del_rules(struct mlx5e_flow_steering * fs)448 static void arfs_del_rules(struct mlx5e_flow_steering *fs)
449 {
450 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
451 struct hlist_node *htmp;
452 struct arfs_rule *rule;
453 HLIST_HEAD(del_list);
454 int i;
455 int j;
456
457 spin_lock_bh(&arfs->arfs_lock);
458 mlx5e_for_each_arfs_rule(rule, htmp, arfs->arfs_tables, i, j) {
459 hlist_del_init(&rule->hlist);
460 hlist_add_head(&rule->hlist, &del_list);
461 }
462 spin_unlock_bh(&arfs->arfs_lock);
463
464 hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) {
465 cancel_work_sync(&rule->arfs_work);
466 if (rule->rule)
467 mlx5_del_flow_rules(rule->rule);
468 hlist_del(&rule->hlist);
469 kfree(rule);
470 }
471 }
472
473 static struct hlist_head *
arfs_hash_bucket(struct arfs_table * arfs_t,__be16 src_port,__be16 dst_port)474 arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port,
475 __be16 dst_port)
476 {
477 unsigned long l;
478 int bucket_idx;
479
480 l = (__force unsigned long)src_port |
481 ((__force unsigned long)dst_port << 2);
482
483 bucket_idx = hash_long(l, ARFS_HASH_SHIFT);
484
485 return &arfs_t->rules_hash[bucket_idx];
486 }
487
arfs_get_table(struct mlx5e_arfs_tables * arfs,u8 ip_proto,__be16 etype)488 static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs,
489 u8 ip_proto, __be16 etype)
490 {
491 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP)
492 return &arfs->arfs_tables[ARFS_IPV4_TCP];
493 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP)
494 return &arfs->arfs_tables[ARFS_IPV4_UDP];
495 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP)
496 return &arfs->arfs_tables[ARFS_IPV6_TCP];
497 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP)
498 return &arfs->arfs_tables[ARFS_IPV6_UDP];
499
500 return NULL;
501 }
502
arfs_add_rule(struct mlx5e_priv * priv,struct arfs_rule * arfs_rule)503 static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv,
504 struct arfs_rule *arfs_rule)
505 {
506 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs);
507 struct arfs_tuple *tuple = &arfs_rule->tuple;
508 struct mlx5_flow_handle *rule = NULL;
509 struct mlx5_flow_destination dest = {};
510 MLX5_DECLARE_FLOW_ACT(flow_act);
511 struct arfs_table *arfs_table;
512 struct mlx5_flow_spec *spec;
513 struct mlx5_flow_table *ft;
514 int err = 0;
515
516 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
517 if (!spec) {
518 priv->channel_stats[arfs_rule->rxq]->rq.arfs_err++;
519 err = -ENOMEM;
520 goto out;
521 }
522 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
523 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
524 outer_headers.ethertype);
525 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype,
526 ntohs(tuple->etype));
527 arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype);
528 if (!arfs_table) {
529 WARN_ONCE(1, "arfs table does not exist for etype %u and ip_proto %u\n",
530 tuple->etype, tuple->ip_proto);
531 err = -EINVAL;
532 goto out;
533 }
534
535 ft = arfs_table->ft.t;
536 if (tuple->ip_proto == IPPROTO_TCP) {
537 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
538 outer_headers.tcp_dport);
539 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
540 outer_headers.tcp_sport);
541 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
542 ntohs(tuple->dst_port));
543 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
544 ntohs(tuple->src_port));
545 } else {
546 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
547 outer_headers.udp_dport);
548 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
549 outer_headers.udp_sport);
550 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport,
551 ntohs(tuple->dst_port));
552 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport,
553 ntohs(tuple->src_port));
554 }
555 if (tuple->etype == htons(ETH_P_IP)) {
556 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
557 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
558 &tuple->src_ipv4,
559 4);
560 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
561 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
562 &tuple->dst_ipv4,
563 4);
564 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
565 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
566 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
567 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
568 } else {
569 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
570 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
571 &tuple->src_ipv6,
572 16);
573 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
574 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
575 &tuple->dst_ipv6,
576 16);
577 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
578 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
579 0xff,
580 16);
581 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
582 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
583 0xff,
584 16);
585 }
586 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
587 dest.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, arfs_rule->rxq);
588 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
589 if (IS_ERR(rule)) {
590 err = PTR_ERR(rule);
591 priv->channel_stats[arfs_rule->rxq]->rq.arfs_err++;
592 netdev_dbg(priv->netdev,
593 "%s: add rule(filter id=%d, rq idx=%d, ip proto=0x%x) failed,err=%d\n",
594 __func__, arfs_rule->filter_id, arfs_rule->rxq,
595 tuple->ip_proto, err);
596 }
597
598 out:
599 kvfree(spec);
600 return err ? ERR_PTR(err) : rule;
601 }
602
arfs_modify_rule_rq(struct mlx5e_priv * priv,struct mlx5_flow_handle * rule,u16 rxq)603 static void arfs_modify_rule_rq(struct mlx5e_priv *priv,
604 struct mlx5_flow_handle *rule, u16 rxq)
605 {
606 struct mlx5_flow_destination dst = {};
607 int err = 0;
608
609 dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
610 dst.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, rxq);
611 err = mlx5_modify_rule_destination(rule, &dst, NULL);
612 if (err) {
613 priv->channel_stats[rxq]->rq.arfs_err++;
614 netdev_warn(priv->netdev,
615 "Failed to modify aRFS rule destination to rq=%d\n", rxq);
616 }
617 }
618
arfs_handle_work(struct work_struct * work)619 static void arfs_handle_work(struct work_struct *work)
620 {
621 struct arfs_rule *arfs_rule = container_of(work,
622 struct arfs_rule,
623 arfs_work);
624 struct mlx5e_priv *priv = arfs_rule->priv;
625 struct mlx5e_arfs_tables *arfs;
626 struct mlx5_flow_handle *rule;
627
628 arfs = mlx5e_fs_get_arfs(priv->fs);
629 mutex_lock(&priv->state_lock);
630 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
631 spin_lock_bh(&arfs->arfs_lock);
632 hlist_del(&arfs_rule->hlist);
633 spin_unlock_bh(&arfs->arfs_lock);
634
635 mutex_unlock(&priv->state_lock);
636 kfree(arfs_rule);
637 goto out;
638 }
639 mutex_unlock(&priv->state_lock);
640
641 if (!arfs_rule->rule) {
642 rule = arfs_add_rule(priv, arfs_rule);
643 if (IS_ERR(rule))
644 goto out;
645 arfs_rule->rule = rule;
646 priv->channel_stats[arfs_rule->rxq]->rq.arfs_add++;
647 } else {
648 arfs_modify_rule_rq(priv, arfs_rule->rule,
649 arfs_rule->rxq);
650 }
651 out:
652 arfs_may_expire_flow(priv);
653 }
654
arfs_alloc_rule(struct mlx5e_priv * priv,struct arfs_table * arfs_t,const struct flow_keys * fk,u16 rxq,u32 flow_id)655 static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv,
656 struct arfs_table *arfs_t,
657 const struct flow_keys *fk,
658 u16 rxq, u32 flow_id)
659 {
660 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs);
661 struct arfs_rule *rule;
662 struct arfs_tuple *tuple;
663
664 rule = kzalloc(sizeof(*rule), GFP_ATOMIC);
665 if (!rule) {
666 priv->channel_stats[rxq]->rq.arfs_err++;
667 return NULL;
668 }
669
670 rule->priv = priv;
671 rule->rxq = rxq;
672 INIT_WORK(&rule->arfs_work, arfs_handle_work);
673
674 tuple = &rule->tuple;
675 tuple->etype = fk->basic.n_proto;
676 tuple->ip_proto = fk->basic.ip_proto;
677 if (tuple->etype == htons(ETH_P_IP)) {
678 tuple->src_ipv4 = fk->addrs.v4addrs.src;
679 tuple->dst_ipv4 = fk->addrs.v4addrs.dst;
680 } else {
681 memcpy(&tuple->src_ipv6, &fk->addrs.v6addrs.src,
682 sizeof(struct in6_addr));
683 memcpy(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst,
684 sizeof(struct in6_addr));
685 }
686 tuple->src_port = fk->ports.src;
687 tuple->dst_port = fk->ports.dst;
688
689 rule->flow_id = flow_id;
690 rule->filter_id = arfs->last_filter_id++ % RPS_NO_FILTER;
691
692 hlist_add_head(&rule->hlist,
693 arfs_hash_bucket(arfs_t, tuple->src_port,
694 tuple->dst_port));
695 return rule;
696 }
697
arfs_cmp(const struct arfs_tuple * tuple,const struct flow_keys * fk)698 static bool arfs_cmp(const struct arfs_tuple *tuple, const struct flow_keys *fk)
699 {
700 if (tuple->src_port != fk->ports.src || tuple->dst_port != fk->ports.dst)
701 return false;
702 if (tuple->etype != fk->basic.n_proto)
703 return false;
704 if (tuple->etype == htons(ETH_P_IP))
705 return tuple->src_ipv4 == fk->addrs.v4addrs.src &&
706 tuple->dst_ipv4 == fk->addrs.v4addrs.dst;
707 if (tuple->etype == htons(ETH_P_IPV6))
708 return !memcmp(&tuple->src_ipv6, &fk->addrs.v6addrs.src,
709 sizeof(struct in6_addr)) &&
710 !memcmp(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst,
711 sizeof(struct in6_addr));
712 return false;
713 }
714
arfs_find_rule(struct arfs_table * arfs_t,const struct flow_keys * fk)715 static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t,
716 const struct flow_keys *fk)
717 {
718 struct arfs_rule *arfs_rule;
719 struct hlist_head *head;
720
721 head = arfs_hash_bucket(arfs_t, fk->ports.src, fk->ports.dst);
722 hlist_for_each_entry(arfs_rule, head, hlist) {
723 if (arfs_cmp(&arfs_rule->tuple, fk))
724 return arfs_rule;
725 }
726
727 return NULL;
728 }
729
mlx5e_rx_flow_steer(struct net_device * dev,const struct sk_buff * skb,u16 rxq_index,u32 flow_id)730 int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
731 u16 rxq_index, u32 flow_id)
732 {
733 struct mlx5e_priv *priv = netdev_priv(dev);
734 struct mlx5e_arfs_tables *arfs;
735 struct arfs_rule *arfs_rule;
736 struct arfs_table *arfs_t;
737 struct flow_keys fk;
738
739 arfs = mlx5e_fs_get_arfs(priv->fs);
740 if (!skb_flow_dissect_flow_keys(skb, &fk, 0))
741 return -EPROTONOSUPPORT;
742
743 if (fk.basic.n_proto != htons(ETH_P_IP) &&
744 fk.basic.n_proto != htons(ETH_P_IPV6))
745 return -EPROTONOSUPPORT;
746
747 if (skb->encapsulation)
748 return -EPROTONOSUPPORT;
749
750 arfs_t = arfs_get_table(arfs, fk.basic.ip_proto, fk.basic.n_proto);
751 if (!arfs_t)
752 return -EPROTONOSUPPORT;
753
754 spin_lock_bh(&arfs->arfs_lock);
755 arfs_rule = arfs_find_rule(arfs_t, &fk);
756 if (arfs_rule) {
757 if (arfs_rule->rxq == rxq_index || work_busy(&arfs_rule->arfs_work)) {
758 spin_unlock_bh(&arfs->arfs_lock);
759 return arfs_rule->filter_id;
760 }
761
762 priv->channel_stats[rxq_index]->rq.arfs_request_in++;
763 priv->channel_stats[arfs_rule->rxq]->rq.arfs_request_out++;
764 arfs_rule->rxq = rxq_index;
765 } else {
766 arfs_rule = arfs_alloc_rule(priv, arfs_t, &fk, rxq_index, flow_id);
767 if (!arfs_rule) {
768 spin_unlock_bh(&arfs->arfs_lock);
769 return -ENOMEM;
770 }
771 }
772 queue_work(arfs->wq, &arfs_rule->arfs_work);
773 spin_unlock_bh(&arfs->arfs_lock);
774 return arfs_rule->filter_id;
775 }
776
777