1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2019 Netronome Systems, Inc. */
3
4 #include <linux/hash.h>
5 #include <linux/hashtable.h>
6 #include <linux/jhash.h>
7 #include <linux/math64.h>
8 #include <linux/vmalloc.h>
9 #include <net/pkt_cls.h>
10 #include <net/pkt_sched.h>
11
12 #include "cmsg.h"
13 #include "main.h"
14 #include "../nfp_port.h"
15
16 #define NFP_FL_QOS_UPDATE msecs_to_jiffies(1000)
17 #define NFP_FL_QOS_PPS BIT(15)
18 #define NFP_FL_QOS_METER BIT(10)
19
20 struct nfp_police_cfg_head {
21 __be32 flags_opts;
22 union {
23 __be32 meter_id;
24 __be32 port;
25 };
26 };
27
28 enum NFP_FL_QOS_TYPES {
29 NFP_FL_QOS_TYPE_BPS,
30 NFP_FL_QOS_TYPE_PPS,
31 NFP_FL_QOS_TYPE_MAX,
32 };
33
34 /* Police cmsg for configuring a trTCM traffic conditioner (8W/32B)
35 * See RFC 2698 for more details.
36 * ----------------------------------------------------------------
37 * 3 2 1
38 * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
39 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
40 * | Reserved |p| Reserved |
41 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
42 * | Port Ingress |
43 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
44 * | Token Bucket Peak |
45 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
46 * | Token Bucket Committed |
47 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
48 * | Peak Burst Size |
49 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
50 * | Committed Burst Size |
51 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
52 * | Peak Information Rate |
53 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
54 * | Committed Information Rate |
55 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
56 * Word[0](FLag options):
57 * [15] p(pps) 1 for pps, 0 for bps
58 *
59 * Meter control message
60 * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
61 * +-------------------------------+-+---+-----+-+---------+-+---+-+
62 * | Reserved |p| Y |TYPE |E|TSHFV |P| PC|R|
63 * +-------------------------------+-+---+-----+-+---------+-+---+-+
64 * | meter ID |
65 * +-------------------------------+-------------------------------+
66 *
67 */
68 struct nfp_police_config {
69 struct nfp_police_cfg_head head;
70 __be32 bkt_tkn_p;
71 __be32 bkt_tkn_c;
72 __be32 pbs;
73 __be32 cbs;
74 __be32 pir;
75 __be32 cir;
76 };
77
78 struct nfp_police_stats_reply {
79 struct nfp_police_cfg_head head;
80 __be64 pass_bytes;
81 __be64 pass_pkts;
82 __be64 drop_bytes;
83 __be64 drop_pkts;
84 };
85
nfp_flower_offload_one_police(struct nfp_app * app,bool ingress,bool pps,u32 id,u32 rate,u32 burst)86 int nfp_flower_offload_one_police(struct nfp_app *app, bool ingress,
87 bool pps, u32 id, u32 rate, u32 burst)
88 {
89 struct nfp_police_config *config;
90 struct sk_buff *skb;
91
92 skb = nfp_flower_cmsg_alloc(app, sizeof(struct nfp_police_config),
93 NFP_FLOWER_CMSG_TYPE_QOS_MOD, GFP_KERNEL);
94 if (!skb)
95 return -ENOMEM;
96
97 config = nfp_flower_cmsg_get_data(skb);
98 memset(config, 0, sizeof(struct nfp_police_config));
99 if (pps)
100 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS);
101 if (!ingress)
102 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_METER);
103
104 if (ingress)
105 config->head.port = cpu_to_be32(id);
106 else
107 config->head.meter_id = cpu_to_be32(id);
108
109 config->bkt_tkn_p = cpu_to_be32(burst);
110 config->bkt_tkn_c = cpu_to_be32(burst);
111 config->pbs = cpu_to_be32(burst);
112 config->cbs = cpu_to_be32(burst);
113 config->pir = cpu_to_be32(rate);
114 config->cir = cpu_to_be32(rate);
115 nfp_ctrl_tx(app->ctrl, skb);
116
117 return 0;
118 }
119
nfp_policer_validate(const struct flow_action * action,const struct flow_action_entry * act,struct netlink_ext_ack * extack)120 static int nfp_policer_validate(const struct flow_action *action,
121 const struct flow_action_entry *act,
122 struct netlink_ext_ack *extack)
123 {
124 if (act->police.exceed.act_id != FLOW_ACTION_DROP) {
125 NL_SET_ERR_MSG_MOD(extack,
126 "Offload not supported when exceed action is not drop");
127 return -EOPNOTSUPP;
128 }
129
130 if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE &&
131 act->police.notexceed.act_id != FLOW_ACTION_PIPE &&
132 act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
133 NL_SET_ERR_MSG_MOD(extack,
134 "Offload not supported when conform action is not continue, pipe or ok");
135 return -EOPNOTSUPP;
136 }
137
138 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT &&
139 !flow_action_is_last_entry(action, act)) {
140 NL_SET_ERR_MSG_MOD(extack,
141 "Offload not supported when conform action is ok, but action is not last");
142 return -EOPNOTSUPP;
143 }
144
145 if (act->police.peakrate_bytes_ps ||
146 act->police.avrate || act->police.overhead) {
147 NL_SET_ERR_MSG_MOD(extack,
148 "Offload not supported when peakrate/avrate/overhead is configured");
149 return -EOPNOTSUPP;
150 }
151
152 return 0;
153 }
154
155 static int
nfp_flower_install_rate_limiter(struct nfp_app * app,struct net_device * netdev,struct tc_cls_matchall_offload * flow,struct netlink_ext_ack * extack)156 nfp_flower_install_rate_limiter(struct nfp_app *app, struct net_device *netdev,
157 struct tc_cls_matchall_offload *flow,
158 struct netlink_ext_ack *extack)
159 {
160 struct flow_action_entry *paction = &flow->rule->action.entries[0];
161 u32 action_num = flow->rule->action.num_entries;
162 struct nfp_flower_priv *fl_priv = app->priv;
163 struct flow_action_entry *action = NULL;
164 struct nfp_flower_repr_priv *repr_priv;
165 u32 netdev_port_id, i;
166 struct nfp_repr *repr;
167 bool pps_support;
168 u32 bps_num = 0;
169 u32 pps_num = 0;
170 u32 burst;
171 bool pps;
172 u64 rate;
173 int err;
174
175 if (!nfp_netdev_is_nfp_repr(netdev)) {
176 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port");
177 return -EOPNOTSUPP;
178 }
179 repr = netdev_priv(netdev);
180 repr_priv = repr->app_priv;
181 netdev_port_id = nfp_repr_get_port_id(netdev);
182 pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS);
183
184 if (repr_priv->block_shared) {
185 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on shared blocks");
186 return -EOPNOTSUPP;
187 }
188
189 if (repr->port->type != NFP_PORT_VF_PORT) {
190 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on non-VF ports");
191 return -EOPNOTSUPP;
192 }
193
194 if (pps_support) {
195 if (action_num > 2 || action_num == 0) {
196 NL_SET_ERR_MSG_MOD(extack,
197 "unsupported offload: qos rate limit offload only support action number 1 or 2");
198 return -EOPNOTSUPP;
199 }
200 } else {
201 if (!flow_offload_has_one_action(&flow->rule->action)) {
202 NL_SET_ERR_MSG_MOD(extack,
203 "unsupported offload: qos rate limit offload requires a single action");
204 return -EOPNOTSUPP;
205 }
206 }
207
208 if (flow->common.prio != 1) {
209 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload requires highest priority");
210 return -EOPNOTSUPP;
211 }
212
213 for (i = 0 ; i < action_num; i++) {
214 action = paction + i;
215 if (action->id != FLOW_ACTION_POLICE) {
216 NL_SET_ERR_MSG_MOD(extack,
217 "unsupported offload: qos rate limit offload requires police action");
218 return -EOPNOTSUPP;
219 }
220
221 err = nfp_policer_validate(&flow->rule->action, action, extack);
222 if (err)
223 return err;
224
225 if (action->police.rate_bytes_ps > 0) {
226 if (bps_num++) {
227 NL_SET_ERR_MSG_MOD(extack,
228 "unsupported offload: qos rate limit offload only support one BPS action");
229 return -EOPNOTSUPP;
230 }
231 }
232 if (action->police.rate_pkt_ps > 0) {
233 if (!pps_support) {
234 NL_SET_ERR_MSG_MOD(extack,
235 "unsupported offload: FW does not support PPS action");
236 return -EOPNOTSUPP;
237 }
238 if (pps_num++) {
239 NL_SET_ERR_MSG_MOD(extack,
240 "unsupported offload: qos rate limit offload only support one PPS action");
241 return -EOPNOTSUPP;
242 }
243 }
244 }
245
246 for (i = 0 ; i < action_num; i++) {
247 /* Set QoS data for this interface */
248 action = paction + i;
249 if (action->police.rate_bytes_ps > 0) {
250 rate = action->police.rate_bytes_ps;
251 burst = action->police.burst;
252 } else if (action->police.rate_pkt_ps > 0) {
253 rate = action->police.rate_pkt_ps;
254 burst = action->police.burst_pkt;
255 } else {
256 NL_SET_ERR_MSG_MOD(extack,
257 "unsupported offload: qos rate limit is not BPS or PPS");
258 continue;
259 }
260
261 if (rate != 0) {
262 pps = false;
263 if (action->police.rate_pkt_ps > 0)
264 pps = true;
265 nfp_flower_offload_one_police(repr->app, true,
266 pps, netdev_port_id,
267 rate, burst);
268 }
269 }
270 repr_priv->qos_table.netdev_port_id = netdev_port_id;
271 fl_priv->qos_rate_limiters++;
272 if (fl_priv->qos_rate_limiters == 1)
273 schedule_delayed_work(&fl_priv->qos_stats_work,
274 NFP_FL_QOS_UPDATE);
275
276 return 0;
277 }
278
279 static int
nfp_flower_remove_rate_limiter(struct nfp_app * app,struct net_device * netdev,struct tc_cls_matchall_offload * flow,struct netlink_ext_ack * extack)280 nfp_flower_remove_rate_limiter(struct nfp_app *app, struct net_device *netdev,
281 struct tc_cls_matchall_offload *flow,
282 struct netlink_ext_ack *extack)
283 {
284 struct nfp_flower_priv *fl_priv = app->priv;
285 struct nfp_flower_repr_priv *repr_priv;
286 struct nfp_police_config *config;
287 u32 netdev_port_id, i;
288 struct nfp_repr *repr;
289 struct sk_buff *skb;
290 bool pps_support;
291
292 if (!nfp_netdev_is_nfp_repr(netdev)) {
293 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port");
294 return -EOPNOTSUPP;
295 }
296 repr = netdev_priv(netdev);
297
298 netdev_port_id = nfp_repr_get_port_id(netdev);
299 repr_priv = repr->app_priv;
300 pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS);
301
302 if (!repr_priv->qos_table.netdev_port_id) {
303 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot remove qos entry that does not exist");
304 return -EOPNOTSUPP;
305 }
306
307 memset(&repr_priv->qos_table, 0, sizeof(struct nfp_fl_qos));
308 fl_priv->qos_rate_limiters--;
309 if (!fl_priv->qos_rate_limiters)
310 cancel_delayed_work_sync(&fl_priv->qos_stats_work);
311 for (i = 0 ; i < NFP_FL_QOS_TYPE_MAX; i++) {
312 if (i == NFP_FL_QOS_TYPE_PPS && !pps_support)
313 break;
314 /* 0:bps 1:pps
315 * Clear QoS data for this interface.
316 * There is no need to check if a specific QOS_TYPE was
317 * configured as the firmware handles clearing a QoS entry
318 * safely, even if it wasn't explicitly added.
319 */
320 skb = nfp_flower_cmsg_alloc(repr->app, sizeof(struct nfp_police_config),
321 NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL);
322 if (!skb)
323 return -ENOMEM;
324
325 config = nfp_flower_cmsg_get_data(skb);
326 memset(config, 0, sizeof(struct nfp_police_config));
327 if (i == NFP_FL_QOS_TYPE_PPS)
328 config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_PPS);
329 config->head.port = cpu_to_be32(netdev_port_id);
330 nfp_ctrl_tx(repr->app->ctrl, skb);
331 }
332
333 return 0;
334 }
335
nfp_flower_stats_rlim_reply(struct nfp_app * app,struct sk_buff * skb)336 void nfp_flower_stats_rlim_reply(struct nfp_app *app, struct sk_buff *skb)
337 {
338 struct nfp_flower_priv *fl_priv = app->priv;
339 struct nfp_flower_repr_priv *repr_priv;
340 struct nfp_police_stats_reply *msg;
341 struct nfp_stat_pair *curr_stats;
342 struct nfp_stat_pair *prev_stats;
343 struct net_device *netdev;
344 struct nfp_repr *repr;
345 u32 netdev_port_id;
346
347 msg = nfp_flower_cmsg_get_data(skb);
348 if (be32_to_cpu(msg->head.flags_opts) & NFP_FL_QOS_METER)
349 return nfp_act_stats_reply(app, msg);
350
351 netdev_port_id = be32_to_cpu(msg->head.port);
352 rcu_read_lock();
353 netdev = nfp_app_dev_get(app, netdev_port_id, NULL);
354 if (!netdev)
355 goto exit_unlock_rcu;
356
357 repr = netdev_priv(netdev);
358 repr_priv = repr->app_priv;
359 curr_stats = &repr_priv->qos_table.curr_stats;
360 prev_stats = &repr_priv->qos_table.prev_stats;
361
362 spin_lock_bh(&fl_priv->qos_stats_lock);
363 curr_stats->pkts = be64_to_cpu(msg->pass_pkts) +
364 be64_to_cpu(msg->drop_pkts);
365 curr_stats->bytes = be64_to_cpu(msg->pass_bytes) +
366 be64_to_cpu(msg->drop_bytes);
367
368 if (!repr_priv->qos_table.last_update) {
369 prev_stats->pkts = curr_stats->pkts;
370 prev_stats->bytes = curr_stats->bytes;
371 }
372
373 repr_priv->qos_table.last_update = jiffies;
374 spin_unlock_bh(&fl_priv->qos_stats_lock);
375
376 exit_unlock_rcu:
377 rcu_read_unlock();
378 }
379
380 static void
nfp_flower_stats_rlim_request(struct nfp_flower_priv * fl_priv,u32 id,bool ingress)381 nfp_flower_stats_rlim_request(struct nfp_flower_priv *fl_priv,
382 u32 id, bool ingress)
383 {
384 struct nfp_police_cfg_head *head;
385 struct sk_buff *skb;
386
387 skb = nfp_flower_cmsg_alloc(fl_priv->app,
388 sizeof(struct nfp_police_cfg_head),
389 NFP_FLOWER_CMSG_TYPE_QOS_STATS,
390 GFP_ATOMIC);
391 if (!skb)
392 return;
393 head = nfp_flower_cmsg_get_data(skb);
394
395 memset(head, 0, sizeof(struct nfp_police_cfg_head));
396 if (ingress) {
397 head->port = cpu_to_be32(id);
398 } else {
399 head->flags_opts = cpu_to_be32(NFP_FL_QOS_METER);
400 head->meter_id = cpu_to_be32(id);
401 }
402
403 nfp_ctrl_tx(fl_priv->app->ctrl, skb);
404 }
405
406 static void
nfp_flower_stats_rlim_request_all(struct nfp_flower_priv * fl_priv)407 nfp_flower_stats_rlim_request_all(struct nfp_flower_priv *fl_priv)
408 {
409 struct nfp_reprs *repr_set;
410 int i;
411
412 rcu_read_lock();
413 repr_set = rcu_dereference(fl_priv->app->reprs[NFP_REPR_TYPE_VF]);
414 if (!repr_set)
415 goto exit_unlock_rcu;
416
417 for (i = 0; i < repr_set->num_reprs; i++) {
418 struct net_device *netdev;
419
420 netdev = rcu_dereference(repr_set->reprs[i]);
421 if (netdev) {
422 struct nfp_repr *priv = netdev_priv(netdev);
423 struct nfp_flower_repr_priv *repr_priv;
424 u32 netdev_port_id;
425
426 repr_priv = priv->app_priv;
427 netdev_port_id = repr_priv->qos_table.netdev_port_id;
428 if (!netdev_port_id)
429 continue;
430
431 nfp_flower_stats_rlim_request(fl_priv,
432 netdev_port_id, true);
433 }
434 }
435
436 exit_unlock_rcu:
437 rcu_read_unlock();
438 }
439
update_stats_cache(struct work_struct * work)440 static void update_stats_cache(struct work_struct *work)
441 {
442 struct delayed_work *delayed_work;
443 struct nfp_flower_priv *fl_priv;
444
445 delayed_work = to_delayed_work(work);
446 fl_priv = container_of(delayed_work, struct nfp_flower_priv,
447 qos_stats_work);
448
449 nfp_flower_stats_rlim_request_all(fl_priv);
450 nfp_flower_stats_meter_request_all(fl_priv);
451
452 schedule_delayed_work(&fl_priv->qos_stats_work, NFP_FL_QOS_UPDATE);
453 }
454
455 static int
nfp_flower_stats_rate_limiter(struct nfp_app * app,struct net_device * netdev,struct tc_cls_matchall_offload * flow,struct netlink_ext_ack * extack)456 nfp_flower_stats_rate_limiter(struct nfp_app *app, struct net_device *netdev,
457 struct tc_cls_matchall_offload *flow,
458 struct netlink_ext_ack *extack)
459 {
460 struct nfp_flower_priv *fl_priv = app->priv;
461 struct nfp_flower_repr_priv *repr_priv;
462 struct nfp_stat_pair *curr_stats;
463 struct nfp_stat_pair *prev_stats;
464 u64 diff_bytes, diff_pkts;
465 struct nfp_repr *repr;
466
467 if (!nfp_netdev_is_nfp_repr(netdev)) {
468 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port");
469 return -EOPNOTSUPP;
470 }
471 repr = netdev_priv(netdev);
472
473 repr_priv = repr->app_priv;
474 if (!repr_priv->qos_table.netdev_port_id) {
475 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot find qos entry for stats update");
476 return -EOPNOTSUPP;
477 }
478
479 spin_lock_bh(&fl_priv->qos_stats_lock);
480 curr_stats = &repr_priv->qos_table.curr_stats;
481 prev_stats = &repr_priv->qos_table.prev_stats;
482 diff_pkts = curr_stats->pkts - prev_stats->pkts;
483 diff_bytes = curr_stats->bytes - prev_stats->bytes;
484 prev_stats->pkts = curr_stats->pkts;
485 prev_stats->bytes = curr_stats->bytes;
486 spin_unlock_bh(&fl_priv->qos_stats_lock);
487
488 flow_stats_update(&flow->stats, diff_bytes, diff_pkts, 0,
489 repr_priv->qos_table.last_update,
490 FLOW_ACTION_HW_STATS_DELAYED);
491 return 0;
492 }
493
nfp_flower_qos_init(struct nfp_app * app)494 void nfp_flower_qos_init(struct nfp_app *app)
495 {
496 struct nfp_flower_priv *fl_priv = app->priv;
497
498 spin_lock_init(&fl_priv->qos_stats_lock);
499 mutex_init(&fl_priv->meter_stats_lock);
500 nfp_init_meter_table(app);
501
502 INIT_DELAYED_WORK(&fl_priv->qos_stats_work, &update_stats_cache);
503 }
504
nfp_flower_qos_cleanup(struct nfp_app * app)505 void nfp_flower_qos_cleanup(struct nfp_app *app)
506 {
507 struct nfp_flower_priv *fl_priv = app->priv;
508
509 cancel_delayed_work_sync(&fl_priv->qos_stats_work);
510 }
511
nfp_flower_setup_qos_offload(struct nfp_app * app,struct net_device * netdev,struct tc_cls_matchall_offload * flow)512 int nfp_flower_setup_qos_offload(struct nfp_app *app, struct net_device *netdev,
513 struct tc_cls_matchall_offload *flow)
514 {
515 struct netlink_ext_ack *extack = flow->common.extack;
516 struct nfp_flower_priv *fl_priv = app->priv;
517
518 if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM)) {
519 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: loaded firmware does not support qos rate limit offload");
520 return -EOPNOTSUPP;
521 }
522
523 switch (flow->command) {
524 case TC_CLSMATCHALL_REPLACE:
525 return nfp_flower_install_rate_limiter(app, netdev, flow,
526 extack);
527 case TC_CLSMATCHALL_DESTROY:
528 return nfp_flower_remove_rate_limiter(app, netdev, flow,
529 extack);
530 case TC_CLSMATCHALL_STATS:
531 return nfp_flower_stats_rate_limiter(app, netdev, flow,
532 extack);
533 default:
534 return -EOPNOTSUPP;
535 }
536 }
537
538 /* offload tc action, currently only for tc police */
539
540 static const struct rhashtable_params stats_meter_table_params = {
541 .key_offset = offsetof(struct nfp_meter_entry, meter_id),
542 .head_offset = offsetof(struct nfp_meter_entry, ht_node),
543 .key_len = sizeof(u32),
544 };
545
546 struct nfp_meter_entry *
nfp_flower_search_meter_entry(struct nfp_app * app,u32 meter_id)547 nfp_flower_search_meter_entry(struct nfp_app *app, u32 meter_id)
548 {
549 struct nfp_flower_priv *priv = app->priv;
550
551 return rhashtable_lookup_fast(&priv->meter_table, &meter_id,
552 stats_meter_table_params);
553 }
554
555 static struct nfp_meter_entry *
nfp_flower_add_meter_entry(struct nfp_app * app,u32 meter_id)556 nfp_flower_add_meter_entry(struct nfp_app *app, u32 meter_id)
557 {
558 struct nfp_meter_entry *meter_entry = NULL;
559 struct nfp_flower_priv *priv = app->priv;
560
561 meter_entry = rhashtable_lookup_fast(&priv->meter_table,
562 &meter_id,
563 stats_meter_table_params);
564 if (meter_entry)
565 return meter_entry;
566
567 meter_entry = kzalloc(sizeof(*meter_entry), GFP_KERNEL);
568 if (!meter_entry)
569 return NULL;
570
571 meter_entry->meter_id = meter_id;
572 meter_entry->used = jiffies;
573 if (rhashtable_insert_fast(&priv->meter_table, &meter_entry->ht_node,
574 stats_meter_table_params)) {
575 kfree(meter_entry);
576 return NULL;
577 }
578
579 priv->qos_rate_limiters++;
580 if (priv->qos_rate_limiters == 1)
581 schedule_delayed_work(&priv->qos_stats_work,
582 NFP_FL_QOS_UPDATE);
583
584 return meter_entry;
585 }
586
nfp_flower_del_meter_entry(struct nfp_app * app,u32 meter_id)587 static void nfp_flower_del_meter_entry(struct nfp_app *app, u32 meter_id)
588 {
589 struct nfp_meter_entry *meter_entry = NULL;
590 struct nfp_flower_priv *priv = app->priv;
591
592 meter_entry = rhashtable_lookup_fast(&priv->meter_table, &meter_id,
593 stats_meter_table_params);
594 if (!meter_entry)
595 return;
596
597 rhashtable_remove_fast(&priv->meter_table,
598 &meter_entry->ht_node,
599 stats_meter_table_params);
600 kfree(meter_entry);
601 priv->qos_rate_limiters--;
602 if (!priv->qos_rate_limiters)
603 cancel_delayed_work_sync(&priv->qos_stats_work);
604 }
605
nfp_flower_setup_meter_entry(struct nfp_app * app,const struct flow_action_entry * action,enum nfp_meter_op op,u32 meter_id)606 int nfp_flower_setup_meter_entry(struct nfp_app *app,
607 const struct flow_action_entry *action,
608 enum nfp_meter_op op,
609 u32 meter_id)
610 {
611 struct nfp_flower_priv *fl_priv = app->priv;
612 struct nfp_meter_entry *meter_entry = NULL;
613 int err = 0;
614
615 mutex_lock(&fl_priv->meter_stats_lock);
616
617 switch (op) {
618 case NFP_METER_DEL:
619 nfp_flower_del_meter_entry(app, meter_id);
620 goto exit_unlock;
621 case NFP_METER_ADD:
622 meter_entry = nfp_flower_add_meter_entry(app, meter_id);
623 break;
624 default:
625 err = -EOPNOTSUPP;
626 goto exit_unlock;
627 }
628
629 if (!meter_entry) {
630 err = -ENOMEM;
631 goto exit_unlock;
632 }
633
634 if (action->police.rate_bytes_ps > 0) {
635 meter_entry->bps = true;
636 meter_entry->rate = action->police.rate_bytes_ps;
637 meter_entry->burst = action->police.burst;
638 } else {
639 meter_entry->bps = false;
640 meter_entry->rate = action->police.rate_pkt_ps;
641 meter_entry->burst = action->police.burst_pkt;
642 }
643
644 exit_unlock:
645 mutex_unlock(&fl_priv->meter_stats_lock);
646 return err;
647 }
648
nfp_init_meter_table(struct nfp_app * app)649 int nfp_init_meter_table(struct nfp_app *app)
650 {
651 struct nfp_flower_priv *priv = app->priv;
652
653 return rhashtable_init(&priv->meter_table, &stats_meter_table_params);
654 }
655
656 void
nfp_flower_stats_meter_request_all(struct nfp_flower_priv * fl_priv)657 nfp_flower_stats_meter_request_all(struct nfp_flower_priv *fl_priv)
658 {
659 struct nfp_meter_entry *meter_entry = NULL;
660 struct rhashtable_iter iter;
661
662 mutex_lock(&fl_priv->meter_stats_lock);
663 rhashtable_walk_enter(&fl_priv->meter_table, &iter);
664 rhashtable_walk_start(&iter);
665
666 while ((meter_entry = rhashtable_walk_next(&iter)) != NULL) {
667 if (IS_ERR(meter_entry))
668 continue;
669 nfp_flower_stats_rlim_request(fl_priv,
670 meter_entry->meter_id, false);
671 }
672
673 rhashtable_walk_stop(&iter);
674 rhashtable_walk_exit(&iter);
675 mutex_unlock(&fl_priv->meter_stats_lock);
676 }
677
678 static int
nfp_act_install_actions(struct nfp_app * app,struct flow_offload_action * fl_act,struct netlink_ext_ack * extack)679 nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
680 struct netlink_ext_ack *extack)
681 {
682 struct flow_action_entry *paction = &fl_act->action.entries[0];
683 u32 action_num = fl_act->action.num_entries;
684 struct nfp_flower_priv *fl_priv = app->priv;
685 struct flow_action_entry *action = NULL;
686 u32 burst, i, meter_id;
687 bool pps_support, pps;
688 bool add = false;
689 u64 rate;
690
691 pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS);
692
693 for (i = 0 ; i < action_num; i++) {
694 /*set qos associate data for this interface */
695 action = paction + i;
696 if (action->id != FLOW_ACTION_POLICE) {
697 NL_SET_ERR_MSG_MOD(extack,
698 "unsupported offload: qos rate limit offload requires police action");
699 continue;
700 }
701 if (action->police.rate_bytes_ps > 0) {
702 rate = action->police.rate_bytes_ps;
703 burst = action->police.burst;
704 } else if (action->police.rate_pkt_ps > 0 && pps_support) {
705 rate = action->police.rate_pkt_ps;
706 burst = action->police.burst_pkt;
707 } else {
708 NL_SET_ERR_MSG_MOD(extack,
709 "unsupported offload: unsupported qos rate limit");
710 continue;
711 }
712
713 if (rate != 0) {
714 meter_id = action->hw_index;
715 if (nfp_flower_setup_meter_entry(app, action, NFP_METER_ADD, meter_id))
716 continue;
717
718 pps = false;
719 if (action->police.rate_pkt_ps > 0)
720 pps = true;
721 nfp_flower_offload_one_police(app, false, pps, meter_id,
722 rate, burst);
723 add = true;
724 }
725 }
726
727 return add ? 0 : -EOPNOTSUPP;
728 }
729
730 static int
nfp_act_remove_actions(struct nfp_app * app,struct flow_offload_action * fl_act,struct netlink_ext_ack * extack)731 nfp_act_remove_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
732 struct netlink_ext_ack *extack)
733 {
734 struct nfp_meter_entry *meter_entry = NULL;
735 struct nfp_police_config *config;
736 struct sk_buff *skb;
737 u32 meter_id;
738 bool pps;
739
740 /*delete qos associate data for this interface */
741 if (fl_act->id != FLOW_ACTION_POLICE) {
742 NL_SET_ERR_MSG_MOD(extack,
743 "unsupported offload: qos rate limit offload requires police action");
744 return -EOPNOTSUPP;
745 }
746
747 meter_id = fl_act->index;
748 meter_entry = nfp_flower_search_meter_entry(app, meter_id);
749 if (!meter_entry) {
750 NL_SET_ERR_MSG_MOD(extack,
751 "no meter entry when delete the action index.");
752 return -ENOENT;
753 }
754 pps = !meter_entry->bps;
755
756 skb = nfp_flower_cmsg_alloc(app, sizeof(struct nfp_police_config),
757 NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL);
758 if (!skb)
759 return -ENOMEM;
760
761 config = nfp_flower_cmsg_get_data(skb);
762 memset(config, 0, sizeof(struct nfp_police_config));
763 config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_METER);
764 config->head.meter_id = cpu_to_be32(meter_id);
765 if (pps)
766 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS);
767
768 nfp_ctrl_tx(app->ctrl, skb);
769 nfp_flower_setup_meter_entry(app, NULL, NFP_METER_DEL, meter_id);
770
771 return 0;
772 }
773
774 void
nfp_act_stats_reply(struct nfp_app * app,void * pmsg)775 nfp_act_stats_reply(struct nfp_app *app, void *pmsg)
776 {
777 struct nfp_flower_priv *fl_priv = app->priv;
778 struct nfp_meter_entry *meter_entry = NULL;
779 struct nfp_police_stats_reply *msg = pmsg;
780 u32 meter_id;
781
782 meter_id = be32_to_cpu(msg->head.meter_id);
783 mutex_lock(&fl_priv->meter_stats_lock);
784
785 meter_entry = nfp_flower_search_meter_entry(app, meter_id);
786 if (!meter_entry)
787 goto exit_unlock;
788
789 meter_entry->stats.curr.pkts = be64_to_cpu(msg->pass_pkts) +
790 be64_to_cpu(msg->drop_pkts);
791 meter_entry->stats.curr.bytes = be64_to_cpu(msg->pass_bytes) +
792 be64_to_cpu(msg->drop_bytes);
793 meter_entry->stats.curr.drops = be64_to_cpu(msg->drop_pkts);
794 if (!meter_entry->stats.update) {
795 meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts;
796 meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes;
797 meter_entry->stats.prev.drops = meter_entry->stats.curr.drops;
798 }
799
800 meter_entry->stats.update = jiffies;
801
802 exit_unlock:
803 mutex_unlock(&fl_priv->meter_stats_lock);
804 }
805
806 static int
nfp_act_stats_actions(struct nfp_app * app,struct flow_offload_action * fl_act,struct netlink_ext_ack * extack)807 nfp_act_stats_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
808 struct netlink_ext_ack *extack)
809 {
810 struct nfp_flower_priv *fl_priv = app->priv;
811 struct nfp_meter_entry *meter_entry = NULL;
812 u64 diff_bytes, diff_pkts, diff_drops;
813 int err = 0;
814
815 if (fl_act->id != FLOW_ACTION_POLICE) {
816 NL_SET_ERR_MSG_MOD(extack,
817 "unsupported offload: qos rate limit offload requires police action");
818 return -EOPNOTSUPP;
819 }
820
821 mutex_lock(&fl_priv->meter_stats_lock);
822 meter_entry = nfp_flower_search_meter_entry(app, fl_act->index);
823 if (!meter_entry) {
824 err = -ENOENT;
825 goto exit_unlock;
826 }
827 diff_pkts = meter_entry->stats.curr.pkts > meter_entry->stats.prev.pkts ?
828 meter_entry->stats.curr.pkts - meter_entry->stats.prev.pkts : 0;
829 diff_bytes = meter_entry->stats.curr.bytes > meter_entry->stats.prev.bytes ?
830 meter_entry->stats.curr.bytes - meter_entry->stats.prev.bytes : 0;
831 diff_drops = meter_entry->stats.curr.drops > meter_entry->stats.prev.drops ?
832 meter_entry->stats.curr.drops - meter_entry->stats.prev.drops : 0;
833
834 flow_stats_update(&fl_act->stats, diff_bytes, diff_pkts, diff_drops,
835 meter_entry->stats.update,
836 FLOW_ACTION_HW_STATS_DELAYED);
837
838 meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts;
839 meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes;
840 meter_entry->stats.prev.drops = meter_entry->stats.curr.drops;
841
842 exit_unlock:
843 mutex_unlock(&fl_priv->meter_stats_lock);
844 return err;
845 }
846
nfp_setup_tc_act_offload(struct nfp_app * app,struct flow_offload_action * fl_act)847 int nfp_setup_tc_act_offload(struct nfp_app *app,
848 struct flow_offload_action *fl_act)
849 {
850 struct netlink_ext_ack *extack = fl_act->extack;
851 struct nfp_flower_priv *fl_priv = app->priv;
852
853 if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_METER))
854 return -EOPNOTSUPP;
855
856 switch (fl_act->command) {
857 case FLOW_ACT_REPLACE:
858 return nfp_act_install_actions(app, fl_act, extack);
859 case FLOW_ACT_DESTROY:
860 return nfp_act_remove_actions(app, fl_act, extack);
861 case FLOW_ACT_STATS:
862 return nfp_act_stats_actions(app, fl_act, extack);
863 default:
864 return -EOPNOTSUPP;
865 }
866 }
867