1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * net/sched/cls_api.c	Packet classifier API.
4  *
5  * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6  *
7  * Changes:
8  *
9  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
10  */
11 
12 #include <linux/module.h>
13 #include <linux/types.h>
14 #include <linux/kernel.h>
15 #include <linux/string.h>
16 #include <linux/errno.h>
17 #include <linux/err.h>
18 #include <linux/skbuff.h>
19 #include <linux/init.h>
20 #include <linux/kmod.h>
21 #include <linux/slab.h>
22 #include <linux/idr.h>
23 #include <linux/jhash.h>
24 #include <linux/rculist.h>
25 #include <net/net_namespace.h>
26 #include <net/sock.h>
27 #include <net/netlink.h>
28 #include <net/pkt_sched.h>
29 #include <net/pkt_cls.h>
30 #include <net/tc_act/tc_pedit.h>
31 #include <net/tc_act/tc_mirred.h>
32 #include <net/tc_act/tc_vlan.h>
33 #include <net/tc_act/tc_tunnel_key.h>
34 #include <net/tc_act/tc_csum.h>
35 #include <net/tc_act/tc_gact.h>
36 #include <net/tc_act/tc_police.h>
37 #include <net/tc_act/tc_sample.h>
38 #include <net/tc_act/tc_skbedit.h>
39 #include <net/tc_act/tc_ct.h>
40 #include <net/tc_act/tc_mpls.h>
41 #include <net/tc_act/tc_gate.h>
42 #include <net/flow_offload.h>
43 
44 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
45 
46 /* The list of all installed classifier types */
47 static LIST_HEAD(tcf_proto_base);
48 
49 /* Protects list of registered TC modules. It is pure SMP lock. */
50 static DEFINE_RWLOCK(cls_mod_lock);
51 
52 #ifdef CONFIG_NET_CLS_ACT
53 DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc);
54 EXPORT_SYMBOL(tc_skb_ext_tc);
55 
tc_skb_ext_tc_enable(void)56 void tc_skb_ext_tc_enable(void)
57 {
58 	static_branch_inc(&tc_skb_ext_tc);
59 }
60 EXPORT_SYMBOL(tc_skb_ext_tc_enable);
61 
tc_skb_ext_tc_disable(void)62 void tc_skb_ext_tc_disable(void)
63 {
64 	static_branch_dec(&tc_skb_ext_tc);
65 }
66 EXPORT_SYMBOL(tc_skb_ext_tc_disable);
67 #endif
68 
destroy_obj_hashfn(const struct tcf_proto * tp)69 static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
70 {
71 	return jhash_3words(tp->chain->index, tp->prio,
72 			    (__force __u32)tp->protocol, 0);
73 }
74 
tcf_proto_signal_destroying(struct tcf_chain * chain,struct tcf_proto * tp)75 static void tcf_proto_signal_destroying(struct tcf_chain *chain,
76 					struct tcf_proto *tp)
77 {
78 	struct tcf_block *block = chain->block;
79 
80 	mutex_lock(&block->proto_destroy_lock);
81 	hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
82 		     destroy_obj_hashfn(tp));
83 	mutex_unlock(&block->proto_destroy_lock);
84 }
85 
tcf_proto_cmp(const struct tcf_proto * tp1,const struct tcf_proto * tp2)86 static bool tcf_proto_cmp(const struct tcf_proto *tp1,
87 			  const struct tcf_proto *tp2)
88 {
89 	return tp1->chain->index == tp2->chain->index &&
90 	       tp1->prio == tp2->prio &&
91 	       tp1->protocol == tp2->protocol;
92 }
93 
tcf_proto_exists_destroying(struct tcf_chain * chain,struct tcf_proto * tp)94 static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
95 					struct tcf_proto *tp)
96 {
97 	u32 hash = destroy_obj_hashfn(tp);
98 	struct tcf_proto *iter;
99 	bool found = false;
100 
101 	rcu_read_lock();
102 	hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
103 				   destroy_ht_node, hash) {
104 		if (tcf_proto_cmp(tp, iter)) {
105 			found = true;
106 			break;
107 		}
108 	}
109 	rcu_read_unlock();
110 
111 	return found;
112 }
113 
114 static void
tcf_proto_signal_destroyed(struct tcf_chain * chain,struct tcf_proto * tp)115 tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
116 {
117 	struct tcf_block *block = chain->block;
118 
119 	mutex_lock(&block->proto_destroy_lock);
120 	if (hash_hashed(&tp->destroy_ht_node))
121 		hash_del_rcu(&tp->destroy_ht_node);
122 	mutex_unlock(&block->proto_destroy_lock);
123 }
124 
125 /* Find classifier type by string name */
126 
__tcf_proto_lookup_ops(const char * kind)127 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
128 {
129 	const struct tcf_proto_ops *t, *res = NULL;
130 
131 	if (kind) {
132 		read_lock(&cls_mod_lock);
133 		list_for_each_entry(t, &tcf_proto_base, head) {
134 			if (strcmp(kind, t->kind) == 0) {
135 				if (try_module_get(t->owner))
136 					res = t;
137 				break;
138 			}
139 		}
140 		read_unlock(&cls_mod_lock);
141 	}
142 	return res;
143 }
144 
145 static const struct tcf_proto_ops *
tcf_proto_lookup_ops(const char * kind,bool rtnl_held,struct netlink_ext_ack * extack)146 tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
147 		     struct netlink_ext_ack *extack)
148 {
149 	const struct tcf_proto_ops *ops;
150 
151 	ops = __tcf_proto_lookup_ops(kind);
152 	if (ops)
153 		return ops;
154 #ifdef CONFIG_MODULES
155 	if (rtnl_held)
156 		rtnl_unlock();
157 	request_module("cls_%s", kind);
158 	if (rtnl_held)
159 		rtnl_lock();
160 	ops = __tcf_proto_lookup_ops(kind);
161 	/* We dropped the RTNL semaphore in order to perform
162 	 * the module load. So, even if we succeeded in loading
163 	 * the module we have to replay the request. We indicate
164 	 * this using -EAGAIN.
165 	 */
166 	if (ops) {
167 		module_put(ops->owner);
168 		return ERR_PTR(-EAGAIN);
169 	}
170 #endif
171 	NL_SET_ERR_MSG(extack, "TC classifier not found");
172 	return ERR_PTR(-ENOENT);
173 }
174 
175 /* Register(unregister) new classifier type */
176 
register_tcf_proto_ops(struct tcf_proto_ops * ops)177 int register_tcf_proto_ops(struct tcf_proto_ops *ops)
178 {
179 	struct tcf_proto_ops *t;
180 	int rc = -EEXIST;
181 
182 	write_lock(&cls_mod_lock);
183 	list_for_each_entry(t, &tcf_proto_base, head)
184 		if (!strcmp(ops->kind, t->kind))
185 			goto out;
186 
187 	list_add_tail(&ops->head, &tcf_proto_base);
188 	rc = 0;
189 out:
190 	write_unlock(&cls_mod_lock);
191 	return rc;
192 }
193 EXPORT_SYMBOL(register_tcf_proto_ops);
194 
195 static struct workqueue_struct *tc_filter_wq;
196 
unregister_tcf_proto_ops(struct tcf_proto_ops * ops)197 void unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
198 {
199 	struct tcf_proto_ops *t;
200 	int rc = -ENOENT;
201 
202 	/* Wait for outstanding call_rcu()s, if any, from a
203 	 * tcf_proto_ops's destroy() handler.
204 	 */
205 	rcu_barrier();
206 	flush_workqueue(tc_filter_wq);
207 
208 	write_lock(&cls_mod_lock);
209 	list_for_each_entry(t, &tcf_proto_base, head) {
210 		if (t == ops) {
211 			list_del(&t->head);
212 			rc = 0;
213 			break;
214 		}
215 	}
216 	write_unlock(&cls_mod_lock);
217 
218 	WARN(rc, "unregister tc filter kind(%s) failed %d\n", ops->kind, rc);
219 }
220 EXPORT_SYMBOL(unregister_tcf_proto_ops);
221 
tcf_queue_work(struct rcu_work * rwork,work_func_t func)222 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
223 {
224 	INIT_RCU_WORK(rwork, func);
225 	return queue_rcu_work(tc_filter_wq, rwork);
226 }
227 EXPORT_SYMBOL(tcf_queue_work);
228 
229 /* Select new prio value from the range, managed by kernel. */
230 
tcf_auto_prio(struct tcf_proto * tp)231 static inline u32 tcf_auto_prio(struct tcf_proto *tp)
232 {
233 	u32 first = TC_H_MAKE(0xC0000000U, 0U);
234 
235 	if (tp)
236 		first = tp->prio - 1;
237 
238 	return TC_H_MAJ(first);
239 }
240 
tcf_proto_check_kind(struct nlattr * kind,char * name)241 static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
242 {
243 	if (kind)
244 		return nla_strscpy(name, kind, IFNAMSIZ) < 0;
245 	memset(name, 0, IFNAMSIZ);
246 	return false;
247 }
248 
tcf_proto_is_unlocked(const char * kind)249 static bool tcf_proto_is_unlocked(const char *kind)
250 {
251 	const struct tcf_proto_ops *ops;
252 	bool ret;
253 
254 	if (strlen(kind) == 0)
255 		return false;
256 
257 	ops = tcf_proto_lookup_ops(kind, false, NULL);
258 	/* On error return false to take rtnl lock. Proto lookup/create
259 	 * functions will perform lookup again and properly handle errors.
260 	 */
261 	if (IS_ERR(ops))
262 		return false;
263 
264 	ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
265 	module_put(ops->owner);
266 	return ret;
267 }
268 
tcf_proto_create(const char * kind,u32 protocol,u32 prio,struct tcf_chain * chain,bool rtnl_held,struct netlink_ext_ack * extack)269 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
270 					  u32 prio, struct tcf_chain *chain,
271 					  bool rtnl_held,
272 					  struct netlink_ext_ack *extack)
273 {
274 	struct tcf_proto *tp;
275 	int err;
276 
277 	tp = kzalloc(sizeof(*tp), GFP_KERNEL);
278 	if (!tp)
279 		return ERR_PTR(-ENOBUFS);
280 
281 	tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
282 	if (IS_ERR(tp->ops)) {
283 		err = PTR_ERR(tp->ops);
284 		goto errout;
285 	}
286 	tp->classify = tp->ops->classify;
287 	tp->protocol = protocol;
288 	tp->prio = prio;
289 	tp->chain = chain;
290 	spin_lock_init(&tp->lock);
291 	refcount_set(&tp->refcnt, 1);
292 
293 	err = tp->ops->init(tp);
294 	if (err) {
295 		module_put(tp->ops->owner);
296 		goto errout;
297 	}
298 	return tp;
299 
300 errout:
301 	kfree(tp);
302 	return ERR_PTR(err);
303 }
304 
tcf_proto_get(struct tcf_proto * tp)305 static void tcf_proto_get(struct tcf_proto *tp)
306 {
307 	refcount_inc(&tp->refcnt);
308 }
309 
310 static void tcf_chain_put(struct tcf_chain *chain);
311 
tcf_proto_destroy(struct tcf_proto * tp,bool rtnl_held,bool sig_destroy,struct netlink_ext_ack * extack)312 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
313 			      bool sig_destroy, struct netlink_ext_ack *extack)
314 {
315 	tp->ops->destroy(tp, rtnl_held, extack);
316 	if (sig_destroy)
317 		tcf_proto_signal_destroyed(tp->chain, tp);
318 	tcf_chain_put(tp->chain);
319 	module_put(tp->ops->owner);
320 	kfree_rcu(tp, rcu);
321 }
322 
tcf_proto_put(struct tcf_proto * tp,bool rtnl_held,struct netlink_ext_ack * extack)323 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
324 			  struct netlink_ext_ack *extack)
325 {
326 	if (refcount_dec_and_test(&tp->refcnt))
327 		tcf_proto_destroy(tp, rtnl_held, true, extack);
328 }
329 
tcf_proto_check_delete(struct tcf_proto * tp)330 static bool tcf_proto_check_delete(struct tcf_proto *tp)
331 {
332 	if (tp->ops->delete_empty)
333 		return tp->ops->delete_empty(tp);
334 
335 	tp->deleting = true;
336 	return tp->deleting;
337 }
338 
tcf_proto_mark_delete(struct tcf_proto * tp)339 static void tcf_proto_mark_delete(struct tcf_proto *tp)
340 {
341 	spin_lock(&tp->lock);
342 	tp->deleting = true;
343 	spin_unlock(&tp->lock);
344 }
345 
tcf_proto_is_deleting(struct tcf_proto * tp)346 static bool tcf_proto_is_deleting(struct tcf_proto *tp)
347 {
348 	bool deleting;
349 
350 	spin_lock(&tp->lock);
351 	deleting = tp->deleting;
352 	spin_unlock(&tp->lock);
353 
354 	return deleting;
355 }
356 
357 #define ASSERT_BLOCK_LOCKED(block)					\
358 	lockdep_assert_held(&(block)->lock)
359 
360 struct tcf_filter_chain_list_item {
361 	struct list_head list;
362 	tcf_chain_head_change_t *chain_head_change;
363 	void *chain_head_change_priv;
364 };
365 
tcf_chain_create(struct tcf_block * block,u32 chain_index)366 static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
367 					  u32 chain_index)
368 {
369 	struct tcf_chain *chain;
370 
371 	ASSERT_BLOCK_LOCKED(block);
372 
373 	chain = kzalloc(sizeof(*chain), GFP_KERNEL);
374 	if (!chain)
375 		return NULL;
376 	list_add_tail_rcu(&chain->list, &block->chain_list);
377 	mutex_init(&chain->filter_chain_lock);
378 	chain->block = block;
379 	chain->index = chain_index;
380 	chain->refcnt = 1;
381 	if (!chain->index)
382 		block->chain0.chain = chain;
383 	return chain;
384 }
385 
tcf_chain_head_change_item(struct tcf_filter_chain_list_item * item,struct tcf_proto * tp_head)386 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
387 				       struct tcf_proto *tp_head)
388 {
389 	if (item->chain_head_change)
390 		item->chain_head_change(tp_head, item->chain_head_change_priv);
391 }
392 
tcf_chain0_head_change(struct tcf_chain * chain,struct tcf_proto * tp_head)393 static void tcf_chain0_head_change(struct tcf_chain *chain,
394 				   struct tcf_proto *tp_head)
395 {
396 	struct tcf_filter_chain_list_item *item;
397 	struct tcf_block *block = chain->block;
398 
399 	if (chain->index)
400 		return;
401 
402 	mutex_lock(&block->lock);
403 	list_for_each_entry(item, &block->chain0.filter_chain_list, list)
404 		tcf_chain_head_change_item(item, tp_head);
405 	mutex_unlock(&block->lock);
406 }
407 
408 /* Returns true if block can be safely freed. */
409 
tcf_chain_detach(struct tcf_chain * chain)410 static bool tcf_chain_detach(struct tcf_chain *chain)
411 {
412 	struct tcf_block *block = chain->block;
413 
414 	ASSERT_BLOCK_LOCKED(block);
415 
416 	list_del_rcu(&chain->list);
417 	if (!chain->index)
418 		block->chain0.chain = NULL;
419 
420 	if (list_empty(&block->chain_list) &&
421 	    refcount_read(&block->refcnt) == 0)
422 		return true;
423 
424 	return false;
425 }
426 
tcf_block_destroy(struct tcf_block * block)427 static void tcf_block_destroy(struct tcf_block *block)
428 {
429 	mutex_destroy(&block->lock);
430 	mutex_destroy(&block->proto_destroy_lock);
431 	kfree_rcu(block, rcu);
432 }
433 
tcf_chain_destroy(struct tcf_chain * chain,bool free_block)434 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
435 {
436 	struct tcf_block *block = chain->block;
437 
438 	mutex_destroy(&chain->filter_chain_lock);
439 	kfree_rcu(chain, rcu);
440 	if (free_block)
441 		tcf_block_destroy(block);
442 }
443 
tcf_chain_hold(struct tcf_chain * chain)444 static void tcf_chain_hold(struct tcf_chain *chain)
445 {
446 	ASSERT_BLOCK_LOCKED(chain->block);
447 
448 	++chain->refcnt;
449 }
450 
tcf_chain_held_by_acts_only(struct tcf_chain * chain)451 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
452 {
453 	ASSERT_BLOCK_LOCKED(chain->block);
454 
455 	/* In case all the references are action references, this
456 	 * chain should not be shown to the user.
457 	 */
458 	return chain->refcnt == chain->action_refcnt;
459 }
460 
tcf_chain_lookup(struct tcf_block * block,u32 chain_index)461 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
462 					  u32 chain_index)
463 {
464 	struct tcf_chain *chain;
465 
466 	ASSERT_BLOCK_LOCKED(block);
467 
468 	list_for_each_entry(chain, &block->chain_list, list) {
469 		if (chain->index == chain_index)
470 			return chain;
471 	}
472 	return NULL;
473 }
474 
475 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
tcf_chain_lookup_rcu(const struct tcf_block * block,u32 chain_index)476 static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
477 					      u32 chain_index)
478 {
479 	struct tcf_chain *chain;
480 
481 	list_for_each_entry_rcu(chain, &block->chain_list, list) {
482 		if (chain->index == chain_index)
483 			return chain;
484 	}
485 	return NULL;
486 }
487 #endif
488 
489 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
490 			   u32 seq, u16 flags, int event, bool unicast);
491 
__tcf_chain_get(struct tcf_block * block,u32 chain_index,bool create,bool by_act)492 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
493 					 u32 chain_index, bool create,
494 					 bool by_act)
495 {
496 	struct tcf_chain *chain = NULL;
497 	bool is_first_reference;
498 
499 	mutex_lock(&block->lock);
500 	chain = tcf_chain_lookup(block, chain_index);
501 	if (chain) {
502 		tcf_chain_hold(chain);
503 	} else {
504 		if (!create)
505 			goto errout;
506 		chain = tcf_chain_create(block, chain_index);
507 		if (!chain)
508 			goto errout;
509 	}
510 
511 	if (by_act)
512 		++chain->action_refcnt;
513 	is_first_reference = chain->refcnt - chain->action_refcnt == 1;
514 	mutex_unlock(&block->lock);
515 
516 	/* Send notification only in case we got the first
517 	 * non-action reference. Until then, the chain acts only as
518 	 * a placeholder for actions pointing to it and user ought
519 	 * not know about them.
520 	 */
521 	if (is_first_reference && !by_act)
522 		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
523 				RTM_NEWCHAIN, false);
524 
525 	return chain;
526 
527 errout:
528 	mutex_unlock(&block->lock);
529 	return chain;
530 }
531 
tcf_chain_get(struct tcf_block * block,u32 chain_index,bool create)532 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
533 				       bool create)
534 {
535 	return __tcf_chain_get(block, chain_index, create, false);
536 }
537 
tcf_chain_get_by_act(struct tcf_block * block,u32 chain_index)538 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
539 {
540 	return __tcf_chain_get(block, chain_index, true, true);
541 }
542 EXPORT_SYMBOL(tcf_chain_get_by_act);
543 
544 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
545 			       void *tmplt_priv);
546 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
547 				  void *tmplt_priv, u32 chain_index,
548 				  struct tcf_block *block, struct sk_buff *oskb,
549 				  u32 seq, u16 flags, bool unicast);
550 
__tcf_chain_put(struct tcf_chain * chain,bool by_act,bool explicitly_created)551 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
552 			    bool explicitly_created)
553 {
554 	struct tcf_block *block = chain->block;
555 	const struct tcf_proto_ops *tmplt_ops;
556 	bool free_block = false;
557 	unsigned int refcnt;
558 	void *tmplt_priv;
559 
560 	mutex_lock(&block->lock);
561 	if (explicitly_created) {
562 		if (!chain->explicitly_created) {
563 			mutex_unlock(&block->lock);
564 			return;
565 		}
566 		chain->explicitly_created = false;
567 	}
568 
569 	if (by_act)
570 		chain->action_refcnt--;
571 
572 	/* tc_chain_notify_delete can't be called while holding block lock.
573 	 * However, when block is unlocked chain can be changed concurrently, so
574 	 * save these to temporary variables.
575 	 */
576 	refcnt = --chain->refcnt;
577 	tmplt_ops = chain->tmplt_ops;
578 	tmplt_priv = chain->tmplt_priv;
579 
580 	/* The last dropped non-action reference will trigger notification. */
581 	if (refcnt - chain->action_refcnt == 0 && !by_act) {
582 		tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index,
583 				       block, NULL, 0, 0, false);
584 		/* Last reference to chain, no need to lock. */
585 		chain->flushing = false;
586 	}
587 
588 	if (refcnt == 0)
589 		free_block = tcf_chain_detach(chain);
590 	mutex_unlock(&block->lock);
591 
592 	if (refcnt == 0) {
593 		tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
594 		tcf_chain_destroy(chain, free_block);
595 	}
596 }
597 
tcf_chain_put(struct tcf_chain * chain)598 static void tcf_chain_put(struct tcf_chain *chain)
599 {
600 	__tcf_chain_put(chain, false, false);
601 }
602 
tcf_chain_put_by_act(struct tcf_chain * chain)603 void tcf_chain_put_by_act(struct tcf_chain *chain)
604 {
605 	__tcf_chain_put(chain, true, false);
606 }
607 EXPORT_SYMBOL(tcf_chain_put_by_act);
608 
tcf_chain_put_explicitly_created(struct tcf_chain * chain)609 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
610 {
611 	__tcf_chain_put(chain, false, true);
612 }
613 
tcf_chain_flush(struct tcf_chain * chain,bool rtnl_held)614 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
615 {
616 	struct tcf_proto *tp, *tp_next;
617 
618 	mutex_lock(&chain->filter_chain_lock);
619 	tp = tcf_chain_dereference(chain->filter_chain, chain);
620 	while (tp) {
621 		tp_next = rcu_dereference_protected(tp->next, 1);
622 		tcf_proto_signal_destroying(chain, tp);
623 		tp = tp_next;
624 	}
625 	tp = tcf_chain_dereference(chain->filter_chain, chain);
626 	RCU_INIT_POINTER(chain->filter_chain, NULL);
627 	tcf_chain0_head_change(chain, NULL);
628 	chain->flushing = true;
629 	mutex_unlock(&chain->filter_chain_lock);
630 
631 	while (tp) {
632 		tp_next = rcu_dereference_protected(tp->next, 1);
633 		tcf_proto_put(tp, rtnl_held, NULL);
634 		tp = tp_next;
635 	}
636 }
637 
638 static int tcf_block_setup(struct tcf_block *block,
639 			   struct flow_block_offload *bo);
640 
tcf_block_offload_init(struct flow_block_offload * bo,struct net_device * dev,struct Qdisc * sch,enum flow_block_command command,enum flow_block_binder_type binder_type,struct flow_block * flow_block,bool shared,struct netlink_ext_ack * extack)641 static void tcf_block_offload_init(struct flow_block_offload *bo,
642 				   struct net_device *dev, struct Qdisc *sch,
643 				   enum flow_block_command command,
644 				   enum flow_block_binder_type binder_type,
645 				   struct flow_block *flow_block,
646 				   bool shared, struct netlink_ext_ack *extack)
647 {
648 	bo->net = dev_net(dev);
649 	bo->command = command;
650 	bo->binder_type = binder_type;
651 	bo->block = flow_block;
652 	bo->block_shared = shared;
653 	bo->extack = extack;
654 	bo->sch = sch;
655 	bo->cb_list_head = &flow_block->cb_list;
656 	INIT_LIST_HEAD(&bo->cb_list);
657 }
658 
659 static void tcf_block_unbind(struct tcf_block *block,
660 			     struct flow_block_offload *bo);
661 
tc_block_indr_cleanup(struct flow_block_cb * block_cb)662 static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
663 {
664 	struct tcf_block *block = block_cb->indr.data;
665 	struct net_device *dev = block_cb->indr.dev;
666 	struct Qdisc *sch = block_cb->indr.sch;
667 	struct netlink_ext_ack extack = {};
668 	struct flow_block_offload bo = {};
669 
670 	tcf_block_offload_init(&bo, dev, sch, FLOW_BLOCK_UNBIND,
671 			       block_cb->indr.binder_type,
672 			       &block->flow_block, tcf_block_shared(block),
673 			       &extack);
674 	rtnl_lock();
675 	down_write(&block->cb_lock);
676 	list_del(&block_cb->driver_list);
677 	list_move(&block_cb->list, &bo.cb_list);
678 	tcf_block_unbind(block, &bo);
679 	up_write(&block->cb_lock);
680 	rtnl_unlock();
681 }
682 
tcf_block_offload_in_use(struct tcf_block * block)683 static bool tcf_block_offload_in_use(struct tcf_block *block)
684 {
685 	return atomic_read(&block->offloadcnt);
686 }
687 
tcf_block_offload_cmd(struct tcf_block * block,struct net_device * dev,struct Qdisc * sch,struct tcf_block_ext_info * ei,enum flow_block_command command,struct netlink_ext_ack * extack)688 static int tcf_block_offload_cmd(struct tcf_block *block,
689 				 struct net_device *dev, struct Qdisc *sch,
690 				 struct tcf_block_ext_info *ei,
691 				 enum flow_block_command command,
692 				 struct netlink_ext_ack *extack)
693 {
694 	struct flow_block_offload bo = {};
695 
696 	tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type,
697 			       &block->flow_block, tcf_block_shared(block),
698 			       extack);
699 
700 	if (dev->netdev_ops->ndo_setup_tc) {
701 		int err;
702 
703 		err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
704 		if (err < 0) {
705 			if (err != -EOPNOTSUPP)
706 				NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed");
707 			return err;
708 		}
709 
710 		return tcf_block_setup(block, &bo);
711 	}
712 
713 	flow_indr_dev_setup_offload(dev, sch, TC_SETUP_BLOCK, block, &bo,
714 				    tc_block_indr_cleanup);
715 	tcf_block_setup(block, &bo);
716 
717 	return -EOPNOTSUPP;
718 }
719 
tcf_block_offload_bind(struct tcf_block * block,struct Qdisc * q,struct tcf_block_ext_info * ei,struct netlink_ext_ack * extack)720 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
721 				  struct tcf_block_ext_info *ei,
722 				  struct netlink_ext_ack *extack)
723 {
724 	struct net_device *dev = q->dev_queue->dev;
725 	int err;
726 
727 	down_write(&block->cb_lock);
728 
729 	/* If tc offload feature is disabled and the block we try to bind
730 	 * to already has some offloaded filters, forbid to bind.
731 	 */
732 	if (dev->netdev_ops->ndo_setup_tc &&
733 	    !tc_can_offload(dev) &&
734 	    tcf_block_offload_in_use(block)) {
735 		NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
736 		err = -EOPNOTSUPP;
737 		goto err_unlock;
738 	}
739 
740 	err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_BIND, extack);
741 	if (err == -EOPNOTSUPP)
742 		goto no_offload_dev_inc;
743 	if (err)
744 		goto err_unlock;
745 
746 	up_write(&block->cb_lock);
747 	return 0;
748 
749 no_offload_dev_inc:
750 	if (tcf_block_offload_in_use(block))
751 		goto err_unlock;
752 
753 	err = 0;
754 	block->nooffloaddevcnt++;
755 err_unlock:
756 	up_write(&block->cb_lock);
757 	return err;
758 }
759 
tcf_block_offload_unbind(struct tcf_block * block,struct Qdisc * q,struct tcf_block_ext_info * ei)760 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
761 				     struct tcf_block_ext_info *ei)
762 {
763 	struct net_device *dev = q->dev_queue->dev;
764 	int err;
765 
766 	down_write(&block->cb_lock);
767 	err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_UNBIND, NULL);
768 	if (err == -EOPNOTSUPP)
769 		goto no_offload_dev_dec;
770 	up_write(&block->cb_lock);
771 	return;
772 
773 no_offload_dev_dec:
774 	WARN_ON(block->nooffloaddevcnt-- == 0);
775 	up_write(&block->cb_lock);
776 }
777 
778 static int
tcf_chain0_head_change_cb_add(struct tcf_block * block,struct tcf_block_ext_info * ei,struct netlink_ext_ack * extack)779 tcf_chain0_head_change_cb_add(struct tcf_block *block,
780 			      struct tcf_block_ext_info *ei,
781 			      struct netlink_ext_ack *extack)
782 {
783 	struct tcf_filter_chain_list_item *item;
784 	struct tcf_chain *chain0;
785 
786 	item = kmalloc(sizeof(*item), GFP_KERNEL);
787 	if (!item) {
788 		NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
789 		return -ENOMEM;
790 	}
791 	item->chain_head_change = ei->chain_head_change;
792 	item->chain_head_change_priv = ei->chain_head_change_priv;
793 
794 	mutex_lock(&block->lock);
795 	chain0 = block->chain0.chain;
796 	if (chain0)
797 		tcf_chain_hold(chain0);
798 	else
799 		list_add(&item->list, &block->chain0.filter_chain_list);
800 	mutex_unlock(&block->lock);
801 
802 	if (chain0) {
803 		struct tcf_proto *tp_head;
804 
805 		mutex_lock(&chain0->filter_chain_lock);
806 
807 		tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
808 		if (tp_head)
809 			tcf_chain_head_change_item(item, tp_head);
810 
811 		mutex_lock(&block->lock);
812 		list_add(&item->list, &block->chain0.filter_chain_list);
813 		mutex_unlock(&block->lock);
814 
815 		mutex_unlock(&chain0->filter_chain_lock);
816 		tcf_chain_put(chain0);
817 	}
818 
819 	return 0;
820 }
821 
822 static void
tcf_chain0_head_change_cb_del(struct tcf_block * block,struct tcf_block_ext_info * ei)823 tcf_chain0_head_change_cb_del(struct tcf_block *block,
824 			      struct tcf_block_ext_info *ei)
825 {
826 	struct tcf_filter_chain_list_item *item;
827 
828 	mutex_lock(&block->lock);
829 	list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
830 		if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
831 		    (item->chain_head_change == ei->chain_head_change &&
832 		     item->chain_head_change_priv == ei->chain_head_change_priv)) {
833 			if (block->chain0.chain)
834 				tcf_chain_head_change_item(item, NULL);
835 			list_del(&item->list);
836 			mutex_unlock(&block->lock);
837 
838 			kfree(item);
839 			return;
840 		}
841 	}
842 	mutex_unlock(&block->lock);
843 	WARN_ON(1);
844 }
845 
846 struct tcf_net {
847 	spinlock_t idr_lock; /* Protects idr */
848 	struct idr idr;
849 };
850 
851 static unsigned int tcf_net_id;
852 
tcf_block_insert(struct tcf_block * block,struct net * net,struct netlink_ext_ack * extack)853 static int tcf_block_insert(struct tcf_block *block, struct net *net,
854 			    struct netlink_ext_ack *extack)
855 {
856 	struct tcf_net *tn = net_generic(net, tcf_net_id);
857 	int err;
858 
859 	idr_preload(GFP_KERNEL);
860 	spin_lock(&tn->idr_lock);
861 	err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
862 			    GFP_NOWAIT);
863 	spin_unlock(&tn->idr_lock);
864 	idr_preload_end();
865 
866 	return err;
867 }
868 
tcf_block_remove(struct tcf_block * block,struct net * net)869 static void tcf_block_remove(struct tcf_block *block, struct net *net)
870 {
871 	struct tcf_net *tn = net_generic(net, tcf_net_id);
872 
873 	spin_lock(&tn->idr_lock);
874 	idr_remove(&tn->idr, block->index);
875 	spin_unlock(&tn->idr_lock);
876 }
877 
tcf_block_create(struct net * net,struct Qdisc * q,u32 block_index,struct netlink_ext_ack * extack)878 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
879 					  u32 block_index,
880 					  struct netlink_ext_ack *extack)
881 {
882 	struct tcf_block *block;
883 
884 	block = kzalloc(sizeof(*block), GFP_KERNEL);
885 	if (!block) {
886 		NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
887 		return ERR_PTR(-ENOMEM);
888 	}
889 	mutex_init(&block->lock);
890 	mutex_init(&block->proto_destroy_lock);
891 	init_rwsem(&block->cb_lock);
892 	flow_block_init(&block->flow_block);
893 	INIT_LIST_HEAD(&block->chain_list);
894 	INIT_LIST_HEAD(&block->owner_list);
895 	INIT_LIST_HEAD(&block->chain0.filter_chain_list);
896 
897 	refcount_set(&block->refcnt, 1);
898 	block->net = net;
899 	block->index = block_index;
900 
901 	/* Don't store q pointer for blocks which are shared */
902 	if (!tcf_block_shared(block))
903 		block->q = q;
904 	return block;
905 }
906 
tcf_block_lookup(struct net * net,u32 block_index)907 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
908 {
909 	struct tcf_net *tn = net_generic(net, tcf_net_id);
910 
911 	return idr_find(&tn->idr, block_index);
912 }
913 
tcf_block_refcnt_get(struct net * net,u32 block_index)914 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
915 {
916 	struct tcf_block *block;
917 
918 	rcu_read_lock();
919 	block = tcf_block_lookup(net, block_index);
920 	if (block && !refcount_inc_not_zero(&block->refcnt))
921 		block = NULL;
922 	rcu_read_unlock();
923 
924 	return block;
925 }
926 
927 static struct tcf_chain *
__tcf_get_next_chain(struct tcf_block * block,struct tcf_chain * chain)928 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
929 {
930 	mutex_lock(&block->lock);
931 	if (chain)
932 		chain = list_is_last(&chain->list, &block->chain_list) ?
933 			NULL : list_next_entry(chain, list);
934 	else
935 		chain = list_first_entry_or_null(&block->chain_list,
936 						 struct tcf_chain, list);
937 
938 	/* skip all action-only chains */
939 	while (chain && tcf_chain_held_by_acts_only(chain))
940 		chain = list_is_last(&chain->list, &block->chain_list) ?
941 			NULL : list_next_entry(chain, list);
942 
943 	if (chain)
944 		tcf_chain_hold(chain);
945 	mutex_unlock(&block->lock);
946 
947 	return chain;
948 }
949 
950 /* Function to be used by all clients that want to iterate over all chains on
951  * block. It properly obtains block->lock and takes reference to chain before
952  * returning it. Users of this function must be tolerant to concurrent chain
953  * insertion/deletion or ensure that no concurrent chain modification is
954  * possible. Note that all netlink dump callbacks cannot guarantee to provide
955  * consistent dump because rtnl lock is released each time skb is filled with
956  * data and sent to user-space.
957  */
958 
959 struct tcf_chain *
tcf_get_next_chain(struct tcf_block * block,struct tcf_chain * chain)960 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
961 {
962 	struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
963 
964 	if (chain)
965 		tcf_chain_put(chain);
966 
967 	return chain_next;
968 }
969 EXPORT_SYMBOL(tcf_get_next_chain);
970 
971 static struct tcf_proto *
__tcf_get_next_proto(struct tcf_chain * chain,struct tcf_proto * tp)972 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
973 {
974 	u32 prio = 0;
975 
976 	ASSERT_RTNL();
977 	mutex_lock(&chain->filter_chain_lock);
978 
979 	if (!tp) {
980 		tp = tcf_chain_dereference(chain->filter_chain, chain);
981 	} else if (tcf_proto_is_deleting(tp)) {
982 		/* 'deleting' flag is set and chain->filter_chain_lock was
983 		 * unlocked, which means next pointer could be invalid. Restart
984 		 * search.
985 		 */
986 		prio = tp->prio + 1;
987 		tp = tcf_chain_dereference(chain->filter_chain, chain);
988 
989 		for (; tp; tp = tcf_chain_dereference(tp->next, chain))
990 			if (!tp->deleting && tp->prio >= prio)
991 				break;
992 	} else {
993 		tp = tcf_chain_dereference(tp->next, chain);
994 	}
995 
996 	if (tp)
997 		tcf_proto_get(tp);
998 
999 	mutex_unlock(&chain->filter_chain_lock);
1000 
1001 	return tp;
1002 }
1003 
1004 /* Function to be used by all clients that want to iterate over all tp's on
1005  * chain. Users of this function must be tolerant to concurrent tp
1006  * insertion/deletion or ensure that no concurrent chain modification is
1007  * possible. Note that all netlink dump callbacks cannot guarantee to provide
1008  * consistent dump because rtnl lock is released each time skb is filled with
1009  * data and sent to user-space.
1010  */
1011 
1012 struct tcf_proto *
tcf_get_next_proto(struct tcf_chain * chain,struct tcf_proto * tp)1013 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1014 {
1015 	struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1016 
1017 	if (tp)
1018 		tcf_proto_put(tp, true, NULL);
1019 
1020 	return tp_next;
1021 }
1022 EXPORT_SYMBOL(tcf_get_next_proto);
1023 
tcf_block_flush_all_chains(struct tcf_block * block,bool rtnl_held)1024 static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
1025 {
1026 	struct tcf_chain *chain;
1027 
1028 	/* Last reference to block. At this point chains cannot be added or
1029 	 * removed concurrently.
1030 	 */
1031 	for (chain = tcf_get_next_chain(block, NULL);
1032 	     chain;
1033 	     chain = tcf_get_next_chain(block, chain)) {
1034 		tcf_chain_put_explicitly_created(chain);
1035 		tcf_chain_flush(chain, rtnl_held);
1036 	}
1037 }
1038 
1039 /* Lookup Qdisc and increments its reference counter.
1040  * Set parent, if necessary.
1041  */
1042 
__tcf_qdisc_find(struct net * net,struct Qdisc ** q,u32 * parent,int ifindex,bool rtnl_held,struct netlink_ext_ack * extack)1043 static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1044 			    u32 *parent, int ifindex, bool rtnl_held,
1045 			    struct netlink_ext_ack *extack)
1046 {
1047 	const struct Qdisc_class_ops *cops;
1048 	struct net_device *dev;
1049 	int err = 0;
1050 
1051 	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1052 		return 0;
1053 
1054 	rcu_read_lock();
1055 
1056 	/* Find link */
1057 	dev = dev_get_by_index_rcu(net, ifindex);
1058 	if (!dev) {
1059 		rcu_read_unlock();
1060 		return -ENODEV;
1061 	}
1062 
1063 	/* Find qdisc */
1064 	if (!*parent) {
1065 		*q = rcu_dereference(dev->qdisc);
1066 		*parent = (*q)->handle;
1067 	} else {
1068 		*q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1069 		if (!*q) {
1070 			NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1071 			err = -EINVAL;
1072 			goto errout_rcu;
1073 		}
1074 	}
1075 
1076 	*q = qdisc_refcount_inc_nz(*q);
1077 	if (!*q) {
1078 		NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1079 		err = -EINVAL;
1080 		goto errout_rcu;
1081 	}
1082 
1083 	/* Is it classful? */
1084 	cops = (*q)->ops->cl_ops;
1085 	if (!cops) {
1086 		NL_SET_ERR_MSG(extack, "Qdisc not classful");
1087 		err = -EINVAL;
1088 		goto errout_qdisc;
1089 	}
1090 
1091 	if (!cops->tcf_block) {
1092 		NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1093 		err = -EOPNOTSUPP;
1094 		goto errout_qdisc;
1095 	}
1096 
1097 errout_rcu:
1098 	/* At this point we know that qdisc is not noop_qdisc,
1099 	 * which means that qdisc holds a reference to net_device
1100 	 * and we hold a reference to qdisc, so it is safe to release
1101 	 * rcu read lock.
1102 	 */
1103 	rcu_read_unlock();
1104 	return err;
1105 
1106 errout_qdisc:
1107 	rcu_read_unlock();
1108 
1109 	if (rtnl_held)
1110 		qdisc_put(*q);
1111 	else
1112 		qdisc_put_unlocked(*q);
1113 	*q = NULL;
1114 
1115 	return err;
1116 }
1117 
__tcf_qdisc_cl_find(struct Qdisc * q,u32 parent,unsigned long * cl,int ifindex,struct netlink_ext_ack * extack)1118 static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1119 			       int ifindex, struct netlink_ext_ack *extack)
1120 {
1121 	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1122 		return 0;
1123 
1124 	/* Do we search for filter, attached to class? */
1125 	if (TC_H_MIN(parent)) {
1126 		const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1127 
1128 		*cl = cops->find(q, parent);
1129 		if (*cl == 0) {
1130 			NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1131 			return -ENOENT;
1132 		}
1133 	}
1134 
1135 	return 0;
1136 }
1137 
__tcf_block_find(struct net * net,struct Qdisc * q,unsigned long cl,int ifindex,u32 block_index,struct netlink_ext_ack * extack)1138 static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1139 					  unsigned long cl, int ifindex,
1140 					  u32 block_index,
1141 					  struct netlink_ext_ack *extack)
1142 {
1143 	struct tcf_block *block;
1144 
1145 	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1146 		block = tcf_block_refcnt_get(net, block_index);
1147 		if (!block) {
1148 			NL_SET_ERR_MSG(extack, "Block of given index was not found");
1149 			return ERR_PTR(-EINVAL);
1150 		}
1151 	} else {
1152 		const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1153 
1154 		block = cops->tcf_block(q, cl, extack);
1155 		if (!block)
1156 			return ERR_PTR(-EINVAL);
1157 
1158 		if (tcf_block_shared(block)) {
1159 			NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1160 			return ERR_PTR(-EOPNOTSUPP);
1161 		}
1162 
1163 		/* Always take reference to block in order to support execution
1164 		 * of rules update path of cls API without rtnl lock. Caller
1165 		 * must release block when it is finished using it. 'if' block
1166 		 * of this conditional obtain reference to block by calling
1167 		 * tcf_block_refcnt_get().
1168 		 */
1169 		refcount_inc(&block->refcnt);
1170 	}
1171 
1172 	return block;
1173 }
1174 
__tcf_block_put(struct tcf_block * block,struct Qdisc * q,struct tcf_block_ext_info * ei,bool rtnl_held)1175 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
1176 			    struct tcf_block_ext_info *ei, bool rtnl_held)
1177 {
1178 	if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
1179 		/* Flushing/putting all chains will cause the block to be
1180 		 * deallocated when last chain is freed. However, if chain_list
1181 		 * is empty, block has to be manually deallocated. After block
1182 		 * reference counter reached 0, it is no longer possible to
1183 		 * increment it or add new chains to block.
1184 		 */
1185 		bool free_block = list_empty(&block->chain_list);
1186 
1187 		mutex_unlock(&block->lock);
1188 		if (tcf_block_shared(block))
1189 			tcf_block_remove(block, block->net);
1190 
1191 		if (q)
1192 			tcf_block_offload_unbind(block, q, ei);
1193 
1194 		if (free_block)
1195 			tcf_block_destroy(block);
1196 		else
1197 			tcf_block_flush_all_chains(block, rtnl_held);
1198 	} else if (q) {
1199 		tcf_block_offload_unbind(block, q, ei);
1200 	}
1201 }
1202 
tcf_block_refcnt_put(struct tcf_block * block,bool rtnl_held)1203 static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
1204 {
1205 	__tcf_block_put(block, NULL, NULL, rtnl_held);
1206 }
1207 
1208 /* Find tcf block.
1209  * Set q, parent, cl when appropriate.
1210  */
1211 
tcf_block_find(struct net * net,struct Qdisc ** q,u32 * parent,unsigned long * cl,int ifindex,u32 block_index,struct netlink_ext_ack * extack)1212 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1213 					u32 *parent, unsigned long *cl,
1214 					int ifindex, u32 block_index,
1215 					struct netlink_ext_ack *extack)
1216 {
1217 	struct tcf_block *block;
1218 	int err = 0;
1219 
1220 	ASSERT_RTNL();
1221 
1222 	err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
1223 	if (err)
1224 		goto errout;
1225 
1226 	err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
1227 	if (err)
1228 		goto errout_qdisc;
1229 
1230 	block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
1231 	if (IS_ERR(block)) {
1232 		err = PTR_ERR(block);
1233 		goto errout_qdisc;
1234 	}
1235 
1236 	return block;
1237 
1238 errout_qdisc:
1239 	if (*q)
1240 		qdisc_put(*q);
1241 errout:
1242 	*q = NULL;
1243 	return ERR_PTR(err);
1244 }
1245 
tcf_block_release(struct Qdisc * q,struct tcf_block * block,bool rtnl_held)1246 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1247 			      bool rtnl_held)
1248 {
1249 	if (!IS_ERR_OR_NULL(block))
1250 		tcf_block_refcnt_put(block, rtnl_held);
1251 
1252 	if (q) {
1253 		if (rtnl_held)
1254 			qdisc_put(q);
1255 		else
1256 			qdisc_put_unlocked(q);
1257 	}
1258 }
1259 
1260 struct tcf_block_owner_item {
1261 	struct list_head list;
1262 	struct Qdisc *q;
1263 	enum flow_block_binder_type binder_type;
1264 };
1265 
1266 static void
tcf_block_owner_netif_keep_dst(struct tcf_block * block,struct Qdisc * q,enum flow_block_binder_type binder_type)1267 tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1268 			       struct Qdisc *q,
1269 			       enum flow_block_binder_type binder_type)
1270 {
1271 	if (block->keep_dst &&
1272 	    binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1273 	    binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1274 		netif_keep_dst(qdisc_dev(q));
1275 }
1276 
tcf_block_netif_keep_dst(struct tcf_block * block)1277 void tcf_block_netif_keep_dst(struct tcf_block *block)
1278 {
1279 	struct tcf_block_owner_item *item;
1280 
1281 	block->keep_dst = true;
1282 	list_for_each_entry(item, &block->owner_list, list)
1283 		tcf_block_owner_netif_keep_dst(block, item->q,
1284 					       item->binder_type);
1285 }
1286 EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1287 
tcf_block_owner_add(struct tcf_block * block,struct Qdisc * q,enum flow_block_binder_type binder_type)1288 static int tcf_block_owner_add(struct tcf_block *block,
1289 			       struct Qdisc *q,
1290 			       enum flow_block_binder_type binder_type)
1291 {
1292 	struct tcf_block_owner_item *item;
1293 
1294 	item = kmalloc(sizeof(*item), GFP_KERNEL);
1295 	if (!item)
1296 		return -ENOMEM;
1297 	item->q = q;
1298 	item->binder_type = binder_type;
1299 	list_add(&item->list, &block->owner_list);
1300 	return 0;
1301 }
1302 
tcf_block_owner_del(struct tcf_block * block,struct Qdisc * q,enum flow_block_binder_type binder_type)1303 static void tcf_block_owner_del(struct tcf_block *block,
1304 				struct Qdisc *q,
1305 				enum flow_block_binder_type binder_type)
1306 {
1307 	struct tcf_block_owner_item *item;
1308 
1309 	list_for_each_entry(item, &block->owner_list, list) {
1310 		if (item->q == q && item->binder_type == binder_type) {
1311 			list_del(&item->list);
1312 			kfree(item);
1313 			return;
1314 		}
1315 	}
1316 	WARN_ON(1);
1317 }
1318 
tcf_block_get_ext(struct tcf_block ** p_block,struct Qdisc * q,struct tcf_block_ext_info * ei,struct netlink_ext_ack * extack)1319 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1320 		      struct tcf_block_ext_info *ei,
1321 		      struct netlink_ext_ack *extack)
1322 {
1323 	struct net *net = qdisc_net(q);
1324 	struct tcf_block *block = NULL;
1325 	int err;
1326 
1327 	if (ei->block_index)
1328 		/* block_index not 0 means the shared block is requested */
1329 		block = tcf_block_refcnt_get(net, ei->block_index);
1330 
1331 	if (!block) {
1332 		block = tcf_block_create(net, q, ei->block_index, extack);
1333 		if (IS_ERR(block))
1334 			return PTR_ERR(block);
1335 		if (tcf_block_shared(block)) {
1336 			err = tcf_block_insert(block, net, extack);
1337 			if (err)
1338 				goto err_block_insert;
1339 		}
1340 	}
1341 
1342 	err = tcf_block_owner_add(block, q, ei->binder_type);
1343 	if (err)
1344 		goto err_block_owner_add;
1345 
1346 	tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1347 
1348 	err = tcf_chain0_head_change_cb_add(block, ei, extack);
1349 	if (err)
1350 		goto err_chain0_head_change_cb_add;
1351 
1352 	err = tcf_block_offload_bind(block, q, ei, extack);
1353 	if (err)
1354 		goto err_block_offload_bind;
1355 
1356 	*p_block = block;
1357 	return 0;
1358 
1359 err_block_offload_bind:
1360 	tcf_chain0_head_change_cb_del(block, ei);
1361 err_chain0_head_change_cb_add:
1362 	tcf_block_owner_del(block, q, ei->binder_type);
1363 err_block_owner_add:
1364 err_block_insert:
1365 	tcf_block_refcnt_put(block, true);
1366 	return err;
1367 }
1368 EXPORT_SYMBOL(tcf_block_get_ext);
1369 
tcf_chain_head_change_dflt(struct tcf_proto * tp_head,void * priv)1370 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1371 {
1372 	struct tcf_proto __rcu **p_filter_chain = priv;
1373 
1374 	rcu_assign_pointer(*p_filter_chain, tp_head);
1375 }
1376 
tcf_block_get(struct tcf_block ** p_block,struct tcf_proto __rcu ** p_filter_chain,struct Qdisc * q,struct netlink_ext_ack * extack)1377 int tcf_block_get(struct tcf_block **p_block,
1378 		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1379 		  struct netlink_ext_ack *extack)
1380 {
1381 	struct tcf_block_ext_info ei = {
1382 		.chain_head_change = tcf_chain_head_change_dflt,
1383 		.chain_head_change_priv = p_filter_chain,
1384 	};
1385 
1386 	WARN_ON(!p_filter_chain);
1387 	return tcf_block_get_ext(p_block, q, &ei, extack);
1388 }
1389 EXPORT_SYMBOL(tcf_block_get);
1390 
1391 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
1392  * actions should be all removed after flushing.
1393  */
tcf_block_put_ext(struct tcf_block * block,struct Qdisc * q,struct tcf_block_ext_info * ei)1394 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1395 		       struct tcf_block_ext_info *ei)
1396 {
1397 	if (!block)
1398 		return;
1399 	tcf_chain0_head_change_cb_del(block, ei);
1400 	tcf_block_owner_del(block, q, ei->binder_type);
1401 
1402 	__tcf_block_put(block, q, ei, true);
1403 }
1404 EXPORT_SYMBOL(tcf_block_put_ext);
1405 
tcf_block_put(struct tcf_block * block)1406 void tcf_block_put(struct tcf_block *block)
1407 {
1408 	struct tcf_block_ext_info ei = {0, };
1409 
1410 	if (!block)
1411 		return;
1412 	tcf_block_put_ext(block, block->q, &ei);
1413 }
1414 
1415 EXPORT_SYMBOL(tcf_block_put);
1416 
1417 static int
tcf_block_playback_offloads(struct tcf_block * block,flow_setup_cb_t * cb,void * cb_priv,bool add,bool offload_in_use,struct netlink_ext_ack * extack)1418 tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
1419 			    void *cb_priv, bool add, bool offload_in_use,
1420 			    struct netlink_ext_ack *extack)
1421 {
1422 	struct tcf_chain *chain, *chain_prev;
1423 	struct tcf_proto *tp, *tp_prev;
1424 	int err;
1425 
1426 	lockdep_assert_held(&block->cb_lock);
1427 
1428 	for (chain = __tcf_get_next_chain(block, NULL);
1429 	     chain;
1430 	     chain_prev = chain,
1431 		     chain = __tcf_get_next_chain(block, chain),
1432 		     tcf_chain_put(chain_prev)) {
1433 		for (tp = __tcf_get_next_proto(chain, NULL); tp;
1434 		     tp_prev = tp,
1435 			     tp = __tcf_get_next_proto(chain, tp),
1436 			     tcf_proto_put(tp_prev, true, NULL)) {
1437 			if (tp->ops->reoffload) {
1438 				err = tp->ops->reoffload(tp, add, cb, cb_priv,
1439 							 extack);
1440 				if (err && add)
1441 					goto err_playback_remove;
1442 			} else if (add && offload_in_use) {
1443 				err = -EOPNOTSUPP;
1444 				NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1445 				goto err_playback_remove;
1446 			}
1447 		}
1448 	}
1449 
1450 	return 0;
1451 
1452 err_playback_remove:
1453 	tcf_proto_put(tp, true, NULL);
1454 	tcf_chain_put(chain);
1455 	tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1456 				    extack);
1457 	return err;
1458 }
1459 
tcf_block_bind(struct tcf_block * block,struct flow_block_offload * bo)1460 static int tcf_block_bind(struct tcf_block *block,
1461 			  struct flow_block_offload *bo)
1462 {
1463 	struct flow_block_cb *block_cb, *next;
1464 	int err, i = 0;
1465 
1466 	lockdep_assert_held(&block->cb_lock);
1467 
1468 	list_for_each_entry(block_cb, &bo->cb_list, list) {
1469 		err = tcf_block_playback_offloads(block, block_cb->cb,
1470 						  block_cb->cb_priv, true,
1471 						  tcf_block_offload_in_use(block),
1472 						  bo->extack);
1473 		if (err)
1474 			goto err_unroll;
1475 		if (!bo->unlocked_driver_cb)
1476 			block->lockeddevcnt++;
1477 
1478 		i++;
1479 	}
1480 	list_splice(&bo->cb_list, &block->flow_block.cb_list);
1481 
1482 	return 0;
1483 
1484 err_unroll:
1485 	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1486 		if (i-- > 0) {
1487 			list_del(&block_cb->list);
1488 			tcf_block_playback_offloads(block, block_cb->cb,
1489 						    block_cb->cb_priv, false,
1490 						    tcf_block_offload_in_use(block),
1491 						    NULL);
1492 			if (!bo->unlocked_driver_cb)
1493 				block->lockeddevcnt--;
1494 		}
1495 		flow_block_cb_free(block_cb);
1496 	}
1497 
1498 	return err;
1499 }
1500 
tcf_block_unbind(struct tcf_block * block,struct flow_block_offload * bo)1501 static void tcf_block_unbind(struct tcf_block *block,
1502 			     struct flow_block_offload *bo)
1503 {
1504 	struct flow_block_cb *block_cb, *next;
1505 
1506 	lockdep_assert_held(&block->cb_lock);
1507 
1508 	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1509 		tcf_block_playback_offloads(block, block_cb->cb,
1510 					    block_cb->cb_priv, false,
1511 					    tcf_block_offload_in_use(block),
1512 					    NULL);
1513 		list_del(&block_cb->list);
1514 		flow_block_cb_free(block_cb);
1515 		if (!bo->unlocked_driver_cb)
1516 			block->lockeddevcnt--;
1517 	}
1518 }
1519 
tcf_block_setup(struct tcf_block * block,struct flow_block_offload * bo)1520 static int tcf_block_setup(struct tcf_block *block,
1521 			   struct flow_block_offload *bo)
1522 {
1523 	int err;
1524 
1525 	switch (bo->command) {
1526 	case FLOW_BLOCK_BIND:
1527 		err = tcf_block_bind(block, bo);
1528 		break;
1529 	case FLOW_BLOCK_UNBIND:
1530 		err = 0;
1531 		tcf_block_unbind(block, bo);
1532 		break;
1533 	default:
1534 		WARN_ON_ONCE(1);
1535 		err = -EOPNOTSUPP;
1536 	}
1537 
1538 	return err;
1539 }
1540 
1541 /* Main classifier routine: scans classifier chain attached
1542  * to this qdisc, (optionally) tests for protocol and asks
1543  * specific classifiers.
1544  */
__tcf_classify(struct sk_buff * skb,const struct tcf_proto * tp,const struct tcf_proto * orig_tp,struct tcf_result * res,bool compat_mode,u32 * last_executed_chain)1545 static inline int __tcf_classify(struct sk_buff *skb,
1546 				 const struct tcf_proto *tp,
1547 				 const struct tcf_proto *orig_tp,
1548 				 struct tcf_result *res,
1549 				 bool compat_mode,
1550 				 u32 *last_executed_chain)
1551 {
1552 #ifdef CONFIG_NET_CLS_ACT
1553 	const int max_reclassify_loop = 16;
1554 	const struct tcf_proto *first_tp;
1555 	int limit = 0;
1556 
1557 reclassify:
1558 #endif
1559 	for (; tp; tp = rcu_dereference_bh(tp->next)) {
1560 		__be16 protocol = skb_protocol(skb, false);
1561 		int err;
1562 
1563 		if (tp->protocol != protocol &&
1564 		    tp->protocol != htons(ETH_P_ALL))
1565 			continue;
1566 
1567 		err = tp->classify(skb, tp, res);
1568 #ifdef CONFIG_NET_CLS_ACT
1569 		if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1570 			first_tp = orig_tp;
1571 			*last_executed_chain = first_tp->chain->index;
1572 			goto reset;
1573 		} else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1574 			first_tp = res->goto_tp;
1575 			*last_executed_chain = err & TC_ACT_EXT_VAL_MASK;
1576 			goto reset;
1577 		}
1578 #endif
1579 		if (err >= 0)
1580 			return err;
1581 	}
1582 
1583 	return TC_ACT_UNSPEC; /* signal: continue lookup */
1584 #ifdef CONFIG_NET_CLS_ACT
1585 reset:
1586 	if (unlikely(limit++ >= max_reclassify_loop)) {
1587 		net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1588 				       tp->chain->block->index,
1589 				       tp->prio & 0xffff,
1590 				       ntohs(tp->protocol));
1591 		return TC_ACT_SHOT;
1592 	}
1593 
1594 	tp = first_tp;
1595 	goto reclassify;
1596 #endif
1597 }
1598 
tcf_classify(struct sk_buff * skb,const struct tcf_block * block,const struct tcf_proto * tp,struct tcf_result * res,bool compat_mode)1599 int tcf_classify(struct sk_buff *skb,
1600 		 const struct tcf_block *block,
1601 		 const struct tcf_proto *tp,
1602 		 struct tcf_result *res, bool compat_mode)
1603 {
1604 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1605 	u32 last_executed_chain = 0;
1606 
1607 	return __tcf_classify(skb, tp, tp, res, compat_mode,
1608 			      &last_executed_chain);
1609 #else
1610 	u32 last_executed_chain = tp ? tp->chain->index : 0;
1611 	const struct tcf_proto *orig_tp = tp;
1612 	struct tc_skb_ext *ext;
1613 	int ret;
1614 
1615 	if (block) {
1616 		ext = skb_ext_find(skb, TC_SKB_EXT);
1617 
1618 		if (ext && ext->chain) {
1619 			struct tcf_chain *fchain;
1620 
1621 			fchain = tcf_chain_lookup_rcu(block, ext->chain);
1622 			if (!fchain)
1623 				return TC_ACT_SHOT;
1624 
1625 			/* Consume, so cloned/redirect skbs won't inherit ext */
1626 			skb_ext_del(skb, TC_SKB_EXT);
1627 
1628 			tp = rcu_dereference_bh(fchain->filter_chain);
1629 			last_executed_chain = fchain->index;
1630 		}
1631 	}
1632 
1633 	ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode,
1634 			     &last_executed_chain);
1635 
1636 	if (tc_skb_ext_tc_enabled()) {
1637 		/* If we missed on some chain */
1638 		if (ret == TC_ACT_UNSPEC && last_executed_chain) {
1639 			struct tc_skb_cb *cb = tc_skb_cb(skb);
1640 
1641 			ext = tc_skb_ext_alloc(skb);
1642 			if (WARN_ON_ONCE(!ext))
1643 				return TC_ACT_SHOT;
1644 			ext->chain = last_executed_chain;
1645 			ext->mru = cb->mru;
1646 			ext->post_ct = cb->post_ct;
1647 			ext->post_ct_snat = cb->post_ct_snat;
1648 			ext->post_ct_dnat = cb->post_ct_dnat;
1649 			ext->zone = cb->zone;
1650 		}
1651 	}
1652 
1653 	return ret;
1654 #endif
1655 }
1656 EXPORT_SYMBOL(tcf_classify);
1657 
1658 struct tcf_chain_info {
1659 	struct tcf_proto __rcu **pprev;
1660 	struct tcf_proto __rcu *next;
1661 };
1662 
tcf_chain_tp_prev(struct tcf_chain * chain,struct tcf_chain_info * chain_info)1663 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1664 					   struct tcf_chain_info *chain_info)
1665 {
1666 	return tcf_chain_dereference(*chain_info->pprev, chain);
1667 }
1668 
tcf_chain_tp_insert(struct tcf_chain * chain,struct tcf_chain_info * chain_info,struct tcf_proto * tp)1669 static int tcf_chain_tp_insert(struct tcf_chain *chain,
1670 			       struct tcf_chain_info *chain_info,
1671 			       struct tcf_proto *tp)
1672 {
1673 	if (chain->flushing)
1674 		return -EAGAIN;
1675 
1676 	RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1677 	if (*chain_info->pprev == chain->filter_chain)
1678 		tcf_chain0_head_change(chain, tp);
1679 	tcf_proto_get(tp);
1680 	rcu_assign_pointer(*chain_info->pprev, tp);
1681 
1682 	return 0;
1683 }
1684 
tcf_chain_tp_remove(struct tcf_chain * chain,struct tcf_chain_info * chain_info,struct tcf_proto * tp)1685 static void tcf_chain_tp_remove(struct tcf_chain *chain,
1686 				struct tcf_chain_info *chain_info,
1687 				struct tcf_proto *tp)
1688 {
1689 	struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1690 
1691 	tcf_proto_mark_delete(tp);
1692 	if (tp == chain->filter_chain)
1693 		tcf_chain0_head_change(chain, next);
1694 	RCU_INIT_POINTER(*chain_info->pprev, next);
1695 }
1696 
1697 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1698 					   struct tcf_chain_info *chain_info,
1699 					   u32 protocol, u32 prio,
1700 					   bool prio_allocate);
1701 
1702 /* Try to insert new proto.
1703  * If proto with specified priority already exists, free new proto
1704  * and return existing one.
1705  */
1706 
tcf_chain_tp_insert_unique(struct tcf_chain * chain,struct tcf_proto * tp_new,u32 protocol,u32 prio,bool rtnl_held)1707 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1708 						    struct tcf_proto *tp_new,
1709 						    u32 protocol, u32 prio,
1710 						    bool rtnl_held)
1711 {
1712 	struct tcf_chain_info chain_info;
1713 	struct tcf_proto *tp;
1714 	int err = 0;
1715 
1716 	mutex_lock(&chain->filter_chain_lock);
1717 
1718 	if (tcf_proto_exists_destroying(chain, tp_new)) {
1719 		mutex_unlock(&chain->filter_chain_lock);
1720 		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1721 		return ERR_PTR(-EAGAIN);
1722 	}
1723 
1724 	tp = tcf_chain_tp_find(chain, &chain_info,
1725 			       protocol, prio, false);
1726 	if (!tp)
1727 		err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
1728 	mutex_unlock(&chain->filter_chain_lock);
1729 
1730 	if (tp) {
1731 		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1732 		tp_new = tp;
1733 	} else if (err) {
1734 		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1735 		tp_new = ERR_PTR(err);
1736 	}
1737 
1738 	return tp_new;
1739 }
1740 
tcf_chain_tp_delete_empty(struct tcf_chain * chain,struct tcf_proto * tp,bool rtnl_held,struct netlink_ext_ack * extack)1741 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
1742 				      struct tcf_proto *tp, bool rtnl_held,
1743 				      struct netlink_ext_ack *extack)
1744 {
1745 	struct tcf_chain_info chain_info;
1746 	struct tcf_proto *tp_iter;
1747 	struct tcf_proto **pprev;
1748 	struct tcf_proto *next;
1749 
1750 	mutex_lock(&chain->filter_chain_lock);
1751 
1752 	/* Atomically find and remove tp from chain. */
1753 	for (pprev = &chain->filter_chain;
1754 	     (tp_iter = tcf_chain_dereference(*pprev, chain));
1755 	     pprev = &tp_iter->next) {
1756 		if (tp_iter == tp) {
1757 			chain_info.pprev = pprev;
1758 			chain_info.next = tp_iter->next;
1759 			WARN_ON(tp_iter->deleting);
1760 			break;
1761 		}
1762 	}
1763 	/* Verify that tp still exists and no new filters were inserted
1764 	 * concurrently.
1765 	 * Mark tp for deletion if it is empty.
1766 	 */
1767 	if (!tp_iter || !tcf_proto_check_delete(tp)) {
1768 		mutex_unlock(&chain->filter_chain_lock);
1769 		return;
1770 	}
1771 
1772 	tcf_proto_signal_destroying(chain, tp);
1773 	next = tcf_chain_dereference(chain_info.next, chain);
1774 	if (tp == chain->filter_chain)
1775 		tcf_chain0_head_change(chain, next);
1776 	RCU_INIT_POINTER(*chain_info.pprev, next);
1777 	mutex_unlock(&chain->filter_chain_lock);
1778 
1779 	tcf_proto_put(tp, rtnl_held, extack);
1780 }
1781 
tcf_chain_tp_find(struct tcf_chain * chain,struct tcf_chain_info * chain_info,u32 protocol,u32 prio,bool prio_allocate)1782 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1783 					   struct tcf_chain_info *chain_info,
1784 					   u32 protocol, u32 prio,
1785 					   bool prio_allocate)
1786 {
1787 	struct tcf_proto **pprev;
1788 	struct tcf_proto *tp;
1789 
1790 	/* Check the chain for existence of proto-tcf with this priority */
1791 	for (pprev = &chain->filter_chain;
1792 	     (tp = tcf_chain_dereference(*pprev, chain));
1793 	     pprev = &tp->next) {
1794 		if (tp->prio >= prio) {
1795 			if (tp->prio == prio) {
1796 				if (prio_allocate ||
1797 				    (tp->protocol != protocol && protocol))
1798 					return ERR_PTR(-EINVAL);
1799 			} else {
1800 				tp = NULL;
1801 			}
1802 			break;
1803 		}
1804 	}
1805 	chain_info->pprev = pprev;
1806 	if (tp) {
1807 		chain_info->next = tp->next;
1808 		tcf_proto_get(tp);
1809 	} else {
1810 		chain_info->next = NULL;
1811 	}
1812 	return tp;
1813 }
1814 
tcf_fill_node(struct net * net,struct sk_buff * skb,struct tcf_proto * tp,struct tcf_block * block,struct Qdisc * q,u32 parent,void * fh,u32 portid,u32 seq,u16 flags,int event,bool terse_dump,bool rtnl_held)1815 static int tcf_fill_node(struct net *net, struct sk_buff *skb,
1816 			 struct tcf_proto *tp, struct tcf_block *block,
1817 			 struct Qdisc *q, u32 parent, void *fh,
1818 			 u32 portid, u32 seq, u16 flags, int event,
1819 			 bool terse_dump, bool rtnl_held)
1820 {
1821 	struct tcmsg *tcm;
1822 	struct nlmsghdr  *nlh;
1823 	unsigned char *b = skb_tail_pointer(skb);
1824 
1825 	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1826 	if (!nlh)
1827 		goto out_nlmsg_trim;
1828 	tcm = nlmsg_data(nlh);
1829 	tcm->tcm_family = AF_UNSPEC;
1830 	tcm->tcm__pad1 = 0;
1831 	tcm->tcm__pad2 = 0;
1832 	if (q) {
1833 		tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1834 		tcm->tcm_parent = parent;
1835 	} else {
1836 		tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1837 		tcm->tcm_block_index = block->index;
1838 	}
1839 	tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1840 	if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1841 		goto nla_put_failure;
1842 	if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1843 		goto nla_put_failure;
1844 	if (!fh) {
1845 		tcm->tcm_handle = 0;
1846 	} else if (terse_dump) {
1847 		if (tp->ops->terse_dump) {
1848 			if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
1849 						rtnl_held) < 0)
1850 				goto nla_put_failure;
1851 		} else {
1852 			goto cls_op_not_supp;
1853 		}
1854 	} else {
1855 		if (tp->ops->dump &&
1856 		    tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
1857 			goto nla_put_failure;
1858 	}
1859 	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1860 	return skb->len;
1861 
1862 out_nlmsg_trim:
1863 nla_put_failure:
1864 cls_op_not_supp:
1865 	nlmsg_trim(skb, b);
1866 	return -1;
1867 }
1868 
tfilter_notify(struct net * net,struct sk_buff * oskb,struct nlmsghdr * n,struct tcf_proto * tp,struct tcf_block * block,struct Qdisc * q,u32 parent,void * fh,int event,bool unicast,bool rtnl_held)1869 static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1870 			  struct nlmsghdr *n, struct tcf_proto *tp,
1871 			  struct tcf_block *block, struct Qdisc *q,
1872 			  u32 parent, void *fh, int event, bool unicast,
1873 			  bool rtnl_held)
1874 {
1875 	struct sk_buff *skb;
1876 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1877 	int err = 0;
1878 
1879 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1880 	if (!skb)
1881 		return -ENOBUFS;
1882 
1883 	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1884 			  n->nlmsg_seq, n->nlmsg_flags, event,
1885 			  false, rtnl_held) <= 0) {
1886 		kfree_skb(skb);
1887 		return -EINVAL;
1888 	}
1889 
1890 	if (unicast)
1891 		err = rtnl_unicast(skb, net, portid);
1892 	else
1893 		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1894 				     n->nlmsg_flags & NLM_F_ECHO);
1895 	return err;
1896 }
1897 
tfilter_del_notify(struct net * net,struct sk_buff * oskb,struct nlmsghdr * n,struct tcf_proto * tp,struct tcf_block * block,struct Qdisc * q,u32 parent,void * fh,bool unicast,bool * last,bool rtnl_held,struct netlink_ext_ack * extack)1898 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1899 			      struct nlmsghdr *n, struct tcf_proto *tp,
1900 			      struct tcf_block *block, struct Qdisc *q,
1901 			      u32 parent, void *fh, bool unicast, bool *last,
1902 			      bool rtnl_held, struct netlink_ext_ack *extack)
1903 {
1904 	struct sk_buff *skb;
1905 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1906 	int err;
1907 
1908 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1909 	if (!skb)
1910 		return -ENOBUFS;
1911 
1912 	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1913 			  n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
1914 			  false, rtnl_held) <= 0) {
1915 		NL_SET_ERR_MSG(extack, "Failed to build del event notification");
1916 		kfree_skb(skb);
1917 		return -EINVAL;
1918 	}
1919 
1920 	err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
1921 	if (err) {
1922 		kfree_skb(skb);
1923 		return err;
1924 	}
1925 
1926 	if (unicast)
1927 		err = rtnl_unicast(skb, net, portid);
1928 	else
1929 		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1930 				     n->nlmsg_flags & NLM_F_ECHO);
1931 	if (err < 0)
1932 		NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
1933 
1934 	return err;
1935 }
1936 
tfilter_notify_chain(struct net * net,struct sk_buff * oskb,struct tcf_block * block,struct Qdisc * q,u32 parent,struct nlmsghdr * n,struct tcf_chain * chain,int event)1937 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
1938 				 struct tcf_block *block, struct Qdisc *q,
1939 				 u32 parent, struct nlmsghdr *n,
1940 				 struct tcf_chain *chain, int event)
1941 {
1942 	struct tcf_proto *tp;
1943 
1944 	for (tp = tcf_get_next_proto(chain, NULL);
1945 	     tp; tp = tcf_get_next_proto(chain, tp))
1946 		tfilter_notify(net, oskb, n, tp, block,
1947 			       q, parent, NULL, event, false, true);
1948 }
1949 
tfilter_put(struct tcf_proto * tp,void * fh)1950 static void tfilter_put(struct tcf_proto *tp, void *fh)
1951 {
1952 	if (tp->ops->put && fh)
1953 		tp->ops->put(tp, fh);
1954 }
1955 
tc_new_tfilter(struct sk_buff * skb,struct nlmsghdr * n,struct netlink_ext_ack * extack)1956 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1957 			  struct netlink_ext_ack *extack)
1958 {
1959 	struct net *net = sock_net(skb->sk);
1960 	struct nlattr *tca[TCA_MAX + 1];
1961 	char name[IFNAMSIZ];
1962 	struct tcmsg *t;
1963 	u32 protocol;
1964 	u32 prio;
1965 	bool prio_allocate;
1966 	u32 parent;
1967 	u32 chain_index;
1968 	struct Qdisc *q;
1969 	struct tcf_chain_info chain_info;
1970 	struct tcf_chain *chain;
1971 	struct tcf_block *block;
1972 	struct tcf_proto *tp;
1973 	unsigned long cl;
1974 	void *fh;
1975 	int err;
1976 	int tp_created;
1977 	bool rtnl_held = false;
1978 	u32 flags;
1979 
1980 replay:
1981 	tp_created = 0;
1982 
1983 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
1984 				     rtm_tca_policy, extack);
1985 	if (err < 0)
1986 		return err;
1987 
1988 	t = nlmsg_data(n);
1989 	protocol = TC_H_MIN(t->tcm_info);
1990 	prio = TC_H_MAJ(t->tcm_info);
1991 	prio_allocate = false;
1992 	parent = t->tcm_parent;
1993 	tp = NULL;
1994 	cl = 0;
1995 	block = NULL;
1996 	q = NULL;
1997 	chain = NULL;
1998 	flags = 0;
1999 
2000 	if (prio == 0) {
2001 		/* If no priority is provided by the user,
2002 		 * we allocate one.
2003 		 */
2004 		if (n->nlmsg_flags & NLM_F_CREATE) {
2005 			prio = TC_H_MAKE(0x80000000U, 0U);
2006 			prio_allocate = true;
2007 		} else {
2008 			NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2009 			return -ENOENT;
2010 		}
2011 	}
2012 
2013 	/* Find head of filter chain. */
2014 
2015 	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2016 	if (err)
2017 		return err;
2018 
2019 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2020 		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2021 		err = -EINVAL;
2022 		goto errout;
2023 	}
2024 
2025 	/* Take rtnl mutex if rtnl_held was set to true on previous iteration,
2026 	 * block is shared (no qdisc found), qdisc is not unlocked, classifier
2027 	 * type is not specified, classifier is not unlocked.
2028 	 */
2029 	if (rtnl_held ||
2030 	    (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2031 	    !tcf_proto_is_unlocked(name)) {
2032 		rtnl_held = true;
2033 		rtnl_lock();
2034 	}
2035 
2036 	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2037 	if (err)
2038 		goto errout;
2039 
2040 	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2041 				 extack);
2042 	if (IS_ERR(block)) {
2043 		err = PTR_ERR(block);
2044 		goto errout;
2045 	}
2046 	block->classid = parent;
2047 
2048 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2049 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2050 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2051 		err = -EINVAL;
2052 		goto errout;
2053 	}
2054 	chain = tcf_chain_get(block, chain_index, true);
2055 	if (!chain) {
2056 		NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
2057 		err = -ENOMEM;
2058 		goto errout;
2059 	}
2060 
2061 	mutex_lock(&chain->filter_chain_lock);
2062 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2063 			       prio, prio_allocate);
2064 	if (IS_ERR(tp)) {
2065 		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2066 		err = PTR_ERR(tp);
2067 		goto errout_locked;
2068 	}
2069 
2070 	if (tp == NULL) {
2071 		struct tcf_proto *tp_new = NULL;
2072 
2073 		if (chain->flushing) {
2074 			err = -EAGAIN;
2075 			goto errout_locked;
2076 		}
2077 
2078 		/* Proto-tcf does not exist, create new one */
2079 
2080 		if (tca[TCA_KIND] == NULL || !protocol) {
2081 			NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
2082 			err = -EINVAL;
2083 			goto errout_locked;
2084 		}
2085 
2086 		if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2087 			NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2088 			err = -ENOENT;
2089 			goto errout_locked;
2090 		}
2091 
2092 		if (prio_allocate)
2093 			prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2094 							       &chain_info));
2095 
2096 		mutex_unlock(&chain->filter_chain_lock);
2097 		tp_new = tcf_proto_create(name, protocol, prio, chain,
2098 					  rtnl_held, extack);
2099 		if (IS_ERR(tp_new)) {
2100 			err = PTR_ERR(tp_new);
2101 			goto errout_tp;
2102 		}
2103 
2104 		tp_created = 1;
2105 		tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2106 						rtnl_held);
2107 		if (IS_ERR(tp)) {
2108 			err = PTR_ERR(tp);
2109 			goto errout_tp;
2110 		}
2111 	} else {
2112 		mutex_unlock(&chain->filter_chain_lock);
2113 	}
2114 
2115 	if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2116 		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2117 		err = -EINVAL;
2118 		goto errout;
2119 	}
2120 
2121 	fh = tp->ops->get(tp, t->tcm_handle);
2122 
2123 	if (!fh) {
2124 		if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2125 			NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2126 			err = -ENOENT;
2127 			goto errout;
2128 		}
2129 	} else if (n->nlmsg_flags & NLM_F_EXCL) {
2130 		tfilter_put(tp, fh);
2131 		NL_SET_ERR_MSG(extack, "Filter already exists");
2132 		err = -EEXIST;
2133 		goto errout;
2134 	}
2135 
2136 	if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2137 		tfilter_put(tp, fh);
2138 		NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2139 		err = -EINVAL;
2140 		goto errout;
2141 	}
2142 
2143 	if (!(n->nlmsg_flags & NLM_F_CREATE))
2144 		flags |= TCA_ACT_FLAGS_REPLACE;
2145 	if (!rtnl_held)
2146 		flags |= TCA_ACT_FLAGS_NO_RTNL;
2147 	err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
2148 			      flags, extack);
2149 	if (err == 0) {
2150 		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2151 			       RTM_NEWTFILTER, false, rtnl_held);
2152 		tfilter_put(tp, fh);
2153 		/* q pointer is NULL for shared blocks */
2154 		if (q)
2155 			q->flags &= ~TCQ_F_CAN_BYPASS;
2156 	}
2157 
2158 errout:
2159 	if (err && tp_created)
2160 		tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
2161 errout_tp:
2162 	if (chain) {
2163 		if (tp && !IS_ERR(tp))
2164 			tcf_proto_put(tp, rtnl_held, NULL);
2165 		if (!tp_created)
2166 			tcf_chain_put(chain);
2167 	}
2168 	tcf_block_release(q, block, rtnl_held);
2169 
2170 	if (rtnl_held)
2171 		rtnl_unlock();
2172 
2173 	if (err == -EAGAIN) {
2174 		/* Take rtnl lock in case EAGAIN is caused by concurrent flush
2175 		 * of target chain.
2176 		 */
2177 		rtnl_held = true;
2178 		/* Replay the request. */
2179 		goto replay;
2180 	}
2181 	return err;
2182 
2183 errout_locked:
2184 	mutex_unlock(&chain->filter_chain_lock);
2185 	goto errout;
2186 }
2187 
tc_del_tfilter(struct sk_buff * skb,struct nlmsghdr * n,struct netlink_ext_ack * extack)2188 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2189 			  struct netlink_ext_ack *extack)
2190 {
2191 	struct net *net = sock_net(skb->sk);
2192 	struct nlattr *tca[TCA_MAX + 1];
2193 	char name[IFNAMSIZ];
2194 	struct tcmsg *t;
2195 	u32 protocol;
2196 	u32 prio;
2197 	u32 parent;
2198 	u32 chain_index;
2199 	struct Qdisc *q = NULL;
2200 	struct tcf_chain_info chain_info;
2201 	struct tcf_chain *chain = NULL;
2202 	struct tcf_block *block = NULL;
2203 	struct tcf_proto *tp = NULL;
2204 	unsigned long cl = 0;
2205 	void *fh = NULL;
2206 	int err;
2207 	bool rtnl_held = false;
2208 
2209 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2210 				     rtm_tca_policy, extack);
2211 	if (err < 0)
2212 		return err;
2213 
2214 	t = nlmsg_data(n);
2215 	protocol = TC_H_MIN(t->tcm_info);
2216 	prio = TC_H_MAJ(t->tcm_info);
2217 	parent = t->tcm_parent;
2218 
2219 	if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2220 		NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2221 		return -ENOENT;
2222 	}
2223 
2224 	/* Find head of filter chain. */
2225 
2226 	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2227 	if (err)
2228 		return err;
2229 
2230 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2231 		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2232 		err = -EINVAL;
2233 		goto errout;
2234 	}
2235 	/* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2236 	 * found), qdisc is not unlocked, classifier type is not specified,
2237 	 * classifier is not unlocked.
2238 	 */
2239 	if (!prio ||
2240 	    (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2241 	    !tcf_proto_is_unlocked(name)) {
2242 		rtnl_held = true;
2243 		rtnl_lock();
2244 	}
2245 
2246 	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2247 	if (err)
2248 		goto errout;
2249 
2250 	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2251 				 extack);
2252 	if (IS_ERR(block)) {
2253 		err = PTR_ERR(block);
2254 		goto errout;
2255 	}
2256 
2257 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2258 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2259 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2260 		err = -EINVAL;
2261 		goto errout;
2262 	}
2263 	chain = tcf_chain_get(block, chain_index, false);
2264 	if (!chain) {
2265 		/* User requested flush on non-existent chain. Nothing to do,
2266 		 * so just return success.
2267 		 */
2268 		if (prio == 0) {
2269 			err = 0;
2270 			goto errout;
2271 		}
2272 		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2273 		err = -ENOENT;
2274 		goto errout;
2275 	}
2276 
2277 	if (prio == 0) {
2278 		tfilter_notify_chain(net, skb, block, q, parent, n,
2279 				     chain, RTM_DELTFILTER);
2280 		tcf_chain_flush(chain, rtnl_held);
2281 		err = 0;
2282 		goto errout;
2283 	}
2284 
2285 	mutex_lock(&chain->filter_chain_lock);
2286 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2287 			       prio, false);
2288 	if (!tp || IS_ERR(tp)) {
2289 		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2290 		err = tp ? PTR_ERR(tp) : -ENOENT;
2291 		goto errout_locked;
2292 	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2293 		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2294 		err = -EINVAL;
2295 		goto errout_locked;
2296 	} else if (t->tcm_handle == 0) {
2297 		tcf_proto_signal_destroying(chain, tp);
2298 		tcf_chain_tp_remove(chain, &chain_info, tp);
2299 		mutex_unlock(&chain->filter_chain_lock);
2300 
2301 		tcf_proto_put(tp, rtnl_held, NULL);
2302 		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2303 			       RTM_DELTFILTER, false, rtnl_held);
2304 		err = 0;
2305 		goto errout;
2306 	}
2307 	mutex_unlock(&chain->filter_chain_lock);
2308 
2309 	fh = tp->ops->get(tp, t->tcm_handle);
2310 
2311 	if (!fh) {
2312 		NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2313 		err = -ENOENT;
2314 	} else {
2315 		bool last;
2316 
2317 		err = tfilter_del_notify(net, skb, n, tp, block,
2318 					 q, parent, fh, false, &last,
2319 					 rtnl_held, extack);
2320 
2321 		if (err)
2322 			goto errout;
2323 		if (last)
2324 			tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
2325 	}
2326 
2327 errout:
2328 	if (chain) {
2329 		if (tp && !IS_ERR(tp))
2330 			tcf_proto_put(tp, rtnl_held, NULL);
2331 		tcf_chain_put(chain);
2332 	}
2333 	tcf_block_release(q, block, rtnl_held);
2334 
2335 	if (rtnl_held)
2336 		rtnl_unlock();
2337 
2338 	return err;
2339 
2340 errout_locked:
2341 	mutex_unlock(&chain->filter_chain_lock);
2342 	goto errout;
2343 }
2344 
tc_get_tfilter(struct sk_buff * skb,struct nlmsghdr * n,struct netlink_ext_ack * extack)2345 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2346 			  struct netlink_ext_ack *extack)
2347 {
2348 	struct net *net = sock_net(skb->sk);
2349 	struct nlattr *tca[TCA_MAX + 1];
2350 	char name[IFNAMSIZ];
2351 	struct tcmsg *t;
2352 	u32 protocol;
2353 	u32 prio;
2354 	u32 parent;
2355 	u32 chain_index;
2356 	struct Qdisc *q = NULL;
2357 	struct tcf_chain_info chain_info;
2358 	struct tcf_chain *chain = NULL;
2359 	struct tcf_block *block = NULL;
2360 	struct tcf_proto *tp = NULL;
2361 	unsigned long cl = 0;
2362 	void *fh = NULL;
2363 	int err;
2364 	bool rtnl_held = false;
2365 
2366 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2367 				     rtm_tca_policy, extack);
2368 	if (err < 0)
2369 		return err;
2370 
2371 	t = nlmsg_data(n);
2372 	protocol = TC_H_MIN(t->tcm_info);
2373 	prio = TC_H_MAJ(t->tcm_info);
2374 	parent = t->tcm_parent;
2375 
2376 	if (prio == 0) {
2377 		NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2378 		return -ENOENT;
2379 	}
2380 
2381 	/* Find head of filter chain. */
2382 
2383 	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2384 	if (err)
2385 		return err;
2386 
2387 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2388 		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2389 		err = -EINVAL;
2390 		goto errout;
2391 	}
2392 	/* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2393 	 * unlocked, classifier type is not specified, classifier is not
2394 	 * unlocked.
2395 	 */
2396 	if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2397 	    !tcf_proto_is_unlocked(name)) {
2398 		rtnl_held = true;
2399 		rtnl_lock();
2400 	}
2401 
2402 	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2403 	if (err)
2404 		goto errout;
2405 
2406 	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2407 				 extack);
2408 	if (IS_ERR(block)) {
2409 		err = PTR_ERR(block);
2410 		goto errout;
2411 	}
2412 
2413 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2414 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2415 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2416 		err = -EINVAL;
2417 		goto errout;
2418 	}
2419 	chain = tcf_chain_get(block, chain_index, false);
2420 	if (!chain) {
2421 		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2422 		err = -EINVAL;
2423 		goto errout;
2424 	}
2425 
2426 	mutex_lock(&chain->filter_chain_lock);
2427 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2428 			       prio, false);
2429 	mutex_unlock(&chain->filter_chain_lock);
2430 	if (!tp || IS_ERR(tp)) {
2431 		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2432 		err = tp ? PTR_ERR(tp) : -ENOENT;
2433 		goto errout;
2434 	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2435 		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2436 		err = -EINVAL;
2437 		goto errout;
2438 	}
2439 
2440 	fh = tp->ops->get(tp, t->tcm_handle);
2441 
2442 	if (!fh) {
2443 		NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2444 		err = -ENOENT;
2445 	} else {
2446 		err = tfilter_notify(net, skb, n, tp, block, q, parent,
2447 				     fh, RTM_NEWTFILTER, true, rtnl_held);
2448 		if (err < 0)
2449 			NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2450 	}
2451 
2452 	tfilter_put(tp, fh);
2453 errout:
2454 	if (chain) {
2455 		if (tp && !IS_ERR(tp))
2456 			tcf_proto_put(tp, rtnl_held, NULL);
2457 		tcf_chain_put(chain);
2458 	}
2459 	tcf_block_release(q, block, rtnl_held);
2460 
2461 	if (rtnl_held)
2462 		rtnl_unlock();
2463 
2464 	return err;
2465 }
2466 
2467 struct tcf_dump_args {
2468 	struct tcf_walker w;
2469 	struct sk_buff *skb;
2470 	struct netlink_callback *cb;
2471 	struct tcf_block *block;
2472 	struct Qdisc *q;
2473 	u32 parent;
2474 	bool terse_dump;
2475 };
2476 
tcf_node_dump(struct tcf_proto * tp,void * n,struct tcf_walker * arg)2477 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2478 {
2479 	struct tcf_dump_args *a = (void *)arg;
2480 	struct net *net = sock_net(a->skb->sk);
2481 
2482 	return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
2483 			     n, NETLINK_CB(a->cb->skb).portid,
2484 			     a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2485 			     RTM_NEWTFILTER, a->terse_dump, true);
2486 }
2487 
tcf_chain_dump(struct tcf_chain * chain,struct Qdisc * q,u32 parent,struct sk_buff * skb,struct netlink_callback * cb,long index_start,long * p_index,bool terse)2488 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2489 			   struct sk_buff *skb, struct netlink_callback *cb,
2490 			   long index_start, long *p_index, bool terse)
2491 {
2492 	struct net *net = sock_net(skb->sk);
2493 	struct tcf_block *block = chain->block;
2494 	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2495 	struct tcf_proto *tp, *tp_prev;
2496 	struct tcf_dump_args arg;
2497 
2498 	for (tp = __tcf_get_next_proto(chain, NULL);
2499 	     tp;
2500 	     tp_prev = tp,
2501 		     tp = __tcf_get_next_proto(chain, tp),
2502 		     tcf_proto_put(tp_prev, true, NULL),
2503 		     (*p_index)++) {
2504 		if (*p_index < index_start)
2505 			continue;
2506 		if (TC_H_MAJ(tcm->tcm_info) &&
2507 		    TC_H_MAJ(tcm->tcm_info) != tp->prio)
2508 			continue;
2509 		if (TC_H_MIN(tcm->tcm_info) &&
2510 		    TC_H_MIN(tcm->tcm_info) != tp->protocol)
2511 			continue;
2512 		if (*p_index > index_start)
2513 			memset(&cb->args[1], 0,
2514 			       sizeof(cb->args) - sizeof(cb->args[0]));
2515 		if (cb->args[1] == 0) {
2516 			if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2517 					  NETLINK_CB(cb->skb).portid,
2518 					  cb->nlh->nlmsg_seq, NLM_F_MULTI,
2519 					  RTM_NEWTFILTER, false, true) <= 0)
2520 				goto errout;
2521 			cb->args[1] = 1;
2522 		}
2523 		if (!tp->ops->walk)
2524 			continue;
2525 		arg.w.fn = tcf_node_dump;
2526 		arg.skb = skb;
2527 		arg.cb = cb;
2528 		arg.block = block;
2529 		arg.q = q;
2530 		arg.parent = parent;
2531 		arg.w.stop = 0;
2532 		arg.w.skip = cb->args[1] - 1;
2533 		arg.w.count = 0;
2534 		arg.w.cookie = cb->args[2];
2535 		arg.terse_dump = terse;
2536 		tp->ops->walk(tp, &arg.w, true);
2537 		cb->args[2] = arg.w.cookie;
2538 		cb->args[1] = arg.w.count + 1;
2539 		if (arg.w.stop)
2540 			goto errout;
2541 	}
2542 	return true;
2543 
2544 errout:
2545 	tcf_proto_put(tp, true, NULL);
2546 	return false;
2547 }
2548 
2549 static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = {
2550 	[TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE),
2551 };
2552 
2553 /* called with RTNL */
tc_dump_tfilter(struct sk_buff * skb,struct netlink_callback * cb)2554 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2555 {
2556 	struct tcf_chain *chain, *chain_prev;
2557 	struct net *net = sock_net(skb->sk);
2558 	struct nlattr *tca[TCA_MAX + 1];
2559 	struct Qdisc *q = NULL;
2560 	struct tcf_block *block;
2561 	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2562 	bool terse_dump = false;
2563 	long index_start;
2564 	long index;
2565 	u32 parent;
2566 	int err;
2567 
2568 	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2569 		return skb->len;
2570 
2571 	err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2572 				     tcf_tfilter_dump_policy, cb->extack);
2573 	if (err)
2574 		return err;
2575 
2576 	if (tca[TCA_DUMP_FLAGS]) {
2577 		struct nla_bitfield32 flags =
2578 			nla_get_bitfield32(tca[TCA_DUMP_FLAGS]);
2579 
2580 		terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE;
2581 	}
2582 
2583 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2584 		block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2585 		if (!block)
2586 			goto out;
2587 		/* If we work with block index, q is NULL and parent value
2588 		 * will never be used in the following code. The check
2589 		 * in tcf_fill_node prevents it. However, compiler does not
2590 		 * see that far, so set parent to zero to silence the warning
2591 		 * about parent being uninitialized.
2592 		 */
2593 		parent = 0;
2594 	} else {
2595 		const struct Qdisc_class_ops *cops;
2596 		struct net_device *dev;
2597 		unsigned long cl = 0;
2598 
2599 		dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2600 		if (!dev)
2601 			return skb->len;
2602 
2603 		parent = tcm->tcm_parent;
2604 		if (!parent)
2605 			q = rtnl_dereference(dev->qdisc);
2606 		else
2607 			q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2608 		if (!q)
2609 			goto out;
2610 		cops = q->ops->cl_ops;
2611 		if (!cops)
2612 			goto out;
2613 		if (!cops->tcf_block)
2614 			goto out;
2615 		if (TC_H_MIN(tcm->tcm_parent)) {
2616 			cl = cops->find(q, tcm->tcm_parent);
2617 			if (cl == 0)
2618 				goto out;
2619 		}
2620 		block = cops->tcf_block(q, cl, NULL);
2621 		if (!block)
2622 			goto out;
2623 		parent = block->classid;
2624 		if (tcf_block_shared(block))
2625 			q = NULL;
2626 	}
2627 
2628 	index_start = cb->args[0];
2629 	index = 0;
2630 
2631 	for (chain = __tcf_get_next_chain(block, NULL);
2632 	     chain;
2633 	     chain_prev = chain,
2634 		     chain = __tcf_get_next_chain(block, chain),
2635 		     tcf_chain_put(chain_prev)) {
2636 		if (tca[TCA_CHAIN] &&
2637 		    nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2638 			continue;
2639 		if (!tcf_chain_dump(chain, q, parent, skb, cb,
2640 				    index_start, &index, terse_dump)) {
2641 			tcf_chain_put(chain);
2642 			err = -EMSGSIZE;
2643 			break;
2644 		}
2645 	}
2646 
2647 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2648 		tcf_block_refcnt_put(block, true);
2649 	cb->args[0] = index;
2650 
2651 out:
2652 	/* If we did no progress, the error (EMSGSIZE) is real */
2653 	if (skb->len == 0 && err)
2654 		return err;
2655 	return skb->len;
2656 }
2657 
tc_chain_fill_node(const struct tcf_proto_ops * tmplt_ops,void * tmplt_priv,u32 chain_index,struct net * net,struct sk_buff * skb,struct tcf_block * block,u32 portid,u32 seq,u16 flags,int event)2658 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2659 			      void *tmplt_priv, u32 chain_index,
2660 			      struct net *net, struct sk_buff *skb,
2661 			      struct tcf_block *block,
2662 			      u32 portid, u32 seq, u16 flags, int event)
2663 {
2664 	unsigned char *b = skb_tail_pointer(skb);
2665 	const struct tcf_proto_ops *ops;
2666 	struct nlmsghdr *nlh;
2667 	struct tcmsg *tcm;
2668 	void *priv;
2669 
2670 	ops = tmplt_ops;
2671 	priv = tmplt_priv;
2672 
2673 	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2674 	if (!nlh)
2675 		goto out_nlmsg_trim;
2676 	tcm = nlmsg_data(nlh);
2677 	tcm->tcm_family = AF_UNSPEC;
2678 	tcm->tcm__pad1 = 0;
2679 	tcm->tcm__pad2 = 0;
2680 	tcm->tcm_handle = 0;
2681 	if (block->q) {
2682 		tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2683 		tcm->tcm_parent = block->q->handle;
2684 	} else {
2685 		tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2686 		tcm->tcm_block_index = block->index;
2687 	}
2688 
2689 	if (nla_put_u32(skb, TCA_CHAIN, chain_index))
2690 		goto nla_put_failure;
2691 
2692 	if (ops) {
2693 		if (nla_put_string(skb, TCA_KIND, ops->kind))
2694 			goto nla_put_failure;
2695 		if (ops->tmplt_dump(skb, net, priv) < 0)
2696 			goto nla_put_failure;
2697 	}
2698 
2699 	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2700 	return skb->len;
2701 
2702 out_nlmsg_trim:
2703 nla_put_failure:
2704 	nlmsg_trim(skb, b);
2705 	return -EMSGSIZE;
2706 }
2707 
tc_chain_notify(struct tcf_chain * chain,struct sk_buff * oskb,u32 seq,u16 flags,int event,bool unicast)2708 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2709 			   u32 seq, u16 flags, int event, bool unicast)
2710 {
2711 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2712 	struct tcf_block *block = chain->block;
2713 	struct net *net = block->net;
2714 	struct sk_buff *skb;
2715 	int err = 0;
2716 
2717 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2718 	if (!skb)
2719 		return -ENOBUFS;
2720 
2721 	if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2722 			       chain->index, net, skb, block, portid,
2723 			       seq, flags, event) <= 0) {
2724 		kfree_skb(skb);
2725 		return -EINVAL;
2726 	}
2727 
2728 	if (unicast)
2729 		err = rtnl_unicast(skb, net, portid);
2730 	else
2731 		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2732 				     flags & NLM_F_ECHO);
2733 
2734 	return err;
2735 }
2736 
tc_chain_notify_delete(const struct tcf_proto_ops * tmplt_ops,void * tmplt_priv,u32 chain_index,struct tcf_block * block,struct sk_buff * oskb,u32 seq,u16 flags,bool unicast)2737 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2738 				  void *tmplt_priv, u32 chain_index,
2739 				  struct tcf_block *block, struct sk_buff *oskb,
2740 				  u32 seq, u16 flags, bool unicast)
2741 {
2742 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2743 	struct net *net = block->net;
2744 	struct sk_buff *skb;
2745 
2746 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2747 	if (!skb)
2748 		return -ENOBUFS;
2749 
2750 	if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2751 			       block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2752 		kfree_skb(skb);
2753 		return -EINVAL;
2754 	}
2755 
2756 	if (unicast)
2757 		return rtnl_unicast(skb, net, portid);
2758 
2759 	return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2760 }
2761 
tc_chain_tmplt_add(struct tcf_chain * chain,struct net * net,struct nlattr ** tca,struct netlink_ext_ack * extack)2762 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2763 			      struct nlattr **tca,
2764 			      struct netlink_ext_ack *extack)
2765 {
2766 	const struct tcf_proto_ops *ops;
2767 	char name[IFNAMSIZ];
2768 	void *tmplt_priv;
2769 
2770 	/* If kind is not set, user did not specify template. */
2771 	if (!tca[TCA_KIND])
2772 		return 0;
2773 
2774 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2775 		NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
2776 		return -EINVAL;
2777 	}
2778 
2779 	ops = tcf_proto_lookup_ops(name, true, extack);
2780 	if (IS_ERR(ops))
2781 		return PTR_ERR(ops);
2782 	if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2783 		NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2784 		return -EOPNOTSUPP;
2785 	}
2786 
2787 	tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2788 	if (IS_ERR(tmplt_priv)) {
2789 		module_put(ops->owner);
2790 		return PTR_ERR(tmplt_priv);
2791 	}
2792 	chain->tmplt_ops = ops;
2793 	chain->tmplt_priv = tmplt_priv;
2794 	return 0;
2795 }
2796 
tc_chain_tmplt_del(const struct tcf_proto_ops * tmplt_ops,void * tmplt_priv)2797 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2798 			       void *tmplt_priv)
2799 {
2800 	/* If template ops are set, no work to do for us. */
2801 	if (!tmplt_ops)
2802 		return;
2803 
2804 	tmplt_ops->tmplt_destroy(tmplt_priv);
2805 	module_put(tmplt_ops->owner);
2806 }
2807 
2808 /* Add/delete/get a chain */
2809 
tc_ctl_chain(struct sk_buff * skb,struct nlmsghdr * n,struct netlink_ext_ack * extack)2810 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2811 			struct netlink_ext_ack *extack)
2812 {
2813 	struct net *net = sock_net(skb->sk);
2814 	struct nlattr *tca[TCA_MAX + 1];
2815 	struct tcmsg *t;
2816 	u32 parent;
2817 	u32 chain_index;
2818 	struct Qdisc *q;
2819 	struct tcf_chain *chain;
2820 	struct tcf_block *block;
2821 	unsigned long cl;
2822 	int err;
2823 
2824 replay:
2825 	q = NULL;
2826 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2827 				     rtm_tca_policy, extack);
2828 	if (err < 0)
2829 		return err;
2830 
2831 	t = nlmsg_data(n);
2832 	parent = t->tcm_parent;
2833 	cl = 0;
2834 
2835 	block = tcf_block_find(net, &q, &parent, &cl,
2836 			       t->tcm_ifindex, t->tcm_block_index, extack);
2837 	if (IS_ERR(block))
2838 		return PTR_ERR(block);
2839 
2840 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2841 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2842 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2843 		err = -EINVAL;
2844 		goto errout_block;
2845 	}
2846 
2847 	mutex_lock(&block->lock);
2848 	chain = tcf_chain_lookup(block, chain_index);
2849 	if (n->nlmsg_type == RTM_NEWCHAIN) {
2850 		if (chain) {
2851 			if (tcf_chain_held_by_acts_only(chain)) {
2852 				/* The chain exists only because there is
2853 				 * some action referencing it.
2854 				 */
2855 				tcf_chain_hold(chain);
2856 			} else {
2857 				NL_SET_ERR_MSG(extack, "Filter chain already exists");
2858 				err = -EEXIST;
2859 				goto errout_block_locked;
2860 			}
2861 		} else {
2862 			if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2863 				NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
2864 				err = -ENOENT;
2865 				goto errout_block_locked;
2866 			}
2867 			chain = tcf_chain_create(block, chain_index);
2868 			if (!chain) {
2869 				NL_SET_ERR_MSG(extack, "Failed to create filter chain");
2870 				err = -ENOMEM;
2871 				goto errout_block_locked;
2872 			}
2873 		}
2874 	} else {
2875 		if (!chain || tcf_chain_held_by_acts_only(chain)) {
2876 			NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2877 			err = -EINVAL;
2878 			goto errout_block_locked;
2879 		}
2880 		tcf_chain_hold(chain);
2881 	}
2882 
2883 	if (n->nlmsg_type == RTM_NEWCHAIN) {
2884 		/* Modifying chain requires holding parent block lock. In case
2885 		 * the chain was successfully added, take a reference to the
2886 		 * chain. This ensures that an empty chain does not disappear at
2887 		 * the end of this function.
2888 		 */
2889 		tcf_chain_hold(chain);
2890 		chain->explicitly_created = true;
2891 	}
2892 	mutex_unlock(&block->lock);
2893 
2894 	switch (n->nlmsg_type) {
2895 	case RTM_NEWCHAIN:
2896 		err = tc_chain_tmplt_add(chain, net, tca, extack);
2897 		if (err) {
2898 			tcf_chain_put_explicitly_created(chain);
2899 			goto errout;
2900 		}
2901 
2902 		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2903 				RTM_NEWCHAIN, false);
2904 		break;
2905 	case RTM_DELCHAIN:
2906 		tfilter_notify_chain(net, skb, block, q, parent, n,
2907 				     chain, RTM_DELTFILTER);
2908 		/* Flush the chain first as the user requested chain removal. */
2909 		tcf_chain_flush(chain, true);
2910 		/* In case the chain was successfully deleted, put a reference
2911 		 * to the chain previously taken during addition.
2912 		 */
2913 		tcf_chain_put_explicitly_created(chain);
2914 		break;
2915 	case RTM_GETCHAIN:
2916 		err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2917 				      n->nlmsg_flags, n->nlmsg_type, true);
2918 		if (err < 0)
2919 			NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2920 		break;
2921 	default:
2922 		err = -EOPNOTSUPP;
2923 		NL_SET_ERR_MSG(extack, "Unsupported message type");
2924 		goto errout;
2925 	}
2926 
2927 errout:
2928 	tcf_chain_put(chain);
2929 errout_block:
2930 	tcf_block_release(q, block, true);
2931 	if (err == -EAGAIN)
2932 		/* Replay the request. */
2933 		goto replay;
2934 	return err;
2935 
2936 errout_block_locked:
2937 	mutex_unlock(&block->lock);
2938 	goto errout_block;
2939 }
2940 
2941 /* called with RTNL */
tc_dump_chain(struct sk_buff * skb,struct netlink_callback * cb)2942 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2943 {
2944 	struct net *net = sock_net(skb->sk);
2945 	struct nlattr *tca[TCA_MAX + 1];
2946 	struct Qdisc *q = NULL;
2947 	struct tcf_block *block;
2948 	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2949 	struct tcf_chain *chain;
2950 	long index_start;
2951 	long index;
2952 	int err;
2953 
2954 	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2955 		return skb->len;
2956 
2957 	err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2958 				     rtm_tca_policy, cb->extack);
2959 	if (err)
2960 		return err;
2961 
2962 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2963 		block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2964 		if (!block)
2965 			goto out;
2966 	} else {
2967 		const struct Qdisc_class_ops *cops;
2968 		struct net_device *dev;
2969 		unsigned long cl = 0;
2970 
2971 		dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2972 		if (!dev)
2973 			return skb->len;
2974 
2975 		if (!tcm->tcm_parent)
2976 			q = rtnl_dereference(dev->qdisc);
2977 		else
2978 			q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2979 
2980 		if (!q)
2981 			goto out;
2982 		cops = q->ops->cl_ops;
2983 		if (!cops)
2984 			goto out;
2985 		if (!cops->tcf_block)
2986 			goto out;
2987 		if (TC_H_MIN(tcm->tcm_parent)) {
2988 			cl = cops->find(q, tcm->tcm_parent);
2989 			if (cl == 0)
2990 				goto out;
2991 		}
2992 		block = cops->tcf_block(q, cl, NULL);
2993 		if (!block)
2994 			goto out;
2995 		if (tcf_block_shared(block))
2996 			q = NULL;
2997 	}
2998 
2999 	index_start = cb->args[0];
3000 	index = 0;
3001 
3002 	mutex_lock(&block->lock);
3003 	list_for_each_entry(chain, &block->chain_list, list) {
3004 		if ((tca[TCA_CHAIN] &&
3005 		     nla_get_u32(tca[TCA_CHAIN]) != chain->index))
3006 			continue;
3007 		if (index < index_start) {
3008 			index++;
3009 			continue;
3010 		}
3011 		if (tcf_chain_held_by_acts_only(chain))
3012 			continue;
3013 		err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
3014 					 chain->index, net, skb, block,
3015 					 NETLINK_CB(cb->skb).portid,
3016 					 cb->nlh->nlmsg_seq, NLM_F_MULTI,
3017 					 RTM_NEWCHAIN);
3018 		if (err <= 0)
3019 			break;
3020 		index++;
3021 	}
3022 	mutex_unlock(&block->lock);
3023 
3024 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
3025 		tcf_block_refcnt_put(block, true);
3026 	cb->args[0] = index;
3027 
3028 out:
3029 	/* If we did no progress, the error (EMSGSIZE) is real */
3030 	if (skb->len == 0 && err)
3031 		return err;
3032 	return skb->len;
3033 }
3034 
tcf_exts_destroy(struct tcf_exts * exts)3035 void tcf_exts_destroy(struct tcf_exts *exts)
3036 {
3037 #ifdef CONFIG_NET_CLS_ACT
3038 	if (exts->actions) {
3039 		tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
3040 		kfree(exts->actions);
3041 	}
3042 	exts->nr_actions = 0;
3043 #endif
3044 }
3045 EXPORT_SYMBOL(tcf_exts_destroy);
3046 
tcf_exts_validate_ex(struct net * net,struct tcf_proto * tp,struct nlattr ** tb,struct nlattr * rate_tlv,struct tcf_exts * exts,u32 flags,u32 fl_flags,struct netlink_ext_ack * extack)3047 int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3048 			 struct nlattr *rate_tlv, struct tcf_exts *exts,
3049 			 u32 flags, u32 fl_flags, struct netlink_ext_ack *extack)
3050 {
3051 #ifdef CONFIG_NET_CLS_ACT
3052 	{
3053 		int init_res[TCA_ACT_MAX_PRIO] = {};
3054 		struct tc_action *act;
3055 		size_t attr_size = 0;
3056 
3057 		if (exts->police && tb[exts->police]) {
3058 			struct tc_action_ops *a_o;
3059 
3060 			a_o = tc_action_load_ops(tb[exts->police], true,
3061 						 !(flags & TCA_ACT_FLAGS_NO_RTNL),
3062 						 extack);
3063 			if (IS_ERR(a_o))
3064 				return PTR_ERR(a_o);
3065 			flags |= TCA_ACT_FLAGS_POLICE | TCA_ACT_FLAGS_BIND;
3066 			act = tcf_action_init_1(net, tp, tb[exts->police],
3067 						rate_tlv, a_o, init_res, flags,
3068 						extack);
3069 			module_put(a_o->owner);
3070 			if (IS_ERR(act))
3071 				return PTR_ERR(act);
3072 
3073 			act->type = exts->type = TCA_OLD_COMPAT;
3074 			exts->actions[0] = act;
3075 			exts->nr_actions = 1;
3076 			tcf_idr_insert_many(exts->actions);
3077 		} else if (exts->action && tb[exts->action]) {
3078 			int err;
3079 
3080 			flags |= TCA_ACT_FLAGS_BIND;
3081 			err = tcf_action_init(net, tp, tb[exts->action],
3082 					      rate_tlv, exts->actions, init_res,
3083 					      &attr_size, flags, fl_flags,
3084 					      extack);
3085 			if (err < 0)
3086 				return err;
3087 			exts->nr_actions = err;
3088 		}
3089 	}
3090 #else
3091 	if ((exts->action && tb[exts->action]) ||
3092 	    (exts->police && tb[exts->police])) {
3093 		NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
3094 		return -EOPNOTSUPP;
3095 	}
3096 #endif
3097 
3098 	return 0;
3099 }
3100 EXPORT_SYMBOL(tcf_exts_validate_ex);
3101 
tcf_exts_validate(struct net * net,struct tcf_proto * tp,struct nlattr ** tb,struct nlattr * rate_tlv,struct tcf_exts * exts,u32 flags,struct netlink_ext_ack * extack)3102 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3103 		      struct nlattr *rate_tlv, struct tcf_exts *exts,
3104 		      u32 flags, struct netlink_ext_ack *extack)
3105 {
3106 	return tcf_exts_validate_ex(net, tp, tb, rate_tlv, exts,
3107 				    flags, 0, extack);
3108 }
3109 EXPORT_SYMBOL(tcf_exts_validate);
3110 
tcf_exts_change(struct tcf_exts * dst,struct tcf_exts * src)3111 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
3112 {
3113 #ifdef CONFIG_NET_CLS_ACT
3114 	struct tcf_exts old = *dst;
3115 
3116 	*dst = *src;
3117 	tcf_exts_destroy(&old);
3118 #endif
3119 }
3120 EXPORT_SYMBOL(tcf_exts_change);
3121 
3122 #ifdef CONFIG_NET_CLS_ACT
tcf_exts_first_act(struct tcf_exts * exts)3123 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3124 {
3125 	if (exts->nr_actions == 0)
3126 		return NULL;
3127 	else
3128 		return exts->actions[0];
3129 }
3130 #endif
3131 
tcf_exts_dump(struct sk_buff * skb,struct tcf_exts * exts)3132 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
3133 {
3134 #ifdef CONFIG_NET_CLS_ACT
3135 	struct nlattr *nest;
3136 
3137 	if (exts->action && tcf_exts_has_actions(exts)) {
3138 		/*
3139 		 * again for backward compatible mode - we want
3140 		 * to work with both old and new modes of entering
3141 		 * tc data even if iproute2  was newer - jhs
3142 		 */
3143 		if (exts->type != TCA_OLD_COMPAT) {
3144 			nest = nla_nest_start_noflag(skb, exts->action);
3145 			if (nest == NULL)
3146 				goto nla_put_failure;
3147 
3148 			if (tcf_action_dump(skb, exts->actions, 0, 0, false)
3149 			    < 0)
3150 				goto nla_put_failure;
3151 			nla_nest_end(skb, nest);
3152 		} else if (exts->police) {
3153 			struct tc_action *act = tcf_exts_first_act(exts);
3154 			nest = nla_nest_start_noflag(skb, exts->police);
3155 			if (nest == NULL || !act)
3156 				goto nla_put_failure;
3157 			if (tcf_action_dump_old(skb, act, 0, 0) < 0)
3158 				goto nla_put_failure;
3159 			nla_nest_end(skb, nest);
3160 		}
3161 	}
3162 	return 0;
3163 
3164 nla_put_failure:
3165 	nla_nest_cancel(skb, nest);
3166 	return -1;
3167 #else
3168 	return 0;
3169 #endif
3170 }
3171 EXPORT_SYMBOL(tcf_exts_dump);
3172 
tcf_exts_terse_dump(struct sk_buff * skb,struct tcf_exts * exts)3173 int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts)
3174 {
3175 #ifdef CONFIG_NET_CLS_ACT
3176 	struct nlattr *nest;
3177 
3178 	if (!exts->action || !tcf_exts_has_actions(exts))
3179 		return 0;
3180 
3181 	nest = nla_nest_start_noflag(skb, exts->action);
3182 	if (!nest)
3183 		goto nla_put_failure;
3184 
3185 	if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0)
3186 		goto nla_put_failure;
3187 	nla_nest_end(skb, nest);
3188 	return 0;
3189 
3190 nla_put_failure:
3191 	nla_nest_cancel(skb, nest);
3192 	return -1;
3193 #else
3194 	return 0;
3195 #endif
3196 }
3197 EXPORT_SYMBOL(tcf_exts_terse_dump);
3198 
tcf_exts_dump_stats(struct sk_buff * skb,struct tcf_exts * exts)3199 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
3200 {
3201 #ifdef CONFIG_NET_CLS_ACT
3202 	struct tc_action *a = tcf_exts_first_act(exts);
3203 	if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
3204 		return -1;
3205 #endif
3206 	return 0;
3207 }
3208 EXPORT_SYMBOL(tcf_exts_dump_stats);
3209 
tcf_block_offload_inc(struct tcf_block * block,u32 * flags)3210 static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
3211 {
3212 	if (*flags & TCA_CLS_FLAGS_IN_HW)
3213 		return;
3214 	*flags |= TCA_CLS_FLAGS_IN_HW;
3215 	atomic_inc(&block->offloadcnt);
3216 }
3217 
tcf_block_offload_dec(struct tcf_block * block,u32 * flags)3218 static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
3219 {
3220 	if (!(*flags & TCA_CLS_FLAGS_IN_HW))
3221 		return;
3222 	*flags &= ~TCA_CLS_FLAGS_IN_HW;
3223 	atomic_dec(&block->offloadcnt);
3224 }
3225 
tc_cls_offload_cnt_update(struct tcf_block * block,struct tcf_proto * tp,u32 * cnt,u32 * flags,u32 diff,bool add)3226 static void tc_cls_offload_cnt_update(struct tcf_block *block,
3227 				      struct tcf_proto *tp, u32 *cnt,
3228 				      u32 *flags, u32 diff, bool add)
3229 {
3230 	lockdep_assert_held(&block->cb_lock);
3231 
3232 	spin_lock(&tp->lock);
3233 	if (add) {
3234 		if (!*cnt)
3235 			tcf_block_offload_inc(block, flags);
3236 		*cnt += diff;
3237 	} else {
3238 		*cnt -= diff;
3239 		if (!*cnt)
3240 			tcf_block_offload_dec(block, flags);
3241 	}
3242 	spin_unlock(&tp->lock);
3243 }
3244 
3245 static void
tc_cls_offload_cnt_reset(struct tcf_block * block,struct tcf_proto * tp,u32 * cnt,u32 * flags)3246 tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3247 			 u32 *cnt, u32 *flags)
3248 {
3249 	lockdep_assert_held(&block->cb_lock);
3250 
3251 	spin_lock(&tp->lock);
3252 	tcf_block_offload_dec(block, flags);
3253 	*cnt = 0;
3254 	spin_unlock(&tp->lock);
3255 }
3256 
3257 static int
__tc_setup_cb_call(struct tcf_block * block,enum tc_setup_type type,void * type_data,bool err_stop)3258 __tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3259 		   void *type_data, bool err_stop)
3260 {
3261 	struct flow_block_cb *block_cb;
3262 	int ok_count = 0;
3263 	int err;
3264 
3265 	list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
3266 		err = block_cb->cb(type, type_data, block_cb->cb_priv);
3267 		if (err) {
3268 			if (err_stop)
3269 				return err;
3270 		} else {
3271 			ok_count++;
3272 		}
3273 	}
3274 	return ok_count;
3275 }
3276 
tc_setup_cb_call(struct tcf_block * block,enum tc_setup_type type,void * type_data,bool err_stop,bool rtnl_held)3277 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3278 		     void *type_data, bool err_stop, bool rtnl_held)
3279 {
3280 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3281 	int ok_count;
3282 
3283 retry:
3284 	if (take_rtnl)
3285 		rtnl_lock();
3286 	down_read(&block->cb_lock);
3287 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3288 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3289 	 * obtain the locks in same order here.
3290 	 */
3291 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3292 		up_read(&block->cb_lock);
3293 		take_rtnl = true;
3294 		goto retry;
3295 	}
3296 
3297 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3298 
3299 	up_read(&block->cb_lock);
3300 	if (take_rtnl)
3301 		rtnl_unlock();
3302 	return ok_count;
3303 }
3304 EXPORT_SYMBOL(tc_setup_cb_call);
3305 
3306 /* Non-destructive filter add. If filter that wasn't already in hardware is
3307  * successfully offloaded, increment block offloads counter. On failure,
3308  * previously offloaded filter is considered to be intact and offloads counter
3309  * is not decremented.
3310  */
3311 
tc_setup_cb_add(struct tcf_block * block,struct tcf_proto * tp,enum tc_setup_type type,void * type_data,bool err_stop,u32 * flags,unsigned int * in_hw_count,bool rtnl_held)3312 int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3313 		    enum tc_setup_type type, void *type_data, bool err_stop,
3314 		    u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3315 {
3316 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3317 	int ok_count;
3318 
3319 retry:
3320 	if (take_rtnl)
3321 		rtnl_lock();
3322 	down_read(&block->cb_lock);
3323 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3324 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3325 	 * obtain the locks in same order here.
3326 	 */
3327 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3328 		up_read(&block->cb_lock);
3329 		take_rtnl = true;
3330 		goto retry;
3331 	}
3332 
3333 	/* Make sure all netdevs sharing this block are offload-capable. */
3334 	if (block->nooffloaddevcnt && err_stop) {
3335 		ok_count = -EOPNOTSUPP;
3336 		goto err_unlock;
3337 	}
3338 
3339 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3340 	if (ok_count < 0)
3341 		goto err_unlock;
3342 
3343 	if (tp->ops->hw_add)
3344 		tp->ops->hw_add(tp, type_data);
3345 	if (ok_count > 0)
3346 		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
3347 					  ok_count, true);
3348 err_unlock:
3349 	up_read(&block->cb_lock);
3350 	if (take_rtnl)
3351 		rtnl_unlock();
3352 	return min(ok_count, 0);
3353 }
3354 EXPORT_SYMBOL(tc_setup_cb_add);
3355 
3356 /* Destructive filter replace. If filter that wasn't already in hardware is
3357  * successfully offloaded, increment block offload counter. On failure,
3358  * previously offloaded filter is considered to be destroyed and offload counter
3359  * is decremented.
3360  */
3361 
tc_setup_cb_replace(struct tcf_block * block,struct tcf_proto * tp,enum tc_setup_type type,void * type_data,bool err_stop,u32 * old_flags,unsigned int * old_in_hw_count,u32 * new_flags,unsigned int * new_in_hw_count,bool rtnl_held)3362 int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3363 			enum tc_setup_type type, void *type_data, bool err_stop,
3364 			u32 *old_flags, unsigned int *old_in_hw_count,
3365 			u32 *new_flags, unsigned int *new_in_hw_count,
3366 			bool rtnl_held)
3367 {
3368 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3369 	int ok_count;
3370 
3371 retry:
3372 	if (take_rtnl)
3373 		rtnl_lock();
3374 	down_read(&block->cb_lock);
3375 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3376 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3377 	 * obtain the locks in same order here.
3378 	 */
3379 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3380 		up_read(&block->cb_lock);
3381 		take_rtnl = true;
3382 		goto retry;
3383 	}
3384 
3385 	/* Make sure all netdevs sharing this block are offload-capable. */
3386 	if (block->nooffloaddevcnt && err_stop) {
3387 		ok_count = -EOPNOTSUPP;
3388 		goto err_unlock;
3389 	}
3390 
3391 	tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
3392 	if (tp->ops->hw_del)
3393 		tp->ops->hw_del(tp, type_data);
3394 
3395 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3396 	if (ok_count < 0)
3397 		goto err_unlock;
3398 
3399 	if (tp->ops->hw_add)
3400 		tp->ops->hw_add(tp, type_data);
3401 	if (ok_count > 0)
3402 		tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
3403 					  new_flags, ok_count, true);
3404 err_unlock:
3405 	up_read(&block->cb_lock);
3406 	if (take_rtnl)
3407 		rtnl_unlock();
3408 	return min(ok_count, 0);
3409 }
3410 EXPORT_SYMBOL(tc_setup_cb_replace);
3411 
3412 /* Destroy filter and decrement block offload counter, if filter was previously
3413  * offloaded.
3414  */
3415 
tc_setup_cb_destroy(struct tcf_block * block,struct tcf_proto * tp,enum tc_setup_type type,void * type_data,bool err_stop,u32 * flags,unsigned int * in_hw_count,bool rtnl_held)3416 int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3417 			enum tc_setup_type type, void *type_data, bool err_stop,
3418 			u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3419 {
3420 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3421 	int ok_count;
3422 
3423 retry:
3424 	if (take_rtnl)
3425 		rtnl_lock();
3426 	down_read(&block->cb_lock);
3427 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3428 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3429 	 * obtain the locks in same order here.
3430 	 */
3431 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3432 		up_read(&block->cb_lock);
3433 		take_rtnl = true;
3434 		goto retry;
3435 	}
3436 
3437 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3438 
3439 	tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
3440 	if (tp->ops->hw_del)
3441 		tp->ops->hw_del(tp, type_data);
3442 
3443 	up_read(&block->cb_lock);
3444 	if (take_rtnl)
3445 		rtnl_unlock();
3446 	return min(ok_count, 0);
3447 }
3448 EXPORT_SYMBOL(tc_setup_cb_destroy);
3449 
tc_setup_cb_reoffload(struct tcf_block * block,struct tcf_proto * tp,bool add,flow_setup_cb_t * cb,enum tc_setup_type type,void * type_data,void * cb_priv,u32 * flags,unsigned int * in_hw_count)3450 int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3451 			  bool add, flow_setup_cb_t *cb,
3452 			  enum tc_setup_type type, void *type_data,
3453 			  void *cb_priv, u32 *flags, unsigned int *in_hw_count)
3454 {
3455 	int err = cb(type, type_data, cb_priv);
3456 
3457 	if (err) {
3458 		if (add && tc_skip_sw(*flags))
3459 			return err;
3460 	} else {
3461 		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
3462 					  add);
3463 	}
3464 
3465 	return 0;
3466 }
3467 EXPORT_SYMBOL(tc_setup_cb_reoffload);
3468 
tcf_act_get_cookie(struct flow_action_entry * entry,const struct tc_action * act)3469 static int tcf_act_get_cookie(struct flow_action_entry *entry,
3470 			      const struct tc_action *act)
3471 {
3472 	struct tc_cookie *cookie;
3473 	int err = 0;
3474 
3475 	rcu_read_lock();
3476 	cookie = rcu_dereference(act->act_cookie);
3477 	if (cookie) {
3478 		entry->cookie = flow_action_cookie_create(cookie->data,
3479 							  cookie->len,
3480 							  GFP_ATOMIC);
3481 		if (!entry->cookie)
3482 			err = -ENOMEM;
3483 	}
3484 	rcu_read_unlock();
3485 	return err;
3486 }
3487 
tcf_act_put_cookie(struct flow_action_entry * entry)3488 static void tcf_act_put_cookie(struct flow_action_entry *entry)
3489 {
3490 	flow_action_cookie_destroy(entry->cookie);
3491 }
3492 
tc_cleanup_offload_action(struct flow_action * flow_action)3493 void tc_cleanup_offload_action(struct flow_action *flow_action)
3494 {
3495 	struct flow_action_entry *entry;
3496 	int i;
3497 
3498 	flow_action_for_each(i, entry, flow_action) {
3499 		tcf_act_put_cookie(entry);
3500 		if (entry->destructor)
3501 			entry->destructor(entry->destructor_priv);
3502 	}
3503 }
3504 EXPORT_SYMBOL(tc_cleanup_offload_action);
3505 
tc_setup_offload_act(struct tc_action * act,struct flow_action_entry * entry,u32 * index_inc,struct netlink_ext_ack * extack)3506 static int tc_setup_offload_act(struct tc_action *act,
3507 				struct flow_action_entry *entry,
3508 				u32 *index_inc,
3509 				struct netlink_ext_ack *extack)
3510 {
3511 #ifdef CONFIG_NET_CLS_ACT
3512 	if (act->ops->offload_act_setup) {
3513 		return act->ops->offload_act_setup(act, entry, index_inc, true,
3514 						   extack);
3515 	} else {
3516 		NL_SET_ERR_MSG(extack, "Action does not support offload");
3517 		return -EOPNOTSUPP;
3518 	}
3519 #else
3520 	return 0;
3521 #endif
3522 }
3523 
tc_setup_action(struct flow_action * flow_action,struct tc_action * actions[],struct netlink_ext_ack * extack)3524 int tc_setup_action(struct flow_action *flow_action,
3525 		    struct tc_action *actions[],
3526 		    struct netlink_ext_ack *extack)
3527 {
3528 	int i, j, k, index, err = 0;
3529 	struct tc_action *act;
3530 
3531 	BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY);
3532 	BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE);
3533 	BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED);
3534 
3535 	if (!actions)
3536 		return 0;
3537 
3538 	j = 0;
3539 	tcf_act_for_each_action(i, act, actions) {
3540 		struct flow_action_entry *entry;
3541 
3542 		entry = &flow_action->entries[j];
3543 		spin_lock_bh(&act->tcfa_lock);
3544 		err = tcf_act_get_cookie(entry, act);
3545 		if (err)
3546 			goto err_out_locked;
3547 
3548 		index = 0;
3549 		err = tc_setup_offload_act(act, entry, &index, extack);
3550 		if (err)
3551 			goto err_out_locked;
3552 
3553 		for (k = 0; k < index ; k++) {
3554 			entry[k].hw_stats = tc_act_hw_stats(act->hw_stats);
3555 			entry[k].hw_index = act->tcfa_index;
3556 		}
3557 
3558 		j += index;
3559 
3560 		spin_unlock_bh(&act->tcfa_lock);
3561 	}
3562 
3563 err_out:
3564 	if (err)
3565 		tc_cleanup_offload_action(flow_action);
3566 
3567 	return err;
3568 err_out_locked:
3569 	spin_unlock_bh(&act->tcfa_lock);
3570 	goto err_out;
3571 }
3572 
tc_setup_offload_action(struct flow_action * flow_action,const struct tcf_exts * exts,struct netlink_ext_ack * extack)3573 int tc_setup_offload_action(struct flow_action *flow_action,
3574 			    const struct tcf_exts *exts,
3575 			    struct netlink_ext_ack *extack)
3576 {
3577 #ifdef CONFIG_NET_CLS_ACT
3578 	if (!exts)
3579 		return 0;
3580 
3581 	return tc_setup_action(flow_action, exts->actions, extack);
3582 #else
3583 	return 0;
3584 #endif
3585 }
3586 EXPORT_SYMBOL(tc_setup_offload_action);
3587 
tcf_exts_num_actions(struct tcf_exts * exts)3588 unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3589 {
3590 	unsigned int num_acts = 0;
3591 	struct tc_action *act;
3592 	int i;
3593 
3594 	tcf_exts_for_each_action(i, act, exts) {
3595 		if (is_tcf_pedit(act))
3596 			num_acts += tcf_pedit_nkeys(act);
3597 		else
3598 			num_acts++;
3599 	}
3600 	return num_acts;
3601 }
3602 EXPORT_SYMBOL(tcf_exts_num_actions);
3603 
3604 #ifdef CONFIG_NET_CLS_ACT
tcf_qevent_parse_block_index(struct nlattr * block_index_attr,u32 * p_block_index,struct netlink_ext_ack * extack)3605 static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
3606 					u32 *p_block_index,
3607 					struct netlink_ext_ack *extack)
3608 {
3609 	*p_block_index = nla_get_u32(block_index_attr);
3610 	if (!*p_block_index) {
3611 		NL_SET_ERR_MSG(extack, "Block number may not be zero");
3612 		return -EINVAL;
3613 	}
3614 
3615 	return 0;
3616 }
3617 
tcf_qevent_init(struct tcf_qevent * qe,struct Qdisc * sch,enum flow_block_binder_type binder_type,struct nlattr * block_index_attr,struct netlink_ext_ack * extack)3618 int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
3619 		    enum flow_block_binder_type binder_type,
3620 		    struct nlattr *block_index_attr,
3621 		    struct netlink_ext_ack *extack)
3622 {
3623 	u32 block_index;
3624 	int err;
3625 
3626 	if (!block_index_attr)
3627 		return 0;
3628 
3629 	err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
3630 	if (err)
3631 		return err;
3632 
3633 	qe->info.binder_type = binder_type;
3634 	qe->info.chain_head_change = tcf_chain_head_change_dflt;
3635 	qe->info.chain_head_change_priv = &qe->filter_chain;
3636 	qe->info.block_index = block_index;
3637 
3638 	return tcf_block_get_ext(&qe->block, sch, &qe->info, extack);
3639 }
3640 EXPORT_SYMBOL(tcf_qevent_init);
3641 
tcf_qevent_destroy(struct tcf_qevent * qe,struct Qdisc * sch)3642 void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
3643 {
3644 	if (qe->info.block_index)
3645 		tcf_block_put_ext(qe->block, sch, &qe->info);
3646 }
3647 EXPORT_SYMBOL(tcf_qevent_destroy);
3648 
tcf_qevent_validate_change(struct tcf_qevent * qe,struct nlattr * block_index_attr,struct netlink_ext_ack * extack)3649 int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
3650 			       struct netlink_ext_ack *extack)
3651 {
3652 	u32 block_index;
3653 	int err;
3654 
3655 	if (!block_index_attr)
3656 		return 0;
3657 
3658 	err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
3659 	if (err)
3660 		return err;
3661 
3662 	/* Bounce newly-configured block or change in block. */
3663 	if (block_index != qe->info.block_index) {
3664 		NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
3665 		return -EINVAL;
3666 	}
3667 
3668 	return 0;
3669 }
3670 EXPORT_SYMBOL(tcf_qevent_validate_change);
3671 
tcf_qevent_handle(struct tcf_qevent * qe,struct Qdisc * sch,struct sk_buff * skb,struct sk_buff ** to_free,int * ret)3672 struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
3673 				  struct sk_buff **to_free, int *ret)
3674 {
3675 	struct tcf_result cl_res;
3676 	struct tcf_proto *fl;
3677 
3678 	if (!qe->info.block_index)
3679 		return skb;
3680 
3681 	fl = rcu_dereference_bh(qe->filter_chain);
3682 
3683 	switch (tcf_classify(skb, NULL, fl, &cl_res, false)) {
3684 	case TC_ACT_SHOT:
3685 		qdisc_qstats_drop(sch);
3686 		__qdisc_drop(skb, to_free);
3687 		*ret = __NET_XMIT_BYPASS;
3688 		return NULL;
3689 	case TC_ACT_STOLEN:
3690 	case TC_ACT_QUEUED:
3691 	case TC_ACT_TRAP:
3692 		__qdisc_drop(skb, to_free);
3693 		*ret = __NET_XMIT_STOLEN;
3694 		return NULL;
3695 	case TC_ACT_REDIRECT:
3696 		skb_do_redirect(skb);
3697 		*ret = __NET_XMIT_STOLEN;
3698 		return NULL;
3699 	}
3700 
3701 	return skb;
3702 }
3703 EXPORT_SYMBOL(tcf_qevent_handle);
3704 
tcf_qevent_dump(struct sk_buff * skb,int attr_name,struct tcf_qevent * qe)3705 int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
3706 {
3707 	if (!qe->info.block_index)
3708 		return 0;
3709 	return nla_put_u32(skb, attr_name, qe->info.block_index);
3710 }
3711 EXPORT_SYMBOL(tcf_qevent_dump);
3712 #endif
3713 
tcf_net_init(struct net * net)3714 static __net_init int tcf_net_init(struct net *net)
3715 {
3716 	struct tcf_net *tn = net_generic(net, tcf_net_id);
3717 
3718 	spin_lock_init(&tn->idr_lock);
3719 	idr_init(&tn->idr);
3720 	return 0;
3721 }
3722 
tcf_net_exit(struct net * net)3723 static void __net_exit tcf_net_exit(struct net *net)
3724 {
3725 	struct tcf_net *tn = net_generic(net, tcf_net_id);
3726 
3727 	idr_destroy(&tn->idr);
3728 }
3729 
3730 static struct pernet_operations tcf_net_ops = {
3731 	.init = tcf_net_init,
3732 	.exit = tcf_net_exit,
3733 	.id   = &tcf_net_id,
3734 	.size = sizeof(struct tcf_net),
3735 };
3736 
tc_filter_init(void)3737 static int __init tc_filter_init(void)
3738 {
3739 	int err;
3740 
3741 	tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
3742 	if (!tc_filter_wq)
3743 		return -ENOMEM;
3744 
3745 	err = register_pernet_subsys(&tcf_net_ops);
3746 	if (err)
3747 		goto err_register_pernet_subsys;
3748 
3749 	rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
3750 		      RTNL_FLAG_DOIT_UNLOCKED);
3751 	rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
3752 		      RTNL_FLAG_DOIT_UNLOCKED);
3753 	rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
3754 		      tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
3755 	rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
3756 	rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
3757 	rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
3758 		      tc_dump_chain, 0);
3759 
3760 	return 0;
3761 
3762 err_register_pernet_subsys:
3763 	destroy_workqueue(tc_filter_wq);
3764 	return err;
3765 }
3766 
3767 subsys_initcall(tc_filter_init);
3768