169 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
170 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
527 	struct xfrm_policy *pol;  in xfrm_dst_hash_transfer()  local
533 	hlist_for_each_entry_safe(pol, tmp, list, bydst) {  in xfrm_dst_hash_transfer()
536 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);  in xfrm_dst_hash_transfer()
537 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,  in xfrm_dst_hash_transfer()
538 				pol->family, nhashmask, dbits, sbits);  in xfrm_dst_hash_transfer()
539 		if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_dst_hash_transfer()
540 			hlist_del_rcu(&pol->bydst);  in xfrm_dst_hash_transfer()
541 			hlist_add_head_rcu(&pol->bydst, ndsttable + h);  in xfrm_dst_hash_transfer()
546 			hlist_del_rcu(&pol->bydst);  in xfrm_dst_hash_transfer()
547 			hlist_add_behind_rcu(&pol->bydst, entry0);  in xfrm_dst_hash_transfer()
549 		entry0 = &pol->bydst;  in xfrm_dst_hash_transfer()
562 	struct xfrm_policy *pol;  in xfrm_idx_hash_transfer()  local
564 	hlist_for_each_entry_safe(pol, tmp, list, byidx) {  in xfrm_idx_hash_transfer()
567 		h = __idx_hash(pol->index, nhashmask);  in xfrm_idx_hash_transfer()
568 		hlist_add_head(&pol->byidx, nidxtable+h);  in xfrm_idx_hash_transfer()
697 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)  in xfrm_policy_inexact_alloc_bin()  argument
701 		.family = pol->family,  in xfrm_policy_inexact_alloc_bin()
702 		.type = pol->type,  in xfrm_policy_inexact_alloc_bin()
704 		.if_id = pol->if_id,  in xfrm_policy_inexact_alloc_bin()
706 	struct net *net = xp_net(pol);  in xfrm_policy_inexact_alloc_bin()
1227 	struct xfrm_policy *pol;  in xfrm_hash_rebuild()  local
1348 		hlist_for_each_entry(pol, chain, bydst) {  in xfrm_hash_rebuild()
1349 			if (policy->priority >= pol->priority)  in xfrm_hash_rebuild()
1350 				newpos = &pol->bydst;  in xfrm_hash_rebuild()
1450 					  struct xfrm_policy *pol)  in xfrm_policy_mark_match()  argument
1452 	return mark->v == pol->mark.v && mark->m == pol->mark.m;  in xfrm_policy_mark_match()
1507 	struct xfrm_policy *pol, *delpol = NULL;  in xfrm_policy_insert_inexact_list()  local
1511 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_policy_insert_inexact_list()
1512 		if (pol->type == policy->type &&  in xfrm_policy_insert_inexact_list()
1513 		    pol->if_id == policy->if_id &&  in xfrm_policy_insert_inexact_list()
1514 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert_inexact_list()
1515 		    xfrm_policy_mark_match(&policy->mark, pol) &&  in xfrm_policy_insert_inexact_list()
1516 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert_inexact_list()
1518 			delpol = pol;  in xfrm_policy_insert_inexact_list()
1519 			if (policy->priority > pol->priority)  in xfrm_policy_insert_inexact_list()
1521 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert_inexact_list()
1522 			newpos = &pol->bydst_inexact_list;  in xfrm_policy_insert_inexact_list()
1534 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_policy_insert_inexact_list()
1535 		pol->pos = i;  in xfrm_policy_insert_inexact_list()
1544 	struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;  in xfrm_policy_insert_list()  local
1546 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_insert_list()
1547 		if (pol->type == policy->type &&  in xfrm_policy_insert_list()
1548 		    pol->if_id == policy->if_id &&  in xfrm_policy_insert_list()
1549 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert_list()
1550 		    xfrm_policy_mark_match(&policy->mark, pol) &&  in xfrm_policy_insert_list()
1551 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert_list()
1555 			delpol = pol;  in xfrm_policy_insert_list()
1556 			if (policy->priority > pol->priority)  in xfrm_policy_insert_list()
1558 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert_list()
1559 			newpos = pol;  in xfrm_policy_insert_list()
1629 	struct xfrm_policy *pol;  in __xfrm_policy_bysel_ctx()  local
1634 	hlist_for_each_entry(pol, chain, bydst) {  in __xfrm_policy_bysel_ctx()
1635 		if (pol->type == type &&  in __xfrm_policy_bysel_ctx()
1636 		    pol->if_id == if_id &&  in __xfrm_policy_bysel_ctx()
1637 		    xfrm_policy_mark_match(mark, pol) &&  in __xfrm_policy_bysel_ctx()
1638 		    !selector_cmp(sel, &pol->selector) &&  in __xfrm_policy_bysel_ctx()
1639 		    xfrm_sec_ctx_match(ctx, pol->security))  in __xfrm_policy_bysel_ctx()
1640 			return pol;  in __xfrm_policy_bysel_ctx()
1652 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_policy_bysel_ctx()  local
1676 		pol = NULL;  in xfrm_policy_bysel_ctx()
1686 			if (!pol || tmp->pos < pol->pos)  in xfrm_policy_bysel_ctx()
1687 				pol = tmp;  in xfrm_policy_bysel_ctx()
1690 		pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,  in xfrm_policy_bysel_ctx()
1694 	if (pol) {  in xfrm_policy_bysel_ctx()
1695 		xfrm_pol_hold(pol);  in xfrm_policy_bysel_ctx()
1697 			*err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_bysel_ctx()
1700 				return pol;  in xfrm_policy_bysel_ctx()
1702 			__xfrm_policy_unlink(pol, dir);  in xfrm_policy_bysel_ctx()
1704 		ret = pol;  in xfrm_policy_bysel_ctx()
1720 	struct xfrm_policy *pol, *ret;  in xfrm_policy_byid()  local
1731 	hlist_for_each_entry(pol, chain, byidx) {  in xfrm_policy_byid()
1732 		if (pol->type == type && pol->index == id &&  in xfrm_policy_byid()
1733 		    pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {  in xfrm_policy_byid()
1734 			xfrm_pol_hold(pol);  in xfrm_policy_byid()
1737 								pol->security);  in xfrm_policy_byid()
1740 					return pol;  in xfrm_policy_byid()
1742 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_byid()
1744 			ret = pol;  in xfrm_policy_byid()
1760 	struct xfrm_policy *pol;  in xfrm_policy_flush_secctx_check()  local
1763 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush_secctx_check()
1764 		if (pol->walk.dead ||  in xfrm_policy_flush_secctx_check()
1765 		    xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||  in xfrm_policy_flush_secctx_check()
1766 		    pol->type != type)  in xfrm_policy_flush_secctx_check()
1769 		err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_flush_secctx_check()
1771 			xfrm_audit_policy_delete(pol, 0, task_valid);  in xfrm_policy_flush_secctx_check()
1782 	struct xfrm_policy *pol;  in xfrm_dev_policy_flush_secctx_check()  local
1785 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_dev_policy_flush_secctx_check()
1786 		if (pol->walk.dead ||  in xfrm_dev_policy_flush_secctx_check()
1787 		    xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||  in xfrm_dev_policy_flush_secctx_check()
1788 		    pol->xdo.dev != dev)  in xfrm_dev_policy_flush_secctx_check()
1791 		err = security_xfrm_policy_delete(pol->security);  in xfrm_dev_policy_flush_secctx_check()
1793 			xfrm_audit_policy_delete(pol, 0, task_valid);  in xfrm_dev_policy_flush_secctx_check()
1817 	struct xfrm_policy *pol;  in xfrm_policy_flush()  local
1826 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush()
1827 		if (pol->walk.dead)  in xfrm_policy_flush()
1830 		dir = xfrm_policy_id2dir(pol->index);  in xfrm_policy_flush()
1832 		    pol->type != type)  in xfrm_policy_flush()
1835 		__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
1837 		xfrm_dev_policy_delete(pol);  in xfrm_policy_flush()
1839 		xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_policy_flush()
1840 		xfrm_policy_kill(pol);  in xfrm_policy_flush()
1858 	struct xfrm_policy *pol;  in xfrm_dev_policy_flush()  local
1867 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_dev_policy_flush()
1868 		if (pol->walk.dead)  in xfrm_dev_policy_flush()
1871 		dir = xfrm_policy_id2dir(pol->index);  in xfrm_dev_policy_flush()
1873 		    pol->xdo.dev != dev)  in xfrm_dev_policy_flush()
1876 		__xfrm_policy_unlink(pol, dir);  in xfrm_dev_policy_flush()
1878 		xfrm_dev_policy_delete(pol);  in xfrm_dev_policy_flush()
1880 		xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_dev_policy_flush()
1881 		xfrm_policy_kill(pol);  in xfrm_dev_policy_flush()
1899 	struct xfrm_policy *pol;  in xfrm_policy_walk()  local
1920 		pol = container_of(x, struct xfrm_policy, walk);  in xfrm_policy_walk()
1922 		    walk->type != pol->type)  in xfrm_policy_walk()
1924 		error = func(pol, xfrm_policy_id2dir(pol->index),  in xfrm_policy_walk()
1968 static int xfrm_policy_match(const struct xfrm_policy *pol,  in xfrm_policy_match()  argument
1972 	const struct xfrm_selector *sel = &pol->selector;  in xfrm_policy_match()
1976 	if (pol->family != family ||  in xfrm_policy_match()
1977 	    pol->if_id != if_id ||  in xfrm_policy_match()
1978 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||  in xfrm_policy_match()
1979 	    pol->type != type)  in xfrm_policy_match()
1984 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid);  in xfrm_policy_match()
2098 	struct xfrm_policy *pol;  in __xfrm_policy_eval_candidates()  local
2103 	hlist_for_each_entry_rcu(pol, chain, bydst) {  in __xfrm_policy_eval_candidates()
2106 		if (pol->priority > priority)  in __xfrm_policy_eval_candidates()
2109 		err = xfrm_policy_match(pol, fl, type, family, if_id);  in __xfrm_policy_eval_candidates()
2119 			if (pol->priority == priority &&  in __xfrm_policy_eval_candidates()
2120 			    prefer->pos < pol->pos)  in __xfrm_policy_eval_candidates()
2124 		return pol;  in __xfrm_policy_eval_candidates()
2162 	struct xfrm_policy *pol, *ret;  in xfrm_policy_lookup_bytype()  local
2180 	hlist_for_each_entry_rcu(pol, chain, bydst) {  in xfrm_policy_lookup_bytype()
2181 		err = xfrm_policy_match(pol, fl, type, family, if_id);  in xfrm_policy_lookup_bytype()
2190 			ret = pol;  in xfrm_policy_lookup_bytype()
2202 	pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,  in xfrm_policy_lookup_bytype()
2204 	if (pol) {  in xfrm_policy_lookup_bytype()
2205 		ret = pol;  in xfrm_policy_lookup_bytype()
2206 		if (IS_ERR(pol))  in xfrm_policy_lookup_bytype()
2227 	struct xfrm_policy *pol;  in xfrm_policy_lookup()  local
2229 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,  in xfrm_policy_lookup()
2231 	if (pol != NULL)  in xfrm_policy_lookup()
2232 		return pol;  in xfrm_policy_lookup()
2242 	struct xfrm_policy *pol;  in xfrm_sk_policy_lookup()  local
2246 	pol = rcu_dereference(sk->sk_policy[dir]);  in xfrm_sk_policy_lookup()
2247 	if (pol != NULL) {  in xfrm_sk_policy_lookup()
2251 		if (pol->family != family) {  in xfrm_sk_policy_lookup()
2252 			pol = NULL;  in xfrm_sk_policy_lookup()
2256 		match = xfrm_selector_match(&pol->selector, fl, family);  in xfrm_sk_policy_lookup()
2258 			if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v ||  in xfrm_sk_policy_lookup()
2259 			    pol->if_id != if_id) {  in xfrm_sk_policy_lookup()
2260 				pol = NULL;  in xfrm_sk_policy_lookup()
2263 			err = security_xfrm_policy_lookup(pol->security,  in xfrm_sk_policy_lookup()
2266 				if (!xfrm_pol_hold_rcu(pol))  in xfrm_sk_policy_lookup()
2269 				pol = NULL;  in xfrm_sk_policy_lookup()
2271 				pol = ERR_PTR(err);  in xfrm_sk_policy_lookup()
2274 			pol = NULL;  in xfrm_sk_policy_lookup()
2278 	return pol;  in xfrm_sk_policy_lookup()
2281 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)  in __xfrm_policy_link()  argument
2283 	struct net *net = xp_net(pol);  in __xfrm_policy_link()
2285 	list_add(&pol->walk.all, &net->xfrm.policy_all);  in __xfrm_policy_link()
2287 	xfrm_pol_hold(pol);  in __xfrm_policy_link()
2290 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,  in __xfrm_policy_unlink()  argument
2293 	struct net *net = xp_net(pol);  in __xfrm_policy_unlink()
2295 	if (list_empty(&pol->walk.all))  in __xfrm_policy_unlink()
2299 	if (!hlist_unhashed(&pol->bydst)) {  in __xfrm_policy_unlink()
2300 		hlist_del_rcu(&pol->bydst);  in __xfrm_policy_unlink()
2301 		hlist_del_init(&pol->bydst_inexact_list);  in __xfrm_policy_unlink()
2302 		hlist_del(&pol->byidx);  in __xfrm_policy_unlink()
2305 	list_del_init(&pol->walk.all);  in __xfrm_policy_unlink()
2308 	return pol;  in __xfrm_policy_unlink()
2311 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_link()  argument
2313 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_link()
2316 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_unlink()  argument
2318 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_unlink()
2321 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)  in xfrm_policy_delete()  argument
2323 	struct net *net = xp_net(pol);  in xfrm_policy_delete()
2326 	pol = __xfrm_policy_unlink(pol, dir);  in xfrm_policy_delete()
2328 	if (pol) {  in xfrm_policy_delete()
2329 		xfrm_dev_policy_delete(pol);  in xfrm_policy_delete()
2330 		xfrm_policy_kill(pol);  in xfrm_policy_delete()
2337 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)  in xfrm_sk_policy_insert()  argument
2343 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)  in xfrm_sk_policy_insert()
2350 	if (pol) {  in xfrm_sk_policy_insert()
2351 		pol->curlft.add_time = ktime_get_real_seconds();  in xfrm_sk_policy_insert()
2352 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);  in xfrm_sk_policy_insert()
2353 		xfrm_sk_policy_link(pol, dir);  in xfrm_sk_policy_insert()
2355 	rcu_assign_pointer(sk->sk_policy[dir], pol);  in xfrm_sk_policy_insert()
2357 		if (pol)  in xfrm_sk_policy_insert()
2358 			xfrm_policy_requeue(old_pol, pol);  in xfrm_sk_policy_insert()
2842 	struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);  in xfrm_policy_queue_process()  local
2843 	struct net *net = xp_net(pol);  in xfrm_policy_queue_process()
2844 	struct xfrm_policy_queue *pq = &pol->polq;  in xfrm_policy_queue_process()
2860 	skb->mark = pol->mark.v;  in xfrm_policy_queue_process()
2878 			xfrm_pol_hold(pol);  in xfrm_policy_queue_process()
2896 		skb->mark = pol->mark.v;  in xfrm_policy_queue_process()
2915 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
2921 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
2929 	struct xfrm_policy *pol = xdst->pols[0];  in xdst_queue_output()  local
2930 	struct xfrm_policy_queue *pq = &pol->polq;  in xdst_queue_output()
2954 		xfrm_pol_put(pol);  in xdst_queue_output()
2959 		xfrm_pol_hold(pol);  in xdst_queue_output()
3598 	struct xfrm_policy *pol;  in __xfrm_policy_check()  local
3647 	pol = NULL;  in __xfrm_policy_check()
3650 		pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);  in __xfrm_policy_check()
3651 		if (IS_ERR(pol)) {  in __xfrm_policy_check()
3657 	if (!pol)  in __xfrm_policy_check()
3658 		pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);  in __xfrm_policy_check()
3660 	if (IS_ERR(pol)) {  in __xfrm_policy_check()
3665 	if (!pol) {  in __xfrm_policy_check()
3680 	WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds());  in __xfrm_policy_check()
3682 	pols[0] = pol;  in __xfrm_policy_check()
3703 	if (pol->action == XFRM_POLICY_ALLOW) {  in __xfrm_policy_check()
3716 			if (pols[pi] != pol &&  in __xfrm_policy_check()
4362 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_migrate_policy_find()  local
4368 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_migrate_policy_find()
4369 		if ((if_id == 0 || pol->if_id == if_id) &&  in xfrm_migrate_policy_find()
4370 		    xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
4371 		    pol->type == type) {  in xfrm_migrate_policy_find()
4372 			ret = pol;  in xfrm_migrate_policy_find()
4378 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_migrate_policy_find()
4379 		if ((pol->priority >= priority) && ret)  in xfrm_migrate_policy_find()
4382 		if ((if_id == 0 || pol->if_id == if_id) &&  in xfrm_migrate_policy_find()
4383 		    xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
4384 		    pol->type == type) {  in xfrm_migrate_policy_find()
4385 			ret = pol;  in xfrm_migrate_policy_find()
4427 static int xfrm_policy_migrate(struct xfrm_policy *pol,  in xfrm_policy_migrate()  argument
4434 	write_lock_bh(&pol->lock);  in xfrm_policy_migrate()
4435 	if (unlikely(pol->walk.dead)) {  in xfrm_policy_migrate()
4438 		write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
4442 	for (i = 0; i < pol->xfrm_nr; i++) {  in xfrm_policy_migrate()
4444 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))  in xfrm_policy_migrate()
4447 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&  in xfrm_policy_migrate()
4448 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET)  in xfrm_policy_migrate()
4451 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,  in xfrm_policy_migrate()
4452 			       sizeof(pol->xfrm_vec[i].id.daddr));  in xfrm_policy_migrate()
4453 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,  in xfrm_policy_migrate()
4454 			       sizeof(pol->xfrm_vec[i].saddr));  in xfrm_policy_migrate()
4455 			pol->xfrm_vec[i].encap_family = mp->new_family;  in xfrm_policy_migrate()
4457 			atomic_inc(&pol->genid);  in xfrm_policy_migrate()
4461 	write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
4512 	struct xfrm_policy *pol = NULL;  in xfrm_migrate()  local
4530 	pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id);  in xfrm_migrate()
4531 	if (!pol) {  in xfrm_migrate()
4554 	err = xfrm_policy_migrate(pol, m, num_migrate, extack);  in xfrm_migrate()
4567 	xfrm_pol_put(pol);  in xfrm_migrate()
4574 	if (pol)  in xfrm_migrate()
4575 		xfrm_pol_put(pol);  in xfrm_migrate()