458 	return __idx_hash(index, net->xfrm.policy_idx_hmask);  in idx_hash()
468 		*dbits = net->xfrm.policy_bydst[dir].dbits4;  in __get_hash_thresh()
469 		*sbits = net->xfrm.policy_bydst[dir].sbits4;  in __get_hash_thresh()
473 		*dbits = net->xfrm.policy_bydst[dir].dbits6;  in __get_hash_thresh()
474 		*sbits = net->xfrm.policy_bydst[dir].sbits6;  in __get_hash_thresh()
487 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in policy_hash_bysel()
498 	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,  in policy_hash_bysel()
499 		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;  in policy_hash_bysel()
507 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in policy_hash_direct()
515 	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,  in policy_hash_direct()
516 		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;  in policy_hash_direct()
578 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in xfrm_bydst_resize()
588 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_bydst_resize()
589 	write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);  in xfrm_bydst_resize()
591 	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,  in xfrm_bydst_resize()
592 				lockdep_is_held(&net->xfrm.xfrm_policy_lock));  in xfrm_bydst_resize()
597 	rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);  in xfrm_bydst_resize()
598 	net->xfrm.policy_bydst[dir].hmask = nhashmask;  in xfrm_bydst_resize()
600 	write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);  in xfrm_bydst_resize()
601 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_bydst_resize()
610 	unsigned int hmask = net->xfrm.policy_idx_hmask;  in xfrm_byidx_resize()
613 	struct hlist_head *oidx = net->xfrm.policy_byidx;  in xfrm_byidx_resize()
620 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_byidx_resize()
625 	net->xfrm.policy_byidx = nidx;  in xfrm_byidx_resize()
626 	net->xfrm.policy_idx_hmask = nhashmask;  in xfrm_byidx_resize()
628 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_byidx_resize()
635 	unsigned int cnt = net->xfrm.policy_count[dir];  in xfrm_bydst_should_resize()
636 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in xfrm_bydst_should_resize()
650 	unsigned int hmask = net->xfrm.policy_idx_hmask;  in xfrm_byidx_should_resize()
661 	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];  in xfrm_spd_getinfo()
662 	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];  in xfrm_spd_getinfo()
663 	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];  in xfrm_spd_getinfo()
664 	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];  in xfrm_spd_getinfo()
665 	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];  in xfrm_spd_getinfo()
666 	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];  in xfrm_spd_getinfo()
667 	si->spdhcnt = net->xfrm.policy_idx_hmask;  in xfrm_spd_getinfo()
675 	struct net *net = container_of(work, struct net, xfrm.policy_hash_work);  in xfrm_hash_resize()
707 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_inexact_alloc_bin()
723 	seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock);  in xfrm_policy_inexact_alloc_bin()
729 		list_add(&bin->inexact_bins, &net->xfrm.inexact_bins);  in xfrm_policy_inexact_alloc_bin()
849 	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_inexact_list_reinsert()
1110 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_inexact_prune_bin()
1112 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_inexact_prune_bin()
1119 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);  in __xfrm_policy_inexact_flush()
1121 	list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins)  in __xfrm_policy_inexact_flush()
1133 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_inexact_alloc_chain()
1198 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_inexact_insert()
1212 	chain = &net->xfrm.policy_inexact[dir];  in xfrm_policy_inexact_insert()
1224 				       xfrm.policy_hthresh.work);  in xfrm_hash_rebuild()
1240 		seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);  in xfrm_hash_rebuild()
1242 		lbits4 = net->xfrm.policy_hthresh.lbits4;  in xfrm_hash_rebuild()
1243 		rbits4 = net->xfrm.policy_hthresh.rbits4;  in xfrm_hash_rebuild()
1244 		lbits6 = net->xfrm.policy_hthresh.lbits6;  in xfrm_hash_rebuild()
1245 		rbits6 = net->xfrm.policy_hthresh.rbits6;  in xfrm_hash_rebuild()
1246 	} while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));  in xfrm_hash_rebuild()
1248 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_hash_rebuild()
1249 	write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);  in xfrm_hash_rebuild()
1254 	list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {  in xfrm_hash_rebuild()
1297 					  &net->xfrm.policy_inexact[dir],  in xfrm_hash_rebuild()
1303 		hmask = net->xfrm.policy_bydst[dir].hmask;  in xfrm_hash_rebuild()
1304 		odst = net->xfrm.policy_bydst[dir].table;  in xfrm_hash_rebuild()
1311 			net->xfrm.policy_bydst[dir].dbits4 = rbits4;  in xfrm_hash_rebuild()
1312 			net->xfrm.policy_bydst[dir].sbits4 = lbits4;  in xfrm_hash_rebuild()
1313 			net->xfrm.policy_bydst[dir].dbits6 = rbits6;  in xfrm_hash_rebuild()
1314 			net->xfrm.policy_bydst[dir].sbits6 = lbits6;  in xfrm_hash_rebuild()
1317 			net->xfrm.policy_bydst[dir].dbits4 = lbits4;  in xfrm_hash_rebuild()
1318 			net->xfrm.policy_bydst[dir].sbits4 = rbits4;  in xfrm_hash_rebuild()
1319 			net->xfrm.policy_bydst[dir].dbits6 = lbits6;  in xfrm_hash_rebuild()
1320 			net->xfrm.policy_bydst[dir].sbits6 = rbits6;  in xfrm_hash_rebuild()
1325 	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {  in xfrm_hash_rebuild()
1358 	write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);  in xfrm_hash_rebuild()
1359 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_hash_rebuild()
1366 	schedule_work(&net->xfrm.policy_hthresh.work);  in xfrm_policy_hash_rebuild()
1392 		list = net->xfrm.policy_byidx + idx_hash(net, idx);  in xfrm_gen_index()
1578 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_insert()
1586 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_insert()
1603 	hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));  in xfrm_policy_insert()
1608 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_insert()
1613 		schedule_work(&net->xfrm.policy_hash_work);  in xfrm_policy_insert()
1651 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_bysel_ctx()
1660 			spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_bysel_ctx()
1667 			spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_bysel_ctx()
1694 				spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_bysel_ctx()
1701 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_bysel_ctx()
1723 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_byid()
1724 	chain = net->xfrm.policy_byidx + idx_hash(net, id);  in xfrm_policy_byid()
1734 					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_byid()
1743 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_byid()
1758 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush_secctx_check()
1785 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_flush()
1792 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush()
1800 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_flush()
1804 		spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_flush()
1812 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_flush()
1832 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_walk()
1834 		x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);  in xfrm_policy_walk()
1839 	list_for_each_entry_from(x, &net->xfrm.policy_all, all) {  in xfrm_policy_walk()
1860 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_walk()
1879 	spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */  in xfrm_policy_walk_done()
1881 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_walk_done()
2004 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_inexact_lookup()
2097 		sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);  in xfrm_policy_lookup_bytype()
2099 	} while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence));  in xfrm_policy_lookup_bytype()
2130 	if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence))  in xfrm_policy_lookup_bytype()
2204 	list_add(&pol->walk.all, &net->xfrm.policy_all);  in __xfrm_policy_link()
2205 	net->xfrm.policy_count[dir]++;  in __xfrm_policy_link()
2225 	net->xfrm.policy_count[dir]--;  in __xfrm_policy_unlink()
2244 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_delete()
2246 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_delete()
2265 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_sk_policy_insert()
2267 				lockdep_is_held(&net->xfrm.xfrm_policy_lock));  in xfrm_sk_policy_insert()
2283 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_sk_policy_insert()
2315 		spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in clone_policy()
2317 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in clone_policy()
2363 		      struct xfrm_state **xfrm, unsigned short family)  in xfrm_tmpl_resolve_one()  argument
2396 			xfrm[nx++] = x;  in xfrm_tmpl_resolve_one()
2416 		xfrm_state_put(xfrm[nx]);  in xfrm_tmpl_resolve_one()
2422 		  struct xfrm_state **xfrm, unsigned short family)  in xfrm_tmpl_resolve()  argument
2425 	struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;  in xfrm_tmpl_resolve()
2447 		xfrm_state_sort(xfrm, tpp, cnx, family);  in xfrm_tmpl_resolve()
2477 		dst_ops = &net->xfrm.xfrm4_dst_ops;  in xfrm_alloc_dst()
2481 		dst_ops = &net->xfrm.xfrm6_dst_ops;  in xfrm_alloc_dst()
2532 					    struct xfrm_state **xfrm,  in xfrm_bundle_create()  argument
2579 		if (xfrm[i]->sel.family == AF_UNSPEC) {  in xfrm_bundle_create()
2580 			inner_mode = xfrm_ip2inner_mode(xfrm[i],  in xfrm_bundle_create()
2588 			inner_mode = &xfrm[i]->inner_mode;  in xfrm_bundle_create()
2593 		if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {  in xfrm_bundle_create()
2597 			if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m)  in xfrm_bundle_create()
2598 				mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]);  in xfrm_bundle_create()
2600 			family = xfrm[i]->props.family;  in xfrm_bundle_create()
2602 			dst = xfrm_dst_lookup(xfrm[i], tos, oif,  in xfrm_bundle_create()
2610 		dst1->xfrm = xfrm[i];  in xfrm_bundle_create()
2611 		xdst->xfrm_genid = xfrm[i]->genid;  in xfrm_bundle_create()
2628 		header_len += xfrm[i]->props.header_len;  in xfrm_bundle_create()
2629 		if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)  in xfrm_bundle_create()
2630 			nfheader_len += xfrm[i]->props.header_len;  in xfrm_bundle_create()
2631 		trailer_len += xfrm[i]->props.trailer_len;  in xfrm_bundle_create()
2653 		header_len -= xdst_prev->u.dst.xfrm->props.header_len;  in xfrm_bundle_create()
2654 		trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;  in xfrm_bundle_create()
2661 		xfrm_state_put(xfrm[i]);  in xfrm_bundle_create()
2723 	struct xfrm_state *xfrm[XFRM_MAX_DEPTH];  in xfrm_resolve_and_create_bundle()  local
2730 	err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);  in xfrm_resolve_and_create_bundle()
2740 	dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig);  in xfrm_resolve_and_create_bundle()
2901 	    net->xfrm.sysctl_larval_drop ||  in xfrm_create_dummy_bundle()
3093 			       !net->xfrm.policy_count[XFRM_POLICY_OUT]))  in xfrm_lookup_with_ifid()
3118 		if (net->xfrm.sysctl_larval_drop) {  in xfrm_lookup_with_ifid()
3158 	if (dst && dst->xfrm &&  in xfrm_lookup_with_ifid()
3159 	    dst->xfrm->props.mode == XFRM_MODE_TUNNEL)  in xfrm_lookup_with_ifid()
3165 	    net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {  in xfrm_lookup_with_ifid()
3576 		if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {  in __xfrm_policy_check()
3637 		if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK &&  in __xfrm_policy_check()
3749 	while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {  in xfrm_dst_ifdown()
3784 		pmtu = xfrm_state_mtu(dst->xfrm, pmtu);  in xfrm_init_pmtu()
3819 		if (dst->xfrm->km.state != XFRM_STATE_VALID)  in xfrm_bundle_ok()
3821 		if (xdst->xfrm_genid != dst->xfrm->genid)  in xfrm_bundle_ok()
3844 	} while (dst->xfrm);  in xfrm_bundle_ok()
3854 		mtu = xfrm_state_mtu(dst->xfrm, mtu);  in xfrm_bundle_ok()
3883 	while (dst->xfrm) {  in xfrm_get_dst_nexthop()
3884 		const struct xfrm_state *xfrm = dst->xfrm;  in xfrm_get_dst_nexthop()  local
3888 		if (xfrm->props.mode == XFRM_MODE_TRANSPORT)  in xfrm_get_dst_nexthop()
3890 		if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)  in xfrm_get_dst_nexthop()
3891 			daddr = xfrm->coaddr;  in xfrm_get_dst_nexthop()
3892 		else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))  in xfrm_get_dst_nexthop()
3893 			daddr = &xfrm->id.daddr;  in xfrm_get_dst_nexthop()
4036 	net->xfrm.policy_byidx = xfrm_hash_alloc(sz);  in xfrm_policy_init()
4037 	if (!net->xfrm.policy_byidx)  in xfrm_policy_init()
4039 	net->xfrm.policy_idx_hmask = hmask;  in xfrm_policy_init()
4044 		net->xfrm.policy_count[dir] = 0;  in xfrm_policy_init()
4045 		net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;  in xfrm_policy_init()
4046 		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);  in xfrm_policy_init()
4048 		htab = &net->xfrm.policy_bydst[dir];  in xfrm_policy_init()
4058 	net->xfrm.policy_hthresh.lbits4 = 32;  in xfrm_policy_init()
4059 	net->xfrm.policy_hthresh.rbits4 = 32;  in xfrm_policy_init()
4060 	net->xfrm.policy_hthresh.lbits6 = 128;  in xfrm_policy_init()
4061 	net->xfrm.policy_hthresh.rbits6 = 128;  in xfrm_policy_init()
4063 	seqlock_init(&net->xfrm.policy_hthresh.lock);  in xfrm_policy_init()
4065 	INIT_LIST_HEAD(&net->xfrm.policy_all);  in xfrm_policy_init()
4066 	INIT_LIST_HEAD(&net->xfrm.inexact_bins);  in xfrm_policy_init()
4067 	INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);  in xfrm_policy_init()
4068 	INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);  in xfrm_policy_init()
4075 		htab = &net->xfrm.policy_bydst[dir];  in xfrm_policy_init()
4078 	xfrm_hash_free(net->xfrm.policy_byidx, sz);  in xfrm_policy_init()
4089 	flush_work(&net->xfrm.policy_hash_work);  in xfrm_policy_fini()
4095 	WARN_ON(!list_empty(&net->xfrm.policy_all));  in xfrm_policy_fini()
4100 		WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));  in xfrm_policy_fini()
4102 		htab = &net->xfrm.policy_bydst[dir];  in xfrm_policy_fini()
4108 	sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);  in xfrm_policy_fini()
4109 	WARN_ON(!hlist_empty(net->xfrm.policy_byidx));  in xfrm_policy_fini()
4110 	xfrm_hash_free(net->xfrm.policy_byidx, sz);  in xfrm_policy_fini()
4112 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_fini()
4113 	list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins)  in xfrm_policy_fini()
4115 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_policy_fini()
4123 	spin_lock_init(&net->xfrm.xfrm_state_lock);  in xfrm_net_init()
4124 	spin_lock_init(&net->xfrm.xfrm_policy_lock);  in xfrm_net_init()
4125 	seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock);  in xfrm_net_init()
4126 	mutex_init(&net->xfrm.xfrm_cfg_mutex);  in xfrm_net_init()
4127 	net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT;  in xfrm_net_init()
4128 	net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT;  in xfrm_net_init()
4129 	net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT;  in xfrm_net_init()
4274 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_migrate_policy_find()
4285 	chain = &net->xfrm.policy_inexact[dir];  in xfrm_migrate_policy_find()
4300 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);  in xfrm_migrate_policy_find()