36 	rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
64 	return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);  in xfrm_dst_hash()
72 	return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);  in xfrm_src_hash()
79 	return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);  in xfrm_spi_hash()
84 	return __xfrm_seq_hash(seq, net->xfrm.state_hmask);  in xfrm_seq_hash()
131 	struct net *net = container_of(work, struct net, xfrm.state_hash_work);  in xfrm_hash_resize()
137 	nsize = xfrm_hash_new_size(net->xfrm.state_hmask);  in xfrm_hash_resize()
160 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
161 	write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
164 	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);  in xfrm_hash_resize()
165 	for (i = net->xfrm.state_hmask; i >= 0; i--)  in xfrm_hash_resize()
168 	osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);  in xfrm_hash_resize()
169 	ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);  in xfrm_hash_resize()
170 	oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);  in xfrm_hash_resize()
171 	ohashmask = net->xfrm.state_hmask;  in xfrm_hash_resize()
173 	rcu_assign_pointer(net->xfrm.state_bydst, ndst);  in xfrm_hash_resize()
174 	rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);  in xfrm_hash_resize()
175 	rcu_assign_pointer(net->xfrm.state_byspi, nspi);  in xfrm_hash_resize()
176 	rcu_assign_pointer(net->xfrm.state_byseq, nseq);  in xfrm_hash_resize()
177 	net->xfrm.state_hmask = nhashmask;  in xfrm_hash_resize()
179 	write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
180 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
686 		spin_lock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
694 		net->xfrm.state_num--;  in __xfrm_state_delete()
695 		spin_unlock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
732 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush_secctx_check()
735 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush_secctx_check()
752 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush_secctx_check()
756 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush_secctx_check()
787 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
793 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush()
796 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush()
800 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
812 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
818 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
830 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
836 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush()
840 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush()
845 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
854 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
863 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
870 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
871 	si->sadcnt = net->xfrm.state_num;  in xfrm_sad_getinfo()
872 	si->sadhcnt = net->xfrm.state_hmask + 1;  in xfrm_sad_getinfo()
874 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
962 	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {  in __xfrm_state_lookup()
987 	hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {  in __xfrm_state_lookup_byaddr()
1023 	    (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&  in xfrm_hash_grow_check()
1024 	    net->xfrm.state_num > net->xfrm.state_hmask)  in xfrm_hash_grow_check()
1025 		schedule_work(&net->xfrm.state_hash_work);  in xfrm_hash_grow_check()
1090 	sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_state_find()
1094 	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {  in xfrm_state_find()
1111 	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {  in xfrm_state_find()
1166 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1168 			list_add(&x->km.all, &net->xfrm.state_all);  in xfrm_state_find()
1169 			hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);  in xfrm_state_find()
1171 			hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);  in xfrm_state_find()
1174 				hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);  in xfrm_state_find()
1178 				hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);  in xfrm_state_find()
1180 			x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in xfrm_state_find()
1182 				      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in xfrm_state_find()
1184 			net->xfrm.state_num++;  in xfrm_state_find()
1186 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1207 	if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {  in xfrm_state_find()
1226 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1228 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_stateonly_find()
1245 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1258 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1259 	list_for_each_entry(w, &net->xfrm.state_all, all) {  in xfrm_state_lookup_byspi()
1266 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1269 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1279 	list_add(&x->km.all, &net->xfrm.state_all);  in __xfrm_state_insert()
1283 	hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);  in __xfrm_state_insert()
1286 	hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);  in __xfrm_state_insert()
1292 		hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);  in __xfrm_state_insert()
1298 		hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);  in __xfrm_state_insert()
1305 	net->xfrm.state_num++;  in __xfrm_state_insert()
1322 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __xfrm_state_bump_genids()
1337 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1340 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1357 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __find_acq_core()
1406 		x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in __find_acq_core()
1409 			      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in __find_acq_core()
1411 		list_add(&x->km.all, &net->xfrm.state_all);  in __find_acq_core()
1412 		hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);  in __find_acq_core()
1414 		hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);  in __find_acq_core()
1416 		net->xfrm.state_num++;  in __find_acq_core()
1439 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1468 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1613 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1618 		hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_migrate_state_find()
1637 		hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {  in xfrm_migrate_state_find()
1653 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1704 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1724 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1761 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1770 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1829 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
1831 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
1843 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
1845 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
1984 	hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {  in __xfrm_find_acq_byseq()
2000 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2002 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2085 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2088 		hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);  in xfrm_alloc_spi()
2089 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2129 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2131 		x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);  in xfrm_state_walk()
2134 	list_for_each_entry_from(x, &net->xfrm.state_all, all) {  in xfrm_state_walk()
2155 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2178 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2180 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2722 	INIT_LIST_HEAD(&net->xfrm.state_all);  in xfrm_state_init()
2726 	net->xfrm.state_bydst = xfrm_hash_alloc(sz);  in xfrm_state_init()
2727 	if (!net->xfrm.state_bydst)  in xfrm_state_init()
2729 	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);  in xfrm_state_init()
2730 	if (!net->xfrm.state_bysrc)  in xfrm_state_init()
2732 	net->xfrm.state_byspi = xfrm_hash_alloc(sz);  in xfrm_state_init()
2733 	if (!net->xfrm.state_byspi)  in xfrm_state_init()
2735 	net->xfrm.state_byseq = xfrm_hash_alloc(sz);  in xfrm_state_init()
2736 	if (!net->xfrm.state_byseq)  in xfrm_state_init()
2738 	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);  in xfrm_state_init()
2740 	net->xfrm.state_num = 0;  in xfrm_state_init()
2741 	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);  in xfrm_state_init()
2742 	spin_lock_init(&net->xfrm.xfrm_state_lock);  in xfrm_state_init()
2743 	seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,  in xfrm_state_init()
2744 			       &net->xfrm.xfrm_state_lock);  in xfrm_state_init()
2748 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_init()
2750 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_init()
2752 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_init()
2761 	flush_work(&net->xfrm.state_hash_work);  in xfrm_state_fini()
2765 	WARN_ON(!list_empty(&net->xfrm.state_all));  in xfrm_state_fini()
2767 	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);  in xfrm_state_fini()
2768 	WARN_ON(!hlist_empty(net->xfrm.state_byseq));  in xfrm_state_fini()
2769 	xfrm_hash_free(net->xfrm.state_byseq, sz);  in xfrm_state_fini()
2770 	WARN_ON(!hlist_empty(net->xfrm.state_byspi));  in xfrm_state_fini()
2771 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_fini()
2772 	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));  in xfrm_state_fini()
2773 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_fini()
2774 	WARN_ON(!hlist_empty(net->xfrm.state_bydst));  in xfrm_state_fini()
2775 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_fini()