575 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)  in fnhe_flush_routes()  argument
579 	rt = rcu_dereference(fnhe->fnhe_rth_input);  in fnhe_flush_routes()
581 		RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);  in fnhe_flush_routes()
585 	rt = rcu_dereference(fnhe->fnhe_rth_output);  in fnhe_flush_routes()
587 		RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);  in fnhe_flush_routes()
596 	struct fib_nh_exception *fnhe, *oldest = NULL;  in fnhe_remove_oldest()  local
598 	for (fnhe_p = &hash->chain; ; fnhe_p = &fnhe->fnhe_next) {  in fnhe_remove_oldest()
599 		fnhe = rcu_dereference_protected(*fnhe_p,  in fnhe_remove_oldest()
601 		if (!fnhe)  in fnhe_remove_oldest()
604 		    time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp)) {  in fnhe_remove_oldest()
605 			oldest = fnhe;  in fnhe_remove_oldest()
624 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)  in fill_route_from_fnhe()  argument
626 	rt->rt_pmtu = fnhe->fnhe_pmtu;  in fill_route_from_fnhe()
627 	rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;  in fill_route_from_fnhe()
628 	rt->dst.expires = fnhe->fnhe_expires;  in fill_route_from_fnhe()
630 	if (fnhe->fnhe_gw) {  in fill_route_from_fnhe()
634 		rt->rt_gw4 = fnhe->fnhe_gw;  in fill_route_from_fnhe()
643 	struct fib_nh_exception *fnhe;  in update_or_create_fnhe()  local
665 	for (fnhe = rcu_dereference(hash->chain); fnhe;  in update_or_create_fnhe()
666 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {  in update_or_create_fnhe()
667 		if (fnhe->fnhe_daddr == daddr)  in update_or_create_fnhe()
672 	if (fnhe) {  in update_or_create_fnhe()
673 		if (fnhe->fnhe_genid != genid)  in update_or_create_fnhe()
674 			fnhe->fnhe_genid = genid;  in update_or_create_fnhe()
676 			fnhe->fnhe_gw = gw;  in update_or_create_fnhe()
678 			fnhe->fnhe_pmtu = pmtu;  in update_or_create_fnhe()
679 			fnhe->fnhe_mtu_locked = lock;  in update_or_create_fnhe()
681 		fnhe->fnhe_expires = max(1UL, expires);  in update_or_create_fnhe()
683 		rt = rcu_dereference(fnhe->fnhe_rth_input);  in update_or_create_fnhe()
685 			fill_route_from_fnhe(rt, fnhe);  in update_or_create_fnhe()
686 		rt = rcu_dereference(fnhe->fnhe_rth_output);  in update_or_create_fnhe()
688 			fill_route_from_fnhe(rt, fnhe);  in update_or_create_fnhe()
699 		fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);  in update_or_create_fnhe()
700 		if (!fnhe)  in update_or_create_fnhe()
703 		fnhe->fnhe_next = hash->chain;  in update_or_create_fnhe()
705 		fnhe->fnhe_genid = genid;  in update_or_create_fnhe()
706 		fnhe->fnhe_daddr = daddr;  in update_or_create_fnhe()
707 		fnhe->fnhe_gw = gw;  in update_or_create_fnhe()
708 		fnhe->fnhe_pmtu = pmtu;  in update_or_create_fnhe()
709 		fnhe->fnhe_mtu_locked = lock;  in update_or_create_fnhe()
710 		fnhe->fnhe_expires = max(1UL, expires);  in update_or_create_fnhe()
712 		rcu_assign_pointer(hash->chain, fnhe);  in update_or_create_fnhe()
732 	fnhe->fnhe_stamp = jiffies;  in update_or_create_fnhe()
1331 	struct fib_nh_exception *fnhe, __rcu **fnhe_p;  in ip_del_fnhe()  local
1341 	fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));  in ip_del_fnhe()
1342 	while (fnhe) {  in ip_del_fnhe()
1343 		if (fnhe->fnhe_daddr == daddr) {  in ip_del_fnhe()
1345 				fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));  in ip_del_fnhe()
1349 			fnhe->fnhe_daddr = 0;  in ip_del_fnhe()
1350 			fnhe_flush_routes(fnhe);  in ip_del_fnhe()
1351 			kfree_rcu(fnhe, rcu);  in ip_del_fnhe()
1354 		fnhe_p = &fnhe->fnhe_next;  in ip_del_fnhe()
1355 		fnhe = rcu_dereference_protected(fnhe->fnhe_next,  in ip_del_fnhe()
1366 	struct fib_nh_exception *fnhe;  in find_exception()  local
1374 	for (fnhe = rcu_dereference(hash[hval].chain); fnhe;  in find_exception()
1375 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {  in find_exception()
1376 		if (fnhe->fnhe_daddr == daddr) {  in find_exception()
1377 			if (fnhe->fnhe_expires &&  in find_exception()
1378 			    time_after(jiffies, fnhe->fnhe_expires)) {  in find_exception()
1382 			return fnhe;  in find_exception()
1406 		struct fib_nh_exception *fnhe;  in ip_mtu_from_fib_result()  local
1408 		fnhe = find_exception(nhc, daddr);  in ip_mtu_from_fib_result()
1409 		if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))  in ip_mtu_from_fib_result()
1410 			mtu = fnhe->fnhe_pmtu;  in ip_mtu_from_fib_result()
1419 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,  in rt_bind_exception()  argument
1426 	if (daddr == fnhe->fnhe_daddr) {  in rt_bind_exception()
1432 			porig = &fnhe->fnhe_rth_input;  in rt_bind_exception()
1434 			porig = &fnhe->fnhe_rth_output;  in rt_bind_exception()
1437 		if (fnhe->fnhe_genid != genid) {  in rt_bind_exception()
1438 			fnhe->fnhe_genid = genid;  in rt_bind_exception()
1439 			fnhe->fnhe_gw = 0;  in rt_bind_exception()
1440 			fnhe->fnhe_pmtu = 0;  in rt_bind_exception()
1441 			fnhe->fnhe_expires = 0;  in rt_bind_exception()
1442 			fnhe->fnhe_mtu_locked = false;  in rt_bind_exception()
1443 			fnhe_flush_routes(fnhe);  in rt_bind_exception()
1446 		fill_route_from_fnhe(rt, fnhe);  in rt_bind_exception()
1462 		fnhe->fnhe_stamp = jiffies;  in rt_bind_exception()
1570 			   struct fib_nh_exception *fnhe,  in rt_set_nexthop()  argument
1600 		if (unlikely(fnhe))  in rt_set_nexthop()
1601 			cached = rt_bind_exception(rt, fnhe, daddr, do_cache);  in rt_set_nexthop()
1797 	struct fib_nh_exception *fnhe;  in __mkroute_input()  local
1849 	fnhe = find_exception(nhc, daddr);  in __mkroute_input()
1851 		if (fnhe)  in __mkroute_input()
1852 			rth = rcu_dereference(fnhe->fnhe_rth_input);  in __mkroute_input()
1873 	rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,  in __mkroute_input()
2508 	struct fib_nh_exception *fnhe;  in __mkroute_output()  local
2565 	fnhe = NULL;  in __mkroute_output()
2571 		fnhe = find_exception(nhc, fl4->daddr);  in __mkroute_output()
2574 		if (fnhe) {  in __mkroute_output()
2575 			prth = &fnhe->fnhe_rth_output;  in __mkroute_output()
2618 	rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);  in __mkroute_output()
3083 		struct fib_nh_exception *fnhe;  in fnhe_dump_bucket()  local
3085 		for (fnhe = rcu_dereference(bucket[i].chain); fnhe;  in fnhe_dump_bucket()
3086 		     fnhe = rcu_dereference(fnhe->fnhe_next)) {  in fnhe_dump_bucket()
3093 			if (fnhe->fnhe_genid != genid)  in fnhe_dump_bucket()
3096 			if (fnhe->fnhe_expires &&  in fnhe_dump_bucket()
3097 			    time_after(jiffies, fnhe->fnhe_expires))  in fnhe_dump_bucket()
3100 			rt = rcu_dereference(fnhe->fnhe_rth_input);  in fnhe_dump_bucket()
3102 				rt = rcu_dereference(fnhe->fnhe_rth_output);  in fnhe_dump_bucket()
3106 			err = rt_fill_info(net, fnhe->fnhe_daddr, 0, rt,  in fnhe_dump_bucket()