1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * net/dst.h	Protocol independent destination cache definitions.
4  *
5  * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6  *
7  */
8 
9 #ifndef _NET_DST_H
10 #define _NET_DST_H
11 
12 #include <net/dst_ops.h>
13 #include <linux/netdevice.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/rcupdate.h>
16 #include <linux/bug.h>
17 #include <linux/jiffies.h>
18 #include <linux/refcount.h>
19 #include <linux/rcuref.h>
20 #include <net/neighbour.h>
21 #include <asm/processor.h>
22 #include <linux/indirect_call_wrapper.h>
23 
24 struct sk_buff;
25 
26 struct dst_entry {
27 	struct net_device       *dev;
28 	struct  dst_ops	        *ops;
29 	unsigned long		_metrics;
30 	unsigned long           expires;
31 #ifdef CONFIG_XFRM
32 	struct xfrm_state	*xfrm;
33 #else
34 	void			*__pad1;
35 #endif
36 	int			(*input)(struct sk_buff *);
37 	int			(*output)(struct net *net, struct sock *sk, struct sk_buff *skb);
38 
39 	unsigned short		flags;
40 #define DST_NOXFRM		0x0002
41 #define DST_NOPOLICY		0x0004
42 #define DST_NOCOUNT		0x0008
43 #define DST_FAKE_RTABLE		0x0010
44 #define DST_XFRM_TUNNEL		0x0020
45 #define DST_XFRM_QUEUE		0x0040
46 #define DST_METADATA		0x0080
47 
48 	/* A non-zero value of dst->obsolete forces by-hand validation
49 	 * of the route entry.  Positive values are set by the generic
50 	 * dst layer to indicate that the entry has been forcefully
51 	 * destroyed.
52 	 *
53 	 * Negative values are used by the implementation layer code to
54 	 * force invocation of the dst_ops->check() method.
55 	 */
56 	short			obsolete;
57 #define DST_OBSOLETE_NONE	0
58 #define DST_OBSOLETE_DEAD	2
59 #define DST_OBSOLETE_FORCE_CHK	-1
60 #define DST_OBSOLETE_KILL	-2
61 	unsigned short		header_len;	/* more space at head required */
62 	unsigned short		trailer_len;	/* space to reserve at tail */
63 
64 	/*
65 	 * __rcuref wants to be on a different cache line from
66 	 * input/output/ops or performance tanks badly
67 	 */
68 #ifdef CONFIG_64BIT
69 	rcuref_t		__rcuref;	/* 64-bit offset 64 */
70 #endif
71 	int			__use;
72 	unsigned long		lastuse;
73 	struct rcu_head		rcu_head;
74 	short			error;
75 	short			__pad;
76 	__u32			tclassid;
77 #ifndef CONFIG_64BIT
78 	struct lwtunnel_state   *lwtstate;
79 	rcuref_t		__rcuref;	/* 32-bit offset 64 */
80 #endif
81 	netdevice_tracker	dev_tracker;
82 
83 	/*
84 	 * Used by rtable and rt6_info. Moves lwtstate into the next cache
85 	 * line on 64bit so that lwtstate does not cause false sharing with
86 	 * __rcuref under contention of __rcuref. This also puts the
87 	 * frequently accessed members of rtable and rt6_info out of the
88 	 * __rcuref cache line.
89 	 */
90 	struct list_head	rt_uncached;
91 	struct uncached_list	*rt_uncached_list;
92 #ifdef CONFIG_64BIT
93 	struct lwtunnel_state   *lwtstate;
94 #endif
95 };
96 
97 struct dst_metrics {
98 	u32		metrics[RTAX_MAX];
99 	refcount_t	refcnt;
100 } __aligned(4);		/* Low pointer bits contain DST_METRICS_FLAGS */
101 extern const struct dst_metrics dst_default_metrics;
102 
103 u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old);
104 
105 #define DST_METRICS_READ_ONLY		0x1UL
106 #define DST_METRICS_REFCOUNTED		0x2UL
107 #define DST_METRICS_FLAGS		0x3UL
108 #define __DST_METRICS_PTR(Y)	\
109 	((u32 *)((Y) & ~DST_METRICS_FLAGS))
110 #define DST_METRICS_PTR(X)	__DST_METRICS_PTR((X)->_metrics)
111 
dst_metrics_read_only(const struct dst_entry * dst)112 static inline bool dst_metrics_read_only(const struct dst_entry *dst)
113 {
114 	return dst->_metrics & DST_METRICS_READ_ONLY;
115 }
116 
117 void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old);
118 
dst_destroy_metrics_generic(struct dst_entry * dst)119 static inline void dst_destroy_metrics_generic(struct dst_entry *dst)
120 {
121 	unsigned long val = dst->_metrics;
122 	if (!(val & DST_METRICS_READ_ONLY))
123 		__dst_destroy_metrics_generic(dst, val);
124 }
125 
dst_metrics_write_ptr(struct dst_entry * dst)126 static inline u32 *dst_metrics_write_ptr(struct dst_entry *dst)
127 {
128 	unsigned long p = dst->_metrics;
129 
130 	BUG_ON(!p);
131 
132 	if (p & DST_METRICS_READ_ONLY)
133 		return dst->ops->cow_metrics(dst, p);
134 	return __DST_METRICS_PTR(p);
135 }
136 
137 /* This may only be invoked before the entry has reached global
138  * visibility.
139  */
dst_init_metrics(struct dst_entry * dst,const u32 * src_metrics,bool read_only)140 static inline void dst_init_metrics(struct dst_entry *dst,
141 				    const u32 *src_metrics,
142 				    bool read_only)
143 {
144 	dst->_metrics = ((unsigned long) src_metrics) |
145 		(read_only ? DST_METRICS_READ_ONLY : 0);
146 }
147 
dst_copy_metrics(struct dst_entry * dest,const struct dst_entry * src)148 static inline void dst_copy_metrics(struct dst_entry *dest, const struct dst_entry *src)
149 {
150 	u32 *dst_metrics = dst_metrics_write_ptr(dest);
151 
152 	if (dst_metrics) {
153 		u32 *src_metrics = DST_METRICS_PTR(src);
154 
155 		memcpy(dst_metrics, src_metrics, RTAX_MAX * sizeof(u32));
156 	}
157 }
158 
dst_metrics_ptr(struct dst_entry * dst)159 static inline u32 *dst_metrics_ptr(struct dst_entry *dst)
160 {
161 	return DST_METRICS_PTR(dst);
162 }
163 
164 static inline u32
dst_metric_raw(const struct dst_entry * dst,const int metric)165 dst_metric_raw(const struct dst_entry *dst, const int metric)
166 {
167 	u32 *p = DST_METRICS_PTR(dst);
168 
169 	return p[metric-1];
170 }
171 
172 static inline u32
dst_metric(const struct dst_entry * dst,const int metric)173 dst_metric(const struct dst_entry *dst, const int metric)
174 {
175 	WARN_ON_ONCE(metric == RTAX_HOPLIMIT ||
176 		     metric == RTAX_ADVMSS ||
177 		     metric == RTAX_MTU);
178 	return dst_metric_raw(dst, metric);
179 }
180 
181 static inline u32
dst_metric_advmss(const struct dst_entry * dst)182 dst_metric_advmss(const struct dst_entry *dst)
183 {
184 	u32 advmss = dst_metric_raw(dst, RTAX_ADVMSS);
185 
186 	if (!advmss)
187 		advmss = dst->ops->default_advmss(dst);
188 
189 	return advmss;
190 }
191 
dst_metric_set(struct dst_entry * dst,int metric,u32 val)192 static inline void dst_metric_set(struct dst_entry *dst, int metric, u32 val)
193 {
194 	u32 *p = dst_metrics_write_ptr(dst);
195 
196 	if (p)
197 		p[metric-1] = val;
198 }
199 
200 /* Kernel-internal feature bits that are unallocated in user space. */
201 #define DST_FEATURE_ECN_CA	(1U << 31)
202 
203 #define DST_FEATURE_MASK	(DST_FEATURE_ECN_CA)
204 #define DST_FEATURE_ECN_MASK	(DST_FEATURE_ECN_CA | RTAX_FEATURE_ECN)
205 
206 static inline u32
dst_feature(const struct dst_entry * dst,u32 feature)207 dst_feature(const struct dst_entry *dst, u32 feature)
208 {
209 	return dst_metric(dst, RTAX_FEATURES) & feature;
210 }
211 
212 INDIRECT_CALLABLE_DECLARE(unsigned int ip6_mtu(const struct dst_entry *));
213 INDIRECT_CALLABLE_DECLARE(unsigned int ipv4_mtu(const struct dst_entry *));
dst_mtu(const struct dst_entry * dst)214 static inline u32 dst_mtu(const struct dst_entry *dst)
215 {
216 	return INDIRECT_CALL_INET(dst->ops->mtu, ip6_mtu, ipv4_mtu, dst);
217 }
218 
219 /* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */
dst_metric_rtt(const struct dst_entry * dst,int metric)220 static inline unsigned long dst_metric_rtt(const struct dst_entry *dst, int metric)
221 {
222 	return msecs_to_jiffies(dst_metric(dst, metric));
223 }
224 
225 static inline u32
dst_allfrag(const struct dst_entry * dst)226 dst_allfrag(const struct dst_entry *dst)
227 {
228 	int ret = dst_feature(dst,  RTAX_FEATURE_ALLFRAG);
229 	return ret;
230 }
231 
232 static inline int
dst_metric_locked(const struct dst_entry * dst,int metric)233 dst_metric_locked(const struct dst_entry *dst, int metric)
234 {
235 	return dst_metric(dst, RTAX_LOCK) & (1 << metric);
236 }
237 
dst_hold(struct dst_entry * dst)238 static inline void dst_hold(struct dst_entry *dst)
239 {
240 	/*
241 	 * If your kernel compilation stops here, please check
242 	 * the placement of __rcuref in struct dst_entry
243 	 */
244 	BUILD_BUG_ON(offsetof(struct dst_entry, __rcuref) & 63);
245 	WARN_ON(!rcuref_get(&dst->__rcuref));
246 }
247 
dst_use_noref(struct dst_entry * dst,unsigned long time)248 static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
249 {
250 	if (unlikely(time != dst->lastuse)) {
251 		dst->__use++;
252 		dst->lastuse = time;
253 	}
254 }
255 
dst_clone(struct dst_entry * dst)256 static inline struct dst_entry *dst_clone(struct dst_entry *dst)
257 {
258 	if (dst)
259 		dst_hold(dst);
260 	return dst;
261 }
262 
263 void dst_release(struct dst_entry *dst);
264 
265 void dst_release_immediate(struct dst_entry *dst);
266 
refdst_drop(unsigned long refdst)267 static inline void refdst_drop(unsigned long refdst)
268 {
269 	if (!(refdst & SKB_DST_NOREF))
270 		dst_release((struct dst_entry *)(refdst & SKB_DST_PTRMASK));
271 }
272 
273 /**
274  * skb_dst_drop - drops skb dst
275  * @skb: buffer
276  *
277  * Drops dst reference count if a reference was taken.
278  */
skb_dst_drop(struct sk_buff * skb)279 static inline void skb_dst_drop(struct sk_buff *skb)
280 {
281 	if (skb->_skb_refdst) {
282 		refdst_drop(skb->_skb_refdst);
283 		skb->_skb_refdst = 0UL;
284 	}
285 }
286 
__skb_dst_copy(struct sk_buff * nskb,unsigned long refdst)287 static inline void __skb_dst_copy(struct sk_buff *nskb, unsigned long refdst)
288 {
289 	nskb->slow_gro |= !!refdst;
290 	nskb->_skb_refdst = refdst;
291 	if (!(nskb->_skb_refdst & SKB_DST_NOREF))
292 		dst_clone(skb_dst(nskb));
293 }
294 
skb_dst_copy(struct sk_buff * nskb,const struct sk_buff * oskb)295 static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb)
296 {
297 	__skb_dst_copy(nskb, oskb->_skb_refdst);
298 }
299 
300 /**
301  * dst_hold_safe - Take a reference on a dst if possible
302  * @dst: pointer to dst entry
303  *
304  * This helper returns false if it could not safely
305  * take a reference on a dst.
306  */
dst_hold_safe(struct dst_entry * dst)307 static inline bool dst_hold_safe(struct dst_entry *dst)
308 {
309 	return rcuref_get(&dst->__rcuref);
310 }
311 
312 /**
313  * skb_dst_force - makes sure skb dst is refcounted
314  * @skb: buffer
315  *
316  * If dst is not yet refcounted and not destroyed, grab a ref on it.
317  * Returns true if dst is refcounted.
318  */
skb_dst_force(struct sk_buff * skb)319 static inline bool skb_dst_force(struct sk_buff *skb)
320 {
321 	if (skb_dst_is_noref(skb)) {
322 		struct dst_entry *dst = skb_dst(skb);
323 
324 		WARN_ON(!rcu_read_lock_held());
325 		if (!dst_hold_safe(dst))
326 			dst = NULL;
327 
328 		skb->_skb_refdst = (unsigned long)dst;
329 		skb->slow_gro |= !!dst;
330 	}
331 
332 	return skb->_skb_refdst != 0UL;
333 }
334 
335 
336 /**
337  *	__skb_tunnel_rx - prepare skb for rx reinsert
338  *	@skb: buffer
339  *	@dev: tunnel device
340  *	@net: netns for packet i/o
341  *
342  *	After decapsulation, packet is going to re-enter (netif_rx()) our stack,
343  *	so make some cleanups. (no accounting done)
344  */
__skb_tunnel_rx(struct sk_buff * skb,struct net_device * dev,struct net * net)345 static inline void __skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev,
346 				   struct net *net)
347 {
348 	skb->dev = dev;
349 
350 	/*
351 	 * Clear hash so that we can recalulate the hash for the
352 	 * encapsulated packet, unless we have already determine the hash
353 	 * over the L4 4-tuple.
354 	 */
355 	skb_clear_hash_if_not_l4(skb);
356 	skb_set_queue_mapping(skb, 0);
357 	skb_scrub_packet(skb, !net_eq(net, dev_net(dev)));
358 }
359 
360 /**
361  *	skb_tunnel_rx - prepare skb for rx reinsert
362  *	@skb: buffer
363  *	@dev: tunnel device
364  *	@net: netns for packet i/o
365  *
366  *	After decapsulation, packet is going to re-enter (netif_rx()) our stack,
367  *	so make some cleanups, and perform accounting.
368  *	Note: this accounting is not SMP safe.
369  */
skb_tunnel_rx(struct sk_buff * skb,struct net_device * dev,struct net * net)370 static inline void skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev,
371 				 struct net *net)
372 {
373 	DEV_STATS_INC(dev, rx_packets);
374 	DEV_STATS_ADD(dev, rx_bytes, skb->len);
375 	__skb_tunnel_rx(skb, dev, net);
376 }
377 
dst_tclassid(const struct sk_buff * skb)378 static inline u32 dst_tclassid(const struct sk_buff *skb)
379 {
380 #ifdef CONFIG_IP_ROUTE_CLASSID
381 	const struct dst_entry *dst;
382 
383 	dst = skb_dst(skb);
384 	if (dst)
385 		return dst->tclassid;
386 #endif
387 	return 0;
388 }
389 
390 int dst_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
dst_discard(struct sk_buff * skb)391 static inline int dst_discard(struct sk_buff *skb)
392 {
393 	return dst_discard_out(&init_net, skb->sk, skb);
394 }
395 void *dst_alloc(struct dst_ops *ops, struct net_device *dev, int initial_ref,
396 		int initial_obsolete, unsigned short flags);
397 void dst_init(struct dst_entry *dst, struct dst_ops *ops,
398 	      struct net_device *dev, int initial_ref, int initial_obsolete,
399 	      unsigned short flags);
400 struct dst_entry *dst_destroy(struct dst_entry *dst);
401 void dst_dev_put(struct dst_entry *dst);
402 
dst_confirm(struct dst_entry * dst)403 static inline void dst_confirm(struct dst_entry *dst)
404 {
405 }
406 
dst_neigh_lookup(const struct dst_entry * dst,const void * daddr)407 static inline struct neighbour *dst_neigh_lookup(const struct dst_entry *dst, const void *daddr)
408 {
409 	struct neighbour *n = dst->ops->neigh_lookup(dst, NULL, daddr);
410 	return IS_ERR(n) ? NULL : n;
411 }
412 
dst_neigh_lookup_skb(const struct dst_entry * dst,struct sk_buff * skb)413 static inline struct neighbour *dst_neigh_lookup_skb(const struct dst_entry *dst,
414 						     struct sk_buff *skb)
415 {
416 	struct neighbour *n;
417 
418 	if (WARN_ON_ONCE(!dst->ops->neigh_lookup))
419 		return NULL;
420 
421 	n = dst->ops->neigh_lookup(dst, skb, NULL);
422 
423 	return IS_ERR(n) ? NULL : n;
424 }
425 
dst_confirm_neigh(const struct dst_entry * dst,const void * daddr)426 static inline void dst_confirm_neigh(const struct dst_entry *dst,
427 				     const void *daddr)
428 {
429 	if (dst->ops->confirm_neigh)
430 		dst->ops->confirm_neigh(dst, daddr);
431 }
432 
dst_link_failure(struct sk_buff * skb)433 static inline void dst_link_failure(struct sk_buff *skb)
434 {
435 	struct dst_entry *dst = skb_dst(skb);
436 	if (dst && dst->ops && dst->ops->link_failure)
437 		dst->ops->link_failure(skb);
438 }
439 
dst_set_expires(struct dst_entry * dst,int timeout)440 static inline void dst_set_expires(struct dst_entry *dst, int timeout)
441 {
442 	unsigned long expires = jiffies + timeout;
443 
444 	if (expires == 0)
445 		expires = 1;
446 
447 	if (dst->expires == 0 || time_before(expires, dst->expires))
448 		dst->expires = expires;
449 }
450 
451 INDIRECT_CALLABLE_DECLARE(int ip6_output(struct net *, struct sock *,
452 					 struct sk_buff *));
453 INDIRECT_CALLABLE_DECLARE(int ip_output(struct net *, struct sock *,
454 					 struct sk_buff *));
455 /* Output packet to network from transport.  */
dst_output(struct net * net,struct sock * sk,struct sk_buff * skb)456 static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb)
457 {
458 	return INDIRECT_CALL_INET(skb_dst(skb)->output,
459 				  ip6_output, ip_output,
460 				  net, sk, skb);
461 }
462 
463 INDIRECT_CALLABLE_DECLARE(int ip6_input(struct sk_buff *));
464 INDIRECT_CALLABLE_DECLARE(int ip_local_deliver(struct sk_buff *));
465 /* Input packet from network to transport.  */
dst_input(struct sk_buff * skb)466 static inline int dst_input(struct sk_buff *skb)
467 {
468 	return INDIRECT_CALL_INET(skb_dst(skb)->input,
469 				  ip6_input, ip_local_deliver, skb);
470 }
471 
472 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
473 							  u32));
474 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
475 							   u32));
dst_check(struct dst_entry * dst,u32 cookie)476 static inline struct dst_entry *dst_check(struct dst_entry *dst, u32 cookie)
477 {
478 	if (dst->obsolete)
479 		dst = INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check,
480 					 ipv4_dst_check, dst, cookie);
481 	return dst;
482 }
483 
484 /* Flags for xfrm_lookup flags argument. */
485 enum {
486 	XFRM_LOOKUP_ICMP = 1 << 0,
487 	XFRM_LOOKUP_QUEUE = 1 << 1,
488 	XFRM_LOOKUP_KEEP_DST_REF = 1 << 2,
489 };
490 
491 struct flowi;
492 #ifndef CONFIG_XFRM
xfrm_lookup(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)493 static inline struct dst_entry *xfrm_lookup(struct net *net,
494 					    struct dst_entry *dst_orig,
495 					    const struct flowi *fl,
496 					    const struct sock *sk,
497 					    int flags)
498 {
499 	return dst_orig;
500 }
501 
502 static inline struct dst_entry *
xfrm_lookup_with_ifid(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags,u32 if_id)503 xfrm_lookup_with_ifid(struct net *net, struct dst_entry *dst_orig,
504 		      const struct flowi *fl, const struct sock *sk,
505 		      int flags, u32 if_id)
506 {
507 	return dst_orig;
508 }
509 
xfrm_lookup_route(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)510 static inline struct dst_entry *xfrm_lookup_route(struct net *net,
511 						  struct dst_entry *dst_orig,
512 						  const struct flowi *fl,
513 						  const struct sock *sk,
514 						  int flags)
515 {
516 	return dst_orig;
517 }
518 
dst_xfrm(const struct dst_entry * dst)519 static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst)
520 {
521 	return NULL;
522 }
523 
524 #else
525 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
526 			      const struct flowi *fl, const struct sock *sk,
527 			      int flags);
528 
529 struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
530 					struct dst_entry *dst_orig,
531 					const struct flowi *fl,
532 					const struct sock *sk, int flags,
533 					u32 if_id);
534 
535 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
536 				    const struct flowi *fl, const struct sock *sk,
537 				    int flags);
538 
539 /* skb attached with this dst needs transformation if dst->xfrm is valid */
dst_xfrm(const struct dst_entry * dst)540 static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst)
541 {
542 	return dst->xfrm;
543 }
544 #endif
545 
skb_dst_update_pmtu(struct sk_buff * skb,u32 mtu)546 static inline void skb_dst_update_pmtu(struct sk_buff *skb, u32 mtu)
547 {
548 	struct dst_entry *dst = skb_dst(skb);
549 
550 	if (dst && dst->ops->update_pmtu)
551 		dst->ops->update_pmtu(dst, NULL, skb, mtu, true);
552 }
553 
554 /* update dst pmtu but not do neighbor confirm */
skb_dst_update_pmtu_no_confirm(struct sk_buff * skb,u32 mtu)555 static inline void skb_dst_update_pmtu_no_confirm(struct sk_buff *skb, u32 mtu)
556 {
557 	struct dst_entry *dst = skb_dst(skb);
558 
559 	if (dst && dst->ops->update_pmtu)
560 		dst->ops->update_pmtu(dst, NULL, skb, mtu, false);
561 }
562 
563 struct dst_entry *dst_blackhole_check(struct dst_entry *dst, u32 cookie);
564 void dst_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
565 			       struct sk_buff *skb, u32 mtu, bool confirm_neigh);
566 void dst_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
567 			    struct sk_buff *skb);
568 u32 *dst_blackhole_cow_metrics(struct dst_entry *dst, unsigned long old);
569 struct neighbour *dst_blackhole_neigh_lookup(const struct dst_entry *dst,
570 					     struct sk_buff *skb,
571 					     const void *daddr);
572 unsigned int dst_blackhole_mtu(const struct dst_entry *dst);
573 
574 #endif /* _NET_DST_H */
575