1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * IPVS         An implementation of the IP virtual server support for the
4  *              LINUX operating system.  IPVS is now implemented as a module
5  *              over the Netfilter framework. IPVS can be used to build a
6  *              high-performance and highly available server based on a
7  *              cluster of servers.
8  *
9  * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
10  *              Peter Kese <peter.kese@ijs.si>
11  *              Julian Anastasov <ja@ssi.bg>
12  *
13  * The IPVS code for kernel 2.2 was done by Wensong Zhang and Peter Kese,
14  * with changes/fixes from Julian Anastasov, Lars Marowsky-Bree, Horms
15  * and others.
16  *
17  * Changes:
18  *	Paul `Rusty' Russell		properly handle non-linear skbs
19  *	Harald Welte			don't use nfcache
20  */
21 
22 #define KMSG_COMPONENT "IPVS"
23 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
24 
25 #include <linux/module.h>
26 #include <linux/kernel.h>
27 #include <linux/ip.h>
28 #include <linux/tcp.h>
29 #include <linux/sctp.h>
30 #include <linux/icmp.h>
31 #include <linux/slab.h>
32 
33 #include <net/ip.h>
34 #include <net/tcp.h>
35 #include <net/udp.h>
36 #include <net/icmp.h>                   /* for icmp_send */
37 #include <net/gue.h>
38 #include <net/gre.h>
39 #include <net/route.h>
40 #include <net/ip6_checksum.h>
41 #include <net/netns/generic.h>		/* net_generic() */
42 
43 #include <linux/netfilter.h>
44 #include <linux/netfilter_ipv4.h>
45 
46 #ifdef CONFIG_IP_VS_IPV6
47 #include <net/ipv6.h>
48 #include <linux/netfilter_ipv6.h>
49 #include <net/ip6_route.h>
50 #endif
51 
52 #include <net/ip_vs.h>
53 #include <linux/indirect_call_wrapper.h>
54 
55 
56 EXPORT_SYMBOL(register_ip_vs_scheduler);
57 EXPORT_SYMBOL(unregister_ip_vs_scheduler);
58 EXPORT_SYMBOL(ip_vs_proto_name);
59 EXPORT_SYMBOL(ip_vs_conn_new);
60 EXPORT_SYMBOL(ip_vs_conn_in_get);
61 EXPORT_SYMBOL(ip_vs_conn_out_get);
62 #ifdef CONFIG_IP_VS_PROTO_TCP
63 EXPORT_SYMBOL(ip_vs_tcp_conn_listen);
64 #endif
65 EXPORT_SYMBOL(ip_vs_conn_put);
66 #ifdef CONFIG_IP_VS_DEBUG
67 EXPORT_SYMBOL(ip_vs_get_debug_level);
68 #endif
69 EXPORT_SYMBOL(ip_vs_new_conn_out);
70 
71 #if defined(CONFIG_IP_VS_PROTO_TCP) && defined(CONFIG_IP_VS_PROTO_UDP)
72 #define SNAT_CALL(f, ...) \
73 	INDIRECT_CALL_2(f, tcp_snat_handler, udp_snat_handler, __VA_ARGS__)
74 #elif defined(CONFIG_IP_VS_PROTO_TCP)
75 #define SNAT_CALL(f, ...) INDIRECT_CALL_1(f, tcp_snat_handler, __VA_ARGS__)
76 #elif defined(CONFIG_IP_VS_PROTO_UDP)
77 #define SNAT_CALL(f, ...) INDIRECT_CALL_1(f, udp_snat_handler, __VA_ARGS__)
78 #else
79 #define SNAT_CALL(f, ...) f(__VA_ARGS__)
80 #endif
81 
82 static unsigned int ip_vs_net_id __read_mostly;
83 /* netns cnt used for uniqueness */
84 static atomic_t ipvs_netns_cnt = ATOMIC_INIT(0);
85 
86 /* ID used in ICMP lookups */
87 #define icmp_id(icmph)          (((icmph)->un).echo.id)
88 #define icmpv6_id(icmph)        (icmph->icmp6_dataun.u_echo.identifier)
89 
ip_vs_proto_name(unsigned int proto)90 const char *ip_vs_proto_name(unsigned int proto)
91 {
92 	static char buf[20];
93 
94 	switch (proto) {
95 	case IPPROTO_IP:
96 		return "IP";
97 	case IPPROTO_UDP:
98 		return "UDP";
99 	case IPPROTO_TCP:
100 		return "TCP";
101 	case IPPROTO_SCTP:
102 		return "SCTP";
103 	case IPPROTO_ICMP:
104 		return "ICMP";
105 #ifdef CONFIG_IP_VS_IPV6
106 	case IPPROTO_ICMPV6:
107 		return "ICMPv6";
108 #endif
109 	default:
110 		sprintf(buf, "IP_%u", proto);
111 		return buf;
112 	}
113 }
114 
ip_vs_init_hash_table(struct list_head * table,int rows)115 void ip_vs_init_hash_table(struct list_head *table, int rows)
116 {
117 	while (--rows >= 0)
118 		INIT_LIST_HEAD(&table[rows]);
119 }
120 
121 static inline void
ip_vs_in_stats(struct ip_vs_conn * cp,struct sk_buff * skb)122 ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
123 {
124 	struct ip_vs_dest *dest = cp->dest;
125 	struct netns_ipvs *ipvs = cp->ipvs;
126 
127 	if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
128 		struct ip_vs_cpu_stats *s;
129 		struct ip_vs_service *svc;
130 
131 		local_bh_disable();
132 
133 		s = this_cpu_ptr(dest->stats.cpustats);
134 		u64_stats_update_begin(&s->syncp);
135 		u64_stats_inc(&s->cnt.inpkts);
136 		u64_stats_add(&s->cnt.inbytes, skb->len);
137 		u64_stats_update_end(&s->syncp);
138 
139 		svc = rcu_dereference(dest->svc);
140 		s = this_cpu_ptr(svc->stats.cpustats);
141 		u64_stats_update_begin(&s->syncp);
142 		u64_stats_inc(&s->cnt.inpkts);
143 		u64_stats_add(&s->cnt.inbytes, skb->len);
144 		u64_stats_update_end(&s->syncp);
145 
146 		s = this_cpu_ptr(ipvs->tot_stats.cpustats);
147 		u64_stats_update_begin(&s->syncp);
148 		u64_stats_inc(&s->cnt.inpkts);
149 		u64_stats_add(&s->cnt.inbytes, skb->len);
150 		u64_stats_update_end(&s->syncp);
151 
152 		local_bh_enable();
153 	}
154 }
155 
156 
157 static inline void
ip_vs_out_stats(struct ip_vs_conn * cp,struct sk_buff * skb)158 ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
159 {
160 	struct ip_vs_dest *dest = cp->dest;
161 	struct netns_ipvs *ipvs = cp->ipvs;
162 
163 	if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
164 		struct ip_vs_cpu_stats *s;
165 		struct ip_vs_service *svc;
166 
167 		local_bh_disable();
168 
169 		s = this_cpu_ptr(dest->stats.cpustats);
170 		u64_stats_update_begin(&s->syncp);
171 		u64_stats_inc(&s->cnt.outpkts);
172 		u64_stats_add(&s->cnt.outbytes, skb->len);
173 		u64_stats_update_end(&s->syncp);
174 
175 		svc = rcu_dereference(dest->svc);
176 		s = this_cpu_ptr(svc->stats.cpustats);
177 		u64_stats_update_begin(&s->syncp);
178 		u64_stats_inc(&s->cnt.outpkts);
179 		u64_stats_add(&s->cnt.outbytes, skb->len);
180 		u64_stats_update_end(&s->syncp);
181 
182 		s = this_cpu_ptr(ipvs->tot_stats.cpustats);
183 		u64_stats_update_begin(&s->syncp);
184 		u64_stats_inc(&s->cnt.outpkts);
185 		u64_stats_add(&s->cnt.outbytes, skb->len);
186 		u64_stats_update_end(&s->syncp);
187 
188 		local_bh_enable();
189 	}
190 }
191 
192 
193 static inline void
ip_vs_conn_stats(struct ip_vs_conn * cp,struct ip_vs_service * svc)194 ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc)
195 {
196 	struct netns_ipvs *ipvs = svc->ipvs;
197 	struct ip_vs_cpu_stats *s;
198 
199 	local_bh_disable();
200 
201 	s = this_cpu_ptr(cp->dest->stats.cpustats);
202 	u64_stats_update_begin(&s->syncp);
203 	u64_stats_inc(&s->cnt.conns);
204 	u64_stats_update_end(&s->syncp);
205 
206 	s = this_cpu_ptr(svc->stats.cpustats);
207 	u64_stats_update_begin(&s->syncp);
208 	u64_stats_inc(&s->cnt.conns);
209 	u64_stats_update_end(&s->syncp);
210 
211 	s = this_cpu_ptr(ipvs->tot_stats.cpustats);
212 	u64_stats_update_begin(&s->syncp);
213 	u64_stats_inc(&s->cnt.conns);
214 	u64_stats_update_end(&s->syncp);
215 
216 	local_bh_enable();
217 }
218 
219 
220 static inline void
ip_vs_set_state(struct ip_vs_conn * cp,int direction,const struct sk_buff * skb,struct ip_vs_proto_data * pd)221 ip_vs_set_state(struct ip_vs_conn *cp, int direction,
222 		const struct sk_buff *skb,
223 		struct ip_vs_proto_data *pd)
224 {
225 	if (likely(pd->pp->state_transition))
226 		pd->pp->state_transition(cp, direction, skb, pd);
227 }
228 
229 static inline int
ip_vs_conn_fill_param_persist(const struct ip_vs_service * svc,struct sk_buff * skb,int protocol,const union nf_inet_addr * caddr,__be16 cport,const union nf_inet_addr * vaddr,__be16 vport,struct ip_vs_conn_param * p)230 ip_vs_conn_fill_param_persist(const struct ip_vs_service *svc,
231 			      struct sk_buff *skb, int protocol,
232 			      const union nf_inet_addr *caddr, __be16 cport,
233 			      const union nf_inet_addr *vaddr, __be16 vport,
234 			      struct ip_vs_conn_param *p)
235 {
236 	ip_vs_conn_fill_param(svc->ipvs, svc->af, protocol, caddr, cport, vaddr,
237 			      vport, p);
238 	p->pe = rcu_dereference(svc->pe);
239 	if (p->pe && p->pe->fill_param)
240 		return p->pe->fill_param(p, skb);
241 
242 	return 0;
243 }
244 
245 /*
246  *  IPVS persistent scheduling function
247  *  It creates a connection entry according to its template if exists,
248  *  or selects a server and creates a connection entry plus a template.
249  *  Locking: we are svc user (svc->refcnt), so we hold all dests too
250  *  Protocols supported: TCP, UDP
251  */
252 static struct ip_vs_conn *
ip_vs_sched_persist(struct ip_vs_service * svc,struct sk_buff * skb,__be16 src_port,__be16 dst_port,int * ignored,struct ip_vs_iphdr * iph)253 ip_vs_sched_persist(struct ip_vs_service *svc,
254 		    struct sk_buff *skb, __be16 src_port, __be16 dst_port,
255 		    int *ignored, struct ip_vs_iphdr *iph)
256 {
257 	struct ip_vs_conn *cp = NULL;
258 	struct ip_vs_dest *dest;
259 	struct ip_vs_conn *ct;
260 	__be16 dport = 0;		/* destination port to forward */
261 	unsigned int flags;
262 	struct ip_vs_conn_param param;
263 	const union nf_inet_addr fwmark = { .ip = htonl(svc->fwmark) };
264 	union nf_inet_addr snet;	/* source network of the client,
265 					   after masking */
266 	const union nf_inet_addr *src_addr, *dst_addr;
267 
268 	if (likely(!ip_vs_iph_inverse(iph))) {
269 		src_addr = &iph->saddr;
270 		dst_addr = &iph->daddr;
271 	} else {
272 		src_addr = &iph->daddr;
273 		dst_addr = &iph->saddr;
274 	}
275 
276 
277 	/* Mask saddr with the netmask to adjust template granularity */
278 #ifdef CONFIG_IP_VS_IPV6
279 	if (svc->af == AF_INET6)
280 		ipv6_addr_prefix(&snet.in6, &src_addr->in6,
281 				 (__force __u32) svc->netmask);
282 	else
283 #endif
284 		snet.ip = src_addr->ip & svc->netmask;
285 
286 	IP_VS_DBG_BUF(6, "p-schedule: src %s:%u dest %s:%u "
287 		      "mnet %s\n",
288 		      IP_VS_DBG_ADDR(svc->af, src_addr), ntohs(src_port),
289 		      IP_VS_DBG_ADDR(svc->af, dst_addr), ntohs(dst_port),
290 		      IP_VS_DBG_ADDR(svc->af, &snet));
291 
292 	/*
293 	 * As far as we know, FTP is a very complicated network protocol, and
294 	 * it uses control connection and data connections. For active FTP,
295 	 * FTP server initialize data connection to the client, its source port
296 	 * is often 20. For passive FTP, FTP server tells the clients the port
297 	 * that it passively listens to,  and the client issues the data
298 	 * connection. In the tunneling or direct routing mode, the load
299 	 * balancer is on the client-to-server half of connection, the port
300 	 * number is unknown to the load balancer. So, a conn template like
301 	 * <caddr, 0, vaddr, 0, daddr, 0> is created for persistent FTP
302 	 * service, and a template like <caddr, 0, vaddr, vport, daddr, dport>
303 	 * is created for other persistent services.
304 	 */
305 	{
306 		int protocol = iph->protocol;
307 		const union nf_inet_addr *vaddr = dst_addr;
308 		__be16 vport = 0;
309 
310 		if (dst_port == svc->port) {
311 			/* non-FTP template:
312 			 * <protocol, caddr, 0, vaddr, vport, daddr, dport>
313 			 * FTP template:
314 			 * <protocol, caddr, 0, vaddr, 0, daddr, 0>
315 			 */
316 			if (svc->port != FTPPORT)
317 				vport = dst_port;
318 		} else {
319 			/* Note: persistent fwmark-based services and
320 			 * persistent port zero service are handled here.
321 			 * fwmark template:
322 			 * <IPPROTO_IP,caddr,0,fwmark,0,daddr,0>
323 			 * port zero template:
324 			 * <protocol,caddr,0,vaddr,0,daddr,0>
325 			 */
326 			if (svc->fwmark) {
327 				protocol = IPPROTO_IP;
328 				vaddr = &fwmark;
329 			}
330 		}
331 		/* return *ignored = -1 so NF_DROP can be used */
332 		if (ip_vs_conn_fill_param_persist(svc, skb, protocol, &snet, 0,
333 						  vaddr, vport, &param) < 0) {
334 			*ignored = -1;
335 			return NULL;
336 		}
337 	}
338 
339 	/* Check if a template already exists */
340 	ct = ip_vs_ct_in_get(&param);
341 	if (!ct || !ip_vs_check_template(ct, NULL)) {
342 		struct ip_vs_scheduler *sched;
343 
344 		/*
345 		 * No template found or the dest of the connection
346 		 * template is not available.
347 		 * return *ignored=0 i.e. ICMP and NF_DROP
348 		 */
349 		sched = rcu_dereference(svc->scheduler);
350 		if (sched) {
351 			/* read svc->sched_data after svc->scheduler */
352 			smp_rmb();
353 			dest = sched->schedule(svc, skb, iph);
354 		} else {
355 			dest = NULL;
356 		}
357 		if (!dest) {
358 			IP_VS_DBG(1, "p-schedule: no dest found.\n");
359 			kfree(param.pe_data);
360 			*ignored = 0;
361 			return NULL;
362 		}
363 
364 		if (dst_port == svc->port && svc->port != FTPPORT)
365 			dport = dest->port;
366 
367 		/* Create a template
368 		 * This adds param.pe_data to the template,
369 		 * and thus param.pe_data will be destroyed
370 		 * when the template expires */
371 		ct = ip_vs_conn_new(&param, dest->af, &dest->addr, dport,
372 				    IP_VS_CONN_F_TEMPLATE, dest, skb->mark);
373 		if (ct == NULL) {
374 			kfree(param.pe_data);
375 			*ignored = -1;
376 			return NULL;
377 		}
378 
379 		ct->timeout = svc->timeout;
380 	} else {
381 		/* set destination with the found template */
382 		dest = ct->dest;
383 		kfree(param.pe_data);
384 	}
385 
386 	dport = dst_port;
387 	if (dport == svc->port && dest->port)
388 		dport = dest->port;
389 
390 	flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
391 		 && iph->protocol == IPPROTO_UDP) ?
392 		IP_VS_CONN_F_ONE_PACKET : 0;
393 
394 	/*
395 	 *    Create a new connection according to the template
396 	 */
397 	ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol, src_addr,
398 			      src_port, dst_addr, dst_port, &param);
399 
400 	cp = ip_vs_conn_new(&param, dest->af, &dest->addr, dport, flags, dest,
401 			    skb->mark);
402 	if (cp == NULL) {
403 		ip_vs_conn_put(ct);
404 		*ignored = -1;
405 		return NULL;
406 	}
407 
408 	/*
409 	 *    Add its control
410 	 */
411 	ip_vs_control_add(cp, ct);
412 	ip_vs_conn_put(ct);
413 
414 	ip_vs_conn_stats(cp, svc);
415 	return cp;
416 }
417 
418 
419 /*
420  *  IPVS main scheduling function
421  *  It selects a server according to the virtual service, and
422  *  creates a connection entry.
423  *  Protocols supported: TCP, UDP
424  *
425  *  Usage of *ignored
426  *
427  * 1 :   protocol tried to schedule (eg. on SYN), found svc but the
428  *       svc/scheduler decides that this packet should be accepted with
429  *       NF_ACCEPT because it must not be scheduled.
430  *
431  * 0 :   scheduler can not find destination, so try bypass or
432  *       return ICMP and then NF_DROP (ip_vs_leave).
433  *
434  * -1 :  scheduler tried to schedule but fatal error occurred, eg.
435  *       ip_vs_conn_new failure (ENOMEM) or ip_vs_sip_fill_param
436  *       failure such as missing Call-ID, ENOMEM on skb_linearize
437  *       or pe_data. In this case we should return NF_DROP without
438  *       any attempts to send ICMP with ip_vs_leave.
439  */
440 struct ip_vs_conn *
ip_vs_schedule(struct ip_vs_service * svc,struct sk_buff * skb,struct ip_vs_proto_data * pd,int * ignored,struct ip_vs_iphdr * iph)441 ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb,
442 	       struct ip_vs_proto_data *pd, int *ignored,
443 	       struct ip_vs_iphdr *iph)
444 {
445 	struct ip_vs_protocol *pp = pd->pp;
446 	struct ip_vs_conn *cp = NULL;
447 	struct ip_vs_scheduler *sched;
448 	struct ip_vs_dest *dest;
449 	__be16 _ports[2], *pptr, cport, vport;
450 	const void *caddr, *vaddr;
451 	unsigned int flags;
452 
453 	*ignored = 1;
454 	/*
455 	 * IPv6 frags, only the first hit here.
456 	 */
457 	pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports);
458 	if (pptr == NULL)
459 		return NULL;
460 
461 	if (likely(!ip_vs_iph_inverse(iph))) {
462 		cport = pptr[0];
463 		caddr = &iph->saddr;
464 		vport = pptr[1];
465 		vaddr = &iph->daddr;
466 	} else {
467 		cport = pptr[1];
468 		caddr = &iph->daddr;
469 		vport = pptr[0];
470 		vaddr = &iph->saddr;
471 	}
472 
473 	/*
474 	 * FTPDATA needs this check when using local real server.
475 	 * Never schedule Active FTPDATA connections from real server.
476 	 * For LVS-NAT they must be already created. For other methods
477 	 * with persistence the connection is created on SYN+ACK.
478 	 */
479 	if (cport == FTPDATA) {
480 		IP_VS_DBG_PKT(12, svc->af, pp, skb, iph->off,
481 			      "Not scheduling FTPDATA");
482 		return NULL;
483 	}
484 
485 	/*
486 	 *    Do not schedule replies from local real server.
487 	 */
488 	if ((!skb->dev || skb->dev->flags & IFF_LOOPBACK)) {
489 		iph->hdr_flags ^= IP_VS_HDR_INVERSE;
490 		cp = INDIRECT_CALL_1(pp->conn_in_get,
491 				     ip_vs_conn_in_get_proto, svc->ipvs,
492 				     svc->af, skb, iph);
493 		iph->hdr_flags ^= IP_VS_HDR_INVERSE;
494 
495 		if (cp) {
496 			IP_VS_DBG_PKT(12, svc->af, pp, skb, iph->off,
497 				      "Not scheduling reply for existing"
498 				      " connection");
499 			__ip_vs_conn_put(cp);
500 			return NULL;
501 		}
502 	}
503 
504 	/*
505 	 *    Persistent service
506 	 */
507 	if (svc->flags & IP_VS_SVC_F_PERSISTENT)
508 		return ip_vs_sched_persist(svc, skb, cport, vport, ignored,
509 					   iph);
510 
511 	*ignored = 0;
512 
513 	/*
514 	 *    Non-persistent service
515 	 */
516 	if (!svc->fwmark && vport != svc->port) {
517 		if (!svc->port)
518 			pr_err("Schedule: port zero only supported "
519 			       "in persistent services, "
520 			       "check your ipvs configuration\n");
521 		return NULL;
522 	}
523 
524 	sched = rcu_dereference(svc->scheduler);
525 	if (sched) {
526 		/* read svc->sched_data after svc->scheduler */
527 		smp_rmb();
528 		dest = sched->schedule(svc, skb, iph);
529 	} else {
530 		dest = NULL;
531 	}
532 	if (dest == NULL) {
533 		IP_VS_DBG(1, "Schedule: no dest found.\n");
534 		return NULL;
535 	}
536 
537 	flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
538 		 && iph->protocol == IPPROTO_UDP) ?
539 		IP_VS_CONN_F_ONE_PACKET : 0;
540 
541 	/*
542 	 *    Create a connection entry.
543 	 */
544 	{
545 		struct ip_vs_conn_param p;
546 
547 		ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
548 				      caddr, cport, vaddr, vport, &p);
549 		cp = ip_vs_conn_new(&p, dest->af, &dest->addr,
550 				    dest->port ? dest->port : vport,
551 				    flags, dest, skb->mark);
552 		if (!cp) {
553 			*ignored = -1;
554 			return NULL;
555 		}
556 	}
557 
558 	IP_VS_DBG_BUF(6, "Schedule fwd:%c c:%s:%u v:%s:%u "
559 		      "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
560 		      ip_vs_fwd_tag(cp),
561 		      IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
562 		      IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
563 		      IP_VS_DBG_ADDR(cp->daf, &cp->daddr), ntohs(cp->dport),
564 		      cp->flags, refcount_read(&cp->refcnt));
565 
566 	ip_vs_conn_stats(cp, svc);
567 	return cp;
568 }
569 
ip_vs_addr_is_unicast(struct net * net,int af,union nf_inet_addr * addr)570 static inline int ip_vs_addr_is_unicast(struct net *net, int af,
571 					union nf_inet_addr *addr)
572 {
573 #ifdef CONFIG_IP_VS_IPV6
574 	if (af == AF_INET6)
575 		return ipv6_addr_type(&addr->in6) & IPV6_ADDR_UNICAST;
576 #endif
577 	return (inet_addr_type(net, addr->ip) == RTN_UNICAST);
578 }
579 
580 /*
581  *  Pass or drop the packet.
582  *  Called by ip_vs_in, when the virtual service is available but
583  *  no destination is available for a new connection.
584  */
ip_vs_leave(struct ip_vs_service * svc,struct sk_buff * skb,struct ip_vs_proto_data * pd,struct ip_vs_iphdr * iph)585 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
586 		struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph)
587 {
588 	__be16 _ports[2], *pptr, dport;
589 	struct netns_ipvs *ipvs = svc->ipvs;
590 	struct net *net = ipvs->net;
591 
592 	pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports);
593 	if (!pptr)
594 		return NF_DROP;
595 	dport = likely(!ip_vs_iph_inverse(iph)) ? pptr[1] : pptr[0];
596 
597 	/* if it is fwmark-based service, the cache_bypass sysctl is up
598 	   and the destination is a non-local unicast, then create
599 	   a cache_bypass connection entry */
600 	if (sysctl_cache_bypass(ipvs) && svc->fwmark &&
601 	    !(iph->hdr_flags & (IP_VS_HDR_INVERSE | IP_VS_HDR_ICMP)) &&
602 	    ip_vs_addr_is_unicast(net, svc->af, &iph->daddr)) {
603 		int ret;
604 		struct ip_vs_conn *cp;
605 		unsigned int flags = (svc->flags & IP_VS_SVC_F_ONEPACKET &&
606 				      iph->protocol == IPPROTO_UDP) ?
607 				      IP_VS_CONN_F_ONE_PACKET : 0;
608 		union nf_inet_addr daddr = { .all = { 0, 0, 0, 0 } };
609 
610 		/* create a new connection entry */
611 		IP_VS_DBG(6, "%s(): create a cache_bypass entry\n", __func__);
612 		{
613 			struct ip_vs_conn_param p;
614 			ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
615 					      &iph->saddr, pptr[0],
616 					      &iph->daddr, pptr[1], &p);
617 			cp = ip_vs_conn_new(&p, svc->af, &daddr, 0,
618 					    IP_VS_CONN_F_BYPASS | flags,
619 					    NULL, skb->mark);
620 			if (!cp)
621 				return NF_DROP;
622 		}
623 
624 		/* statistics */
625 		ip_vs_in_stats(cp, skb);
626 
627 		/* set state */
628 		ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);
629 
630 		/* transmit the first SYN packet */
631 		ret = cp->packet_xmit(skb, cp, pd->pp, iph);
632 		/* do not touch skb anymore */
633 
634 		if ((cp->flags & IP_VS_CONN_F_ONE_PACKET) && cp->control)
635 			atomic_inc(&cp->control->in_pkts);
636 		else
637 			atomic_inc(&cp->in_pkts);
638 		ip_vs_conn_put(cp);
639 		return ret;
640 	}
641 
642 	/*
643 	 * When the virtual ftp service is presented, packets destined
644 	 * for other services on the VIP may get here (except services
645 	 * listed in the ipvs table), pass the packets, because it is
646 	 * not ipvs job to decide to drop the packets.
647 	 */
648 	if (svc->port == FTPPORT && dport != FTPPORT)
649 		return NF_ACCEPT;
650 
651 	if (unlikely(ip_vs_iph_icmp(iph)))
652 		return NF_DROP;
653 
654 	/*
655 	 * Notify the client that the destination is unreachable, and
656 	 * release the socket buffer.
657 	 * Since it is in IP layer, the TCP socket is not actually
658 	 * created, the TCP RST packet cannot be sent, instead that
659 	 * ICMP_PORT_UNREACH is sent here no matter it is TCP/UDP. --WZ
660 	 */
661 #ifdef CONFIG_IP_VS_IPV6
662 	if (svc->af == AF_INET6) {
663 		if (!skb->dev)
664 			skb->dev = net->loopback_dev;
665 		icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
666 	} else
667 #endif
668 		icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
669 
670 	return NF_DROP;
671 }
672 
673 #ifdef CONFIG_SYSCTL
674 
sysctl_snat_reroute(struct netns_ipvs * ipvs)675 static int sysctl_snat_reroute(struct netns_ipvs *ipvs)
676 {
677 	return ipvs->sysctl_snat_reroute;
678 }
679 
sysctl_nat_icmp_send(struct netns_ipvs * ipvs)680 static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs)
681 {
682 	return ipvs->sysctl_nat_icmp_send;
683 }
684 
685 #else
686 
sysctl_snat_reroute(struct netns_ipvs * ipvs)687 static int sysctl_snat_reroute(struct netns_ipvs *ipvs) { return 0; }
sysctl_nat_icmp_send(struct netns_ipvs * ipvs)688 static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs) { return 0; }
689 
690 #endif
691 
ip_vs_checksum_complete(struct sk_buff * skb,int offset)692 __sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset)
693 {
694 	return csum_fold(skb_checksum(skb, offset, skb->len - offset, 0));
695 }
696 
ip_vs_defrag_user(unsigned int hooknum)697 static inline enum ip_defrag_users ip_vs_defrag_user(unsigned int hooknum)
698 {
699 	if (NF_INET_LOCAL_IN == hooknum)
700 		return IP_DEFRAG_VS_IN;
701 	if (NF_INET_FORWARD == hooknum)
702 		return IP_DEFRAG_VS_FWD;
703 	return IP_DEFRAG_VS_OUT;
704 }
705 
ip_vs_gather_frags(struct netns_ipvs * ipvs,struct sk_buff * skb,u_int32_t user)706 static inline int ip_vs_gather_frags(struct netns_ipvs *ipvs,
707 				     struct sk_buff *skb, u_int32_t user)
708 {
709 	int err;
710 
711 	local_bh_disable();
712 	err = ip_defrag(ipvs->net, skb, user);
713 	local_bh_enable();
714 	if (!err)
715 		ip_send_check(ip_hdr(skb));
716 
717 	return err;
718 }
719 
ip_vs_route_me_harder(struct netns_ipvs * ipvs,int af,struct sk_buff * skb,unsigned int hooknum)720 static int ip_vs_route_me_harder(struct netns_ipvs *ipvs, int af,
721 				 struct sk_buff *skb, unsigned int hooknum)
722 {
723 	if (!sysctl_snat_reroute(ipvs))
724 		return 0;
725 	/* Reroute replies only to remote clients (FORWARD and LOCAL_OUT) */
726 	if (NF_INET_LOCAL_IN == hooknum)
727 		return 0;
728 #ifdef CONFIG_IP_VS_IPV6
729 	if (af == AF_INET6) {
730 		struct dst_entry *dst = skb_dst(skb);
731 
732 		if (dst->dev && !(dst->dev->flags & IFF_LOOPBACK) &&
733 		    ip6_route_me_harder(ipvs->net, skb->sk, skb) != 0)
734 			return 1;
735 	} else
736 #endif
737 		if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL) &&
738 		    ip_route_me_harder(ipvs->net, skb->sk, skb, RTN_LOCAL) != 0)
739 			return 1;
740 
741 	return 0;
742 }
743 
744 /*
745  * Packet has been made sufficiently writable in caller
746  * - inout: 1=in->out, 0=out->in
747  */
ip_vs_nat_icmp(struct sk_buff * skb,struct ip_vs_protocol * pp,struct ip_vs_conn * cp,int inout)748 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
749 		    struct ip_vs_conn *cp, int inout)
750 {
751 	struct iphdr *iph	 = ip_hdr(skb);
752 	unsigned int icmp_offset = iph->ihl*4;
753 	struct icmphdr *icmph	 = (struct icmphdr *)(skb_network_header(skb) +
754 						      icmp_offset);
755 	struct iphdr *ciph	 = (struct iphdr *)(icmph + 1);
756 
757 	if (inout) {
758 		iph->saddr = cp->vaddr.ip;
759 		ip_send_check(iph);
760 		ciph->daddr = cp->vaddr.ip;
761 		ip_send_check(ciph);
762 	} else {
763 		iph->daddr = cp->daddr.ip;
764 		ip_send_check(iph);
765 		ciph->saddr = cp->daddr.ip;
766 		ip_send_check(ciph);
767 	}
768 
769 	/* the TCP/UDP/SCTP port */
770 	if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol ||
771 	    IPPROTO_SCTP == ciph->protocol) {
772 		__be16 *ports = (void *)ciph + ciph->ihl*4;
773 
774 		if (inout)
775 			ports[1] = cp->vport;
776 		else
777 			ports[0] = cp->dport;
778 	}
779 
780 	/* And finally the ICMP checksum */
781 	icmph->checksum = 0;
782 	icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset);
783 	skb->ip_summed = CHECKSUM_UNNECESSARY;
784 
785 	if (inout)
786 		IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
787 			"Forwarding altered outgoing ICMP");
788 	else
789 		IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
790 			"Forwarding altered incoming ICMP");
791 }
792 
793 #ifdef CONFIG_IP_VS_IPV6
ip_vs_nat_icmp_v6(struct sk_buff * skb,struct ip_vs_protocol * pp,struct ip_vs_conn * cp,int inout)794 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp,
795 		    struct ip_vs_conn *cp, int inout)
796 {
797 	struct ipv6hdr *iph	 = ipv6_hdr(skb);
798 	unsigned int icmp_offset = 0;
799 	unsigned int offs	 = 0; /* header offset*/
800 	int protocol;
801 	struct icmp6hdr *icmph;
802 	struct ipv6hdr *ciph;
803 	unsigned short fragoffs;
804 
805 	ipv6_find_hdr(skb, &icmp_offset, IPPROTO_ICMPV6, &fragoffs, NULL);
806 	icmph = (struct icmp6hdr *)(skb_network_header(skb) + icmp_offset);
807 	offs = icmp_offset + sizeof(struct icmp6hdr);
808 	ciph = (struct ipv6hdr *)(skb_network_header(skb) + offs);
809 
810 	protocol = ipv6_find_hdr(skb, &offs, -1, &fragoffs, NULL);
811 
812 	if (inout) {
813 		iph->saddr = cp->vaddr.in6;
814 		ciph->daddr = cp->vaddr.in6;
815 	} else {
816 		iph->daddr = cp->daddr.in6;
817 		ciph->saddr = cp->daddr.in6;
818 	}
819 
820 	/* the TCP/UDP/SCTP port */
821 	if (!fragoffs && (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
822 			  IPPROTO_SCTP == protocol)) {
823 		__be16 *ports = (void *)(skb_network_header(skb) + offs);
824 
825 		IP_VS_DBG(11, "%s() changed port %d to %d\n", __func__,
826 			      ntohs(inout ? ports[1] : ports[0]),
827 			      ntohs(inout ? cp->vport : cp->dport));
828 		if (inout)
829 			ports[1] = cp->vport;
830 		else
831 			ports[0] = cp->dport;
832 	}
833 
834 	/* And finally the ICMP checksum */
835 	icmph->icmp6_cksum = ~csum_ipv6_magic(&iph->saddr, &iph->daddr,
836 					      skb->len - icmp_offset,
837 					      IPPROTO_ICMPV6, 0);
838 	skb->csum_start = skb_network_header(skb) - skb->head + icmp_offset;
839 	skb->csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
840 	skb->ip_summed = CHECKSUM_PARTIAL;
841 
842 	if (inout)
843 		IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
844 			      (void *)ciph - (void *)iph,
845 			      "Forwarding altered outgoing ICMPv6");
846 	else
847 		IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
848 			      (void *)ciph - (void *)iph,
849 			      "Forwarding altered incoming ICMPv6");
850 }
851 #endif
852 
853 /* Handle relevant response ICMP messages - forward to the right
854  * destination host.
855  */
handle_response_icmp(int af,struct sk_buff * skb,union nf_inet_addr * snet,__u8 protocol,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,unsigned int offset,unsigned int ihl,unsigned int hooknum)856 static int handle_response_icmp(int af, struct sk_buff *skb,
857 				union nf_inet_addr *snet,
858 				__u8 protocol, struct ip_vs_conn *cp,
859 				struct ip_vs_protocol *pp,
860 				unsigned int offset, unsigned int ihl,
861 				unsigned int hooknum)
862 {
863 	unsigned int verdict = NF_DROP;
864 
865 	if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
866 		goto after_nat;
867 
868 	/* Ensure the checksum is correct */
869 	if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
870 		/* Failed checksum! */
871 		IP_VS_DBG_BUF(1, "Forward ICMP: failed checksum from %s!\n",
872 			      IP_VS_DBG_ADDR(af, snet));
873 		goto out;
874 	}
875 
876 	if (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
877 	    IPPROTO_SCTP == protocol)
878 		offset += 2 * sizeof(__u16);
879 	if (skb_ensure_writable(skb, offset))
880 		goto out;
881 
882 #ifdef CONFIG_IP_VS_IPV6
883 	if (af == AF_INET6)
884 		ip_vs_nat_icmp_v6(skb, pp, cp, 1);
885 	else
886 #endif
887 		ip_vs_nat_icmp(skb, pp, cp, 1);
888 
889 	if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum))
890 		goto out;
891 
892 after_nat:
893 	/* do the statistics and put it back */
894 	ip_vs_out_stats(cp, skb);
895 
896 	skb->ipvs_property = 1;
897 	if (!(cp->flags & IP_VS_CONN_F_NFCT))
898 		ip_vs_notrack(skb);
899 	else
900 		ip_vs_update_conntrack(skb, cp, 0);
901 	verdict = NF_ACCEPT;
902 
903 out:
904 	__ip_vs_conn_put(cp);
905 
906 	return verdict;
907 }
908 
909 /*
910  *	Handle ICMP messages in the inside-to-outside direction (outgoing).
911  *	Find any that might be relevant, check against existing connections.
912  *	Currently handles error types - unreachable, quench, ttl exceeded.
913  */
ip_vs_out_icmp(struct netns_ipvs * ipvs,struct sk_buff * skb,int * related,unsigned int hooknum)914 static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb,
915 			  int *related, unsigned int hooknum)
916 {
917 	struct iphdr *iph;
918 	struct icmphdr	_icmph, *ic;
919 	struct iphdr	_ciph, *cih;	/* The ip header contained within the ICMP */
920 	struct ip_vs_iphdr ciph;
921 	struct ip_vs_conn *cp;
922 	struct ip_vs_protocol *pp;
923 	unsigned int offset, ihl;
924 	union nf_inet_addr snet;
925 
926 	*related = 1;
927 
928 	/* reassemble IP fragments */
929 	if (ip_is_fragment(ip_hdr(skb))) {
930 		if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum)))
931 			return NF_STOLEN;
932 	}
933 
934 	iph = ip_hdr(skb);
935 	offset = ihl = iph->ihl * 4;
936 	ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
937 	if (ic == NULL)
938 		return NF_DROP;
939 
940 	IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %pI4->%pI4\n",
941 		  ic->type, ntohs(icmp_id(ic)),
942 		  &iph->saddr, &iph->daddr);
943 
944 	/*
945 	 * Work through seeing if this is for us.
946 	 * These checks are supposed to be in an order that means easy
947 	 * things are checked first to speed up processing.... however
948 	 * this means that some packets will manage to get a long way
949 	 * down this stack and then be rejected, but that's life.
950 	 */
951 	if ((ic->type != ICMP_DEST_UNREACH) &&
952 	    (ic->type != ICMP_SOURCE_QUENCH) &&
953 	    (ic->type != ICMP_TIME_EXCEEDED)) {
954 		*related = 0;
955 		return NF_ACCEPT;
956 	}
957 
958 	/* Now find the contained IP header */
959 	offset += sizeof(_icmph);
960 	cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
961 	if (cih == NULL)
962 		return NF_ACCEPT; /* The packet looks wrong, ignore */
963 
964 	pp = ip_vs_proto_get(cih->protocol);
965 	if (!pp)
966 		return NF_ACCEPT;
967 
968 	/* Is the embedded protocol header present? */
969 	if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
970 		     pp->dont_defrag))
971 		return NF_ACCEPT;
972 
973 	IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
974 		      "Checking outgoing ICMP for");
975 
976 	ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, true, &ciph);
977 
978 	/* The embedded headers contain source and dest in reverse order */
979 	cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto,
980 			     ipvs, AF_INET, skb, &ciph);
981 	if (!cp)
982 		return NF_ACCEPT;
983 
984 	snet.ip = iph->saddr;
985 	return handle_response_icmp(AF_INET, skb, &snet, cih->protocol, cp,
986 				    pp, ciph.len, ihl, hooknum);
987 }
988 
989 #ifdef CONFIG_IP_VS_IPV6
ip_vs_out_icmp_v6(struct netns_ipvs * ipvs,struct sk_buff * skb,int * related,unsigned int hooknum,struct ip_vs_iphdr * ipvsh)990 static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb,
991 			     int *related,  unsigned int hooknum,
992 			     struct ip_vs_iphdr *ipvsh)
993 {
994 	struct icmp6hdr	_icmph, *ic;
995 	struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};/*Contained IP */
996 	struct ip_vs_conn *cp;
997 	struct ip_vs_protocol *pp;
998 	union nf_inet_addr snet;
999 	unsigned int offset;
1000 
1001 	*related = 1;
1002 	ic = frag_safe_skb_hp(skb, ipvsh->len, sizeof(_icmph), &_icmph);
1003 	if (ic == NULL)
1004 		return NF_DROP;
1005 
1006 	/*
1007 	 * Work through seeing if this is for us.
1008 	 * These checks are supposed to be in an order that means easy
1009 	 * things are checked first to speed up processing.... however
1010 	 * this means that some packets will manage to get a long way
1011 	 * down this stack and then be rejected, but that's life.
1012 	 */
1013 	if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
1014 		*related = 0;
1015 		return NF_ACCEPT;
1016 	}
1017 	/* Fragment header that is before ICMP header tells us that:
1018 	 * it's not an error message since they can't be fragmented.
1019 	 */
1020 	if (ipvsh->flags & IP6_FH_F_FRAG)
1021 		return NF_DROP;
1022 
1023 	IP_VS_DBG(8, "Outgoing ICMPv6 (%d,%d) %pI6c->%pI6c\n",
1024 		  ic->icmp6_type, ntohs(icmpv6_id(ic)),
1025 		  &ipvsh->saddr, &ipvsh->daddr);
1026 
1027 	if (!ip_vs_fill_iph_skb_icmp(AF_INET6, skb, ipvsh->len + sizeof(_icmph),
1028 				     true, &ciph))
1029 		return NF_ACCEPT; /* The packet looks wrong, ignore */
1030 
1031 	pp = ip_vs_proto_get(ciph.protocol);
1032 	if (!pp)
1033 		return NF_ACCEPT;
1034 
1035 	/* The embedded headers contain source and dest in reverse order */
1036 	cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto,
1037 			     ipvs, AF_INET6, skb, &ciph);
1038 	if (!cp)
1039 		return NF_ACCEPT;
1040 
1041 	snet.in6 = ciph.saddr.in6;
1042 	offset = ciph.len;
1043 	return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp,
1044 				    pp, offset, sizeof(struct ipv6hdr),
1045 				    hooknum);
1046 }
1047 #endif
1048 
1049 /*
1050  * Check if sctp chunc is ABORT chunk
1051  */
is_sctp_abort(const struct sk_buff * skb,int nh_len)1052 static inline int is_sctp_abort(const struct sk_buff *skb, int nh_len)
1053 {
1054 	struct sctp_chunkhdr *sch, schunk;
1055 	sch = skb_header_pointer(skb, nh_len + sizeof(struct sctphdr),
1056 				 sizeof(schunk), &schunk);
1057 	if (sch == NULL)
1058 		return 0;
1059 	if (sch->type == SCTP_CID_ABORT)
1060 		return 1;
1061 	return 0;
1062 }
1063 
is_tcp_reset(const struct sk_buff * skb,int nh_len)1064 static inline int is_tcp_reset(const struct sk_buff *skb, int nh_len)
1065 {
1066 	struct tcphdr _tcph, *th;
1067 
1068 	th = skb_header_pointer(skb, nh_len, sizeof(_tcph), &_tcph);
1069 	if (th == NULL)
1070 		return 0;
1071 	return th->rst;
1072 }
1073 
is_new_conn(const struct sk_buff * skb,struct ip_vs_iphdr * iph)1074 static inline bool is_new_conn(const struct sk_buff *skb,
1075 			       struct ip_vs_iphdr *iph)
1076 {
1077 	switch (iph->protocol) {
1078 	case IPPROTO_TCP: {
1079 		struct tcphdr _tcph, *th;
1080 
1081 		th = skb_header_pointer(skb, iph->len, sizeof(_tcph), &_tcph);
1082 		if (th == NULL)
1083 			return false;
1084 		return th->syn;
1085 	}
1086 	case IPPROTO_SCTP: {
1087 		struct sctp_chunkhdr *sch, schunk;
1088 
1089 		sch = skb_header_pointer(skb, iph->len + sizeof(struct sctphdr),
1090 					 sizeof(schunk), &schunk);
1091 		if (sch == NULL)
1092 			return false;
1093 		return sch->type == SCTP_CID_INIT;
1094 	}
1095 	default:
1096 		return false;
1097 	}
1098 }
1099 
is_new_conn_expected(const struct ip_vs_conn * cp,int conn_reuse_mode)1100 static inline bool is_new_conn_expected(const struct ip_vs_conn *cp,
1101 					int conn_reuse_mode)
1102 {
1103 	/* Controlled (FTP DATA or persistence)? */
1104 	if (cp->control)
1105 		return false;
1106 
1107 	switch (cp->protocol) {
1108 	case IPPROTO_TCP:
1109 		return (cp->state == IP_VS_TCP_S_TIME_WAIT) ||
1110 		       (cp->state == IP_VS_TCP_S_CLOSE) ||
1111 			((conn_reuse_mode & 2) &&
1112 			 (cp->state == IP_VS_TCP_S_FIN_WAIT) &&
1113 			 (cp->flags & IP_VS_CONN_F_NOOUTPUT));
1114 	case IPPROTO_SCTP:
1115 		return cp->state == IP_VS_SCTP_S_CLOSED;
1116 	default:
1117 		return false;
1118 	}
1119 }
1120 
1121 /* Generic function to create new connections for outgoing RS packets
1122  *
1123  * Pre-requisites for successful connection creation:
1124  * 1) Virtual Service is NOT fwmark based:
1125  *    In fwmark-VS actual vaddr and vport are unknown to IPVS
1126  * 2) Real Server and Virtual Service were NOT configured without port:
1127  *    This is to allow match of different VS to the same RS ip-addr
1128  */
ip_vs_new_conn_out(struct ip_vs_service * svc,struct ip_vs_dest * dest,struct sk_buff * skb,const struct ip_vs_iphdr * iph,__be16 dport,__be16 cport)1129 struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc,
1130 				      struct ip_vs_dest *dest,
1131 				      struct sk_buff *skb,
1132 				      const struct ip_vs_iphdr *iph,
1133 				      __be16 dport,
1134 				      __be16 cport)
1135 {
1136 	struct ip_vs_conn_param param;
1137 	struct ip_vs_conn *ct = NULL, *cp = NULL;
1138 	const union nf_inet_addr *vaddr, *daddr, *caddr;
1139 	union nf_inet_addr snet;
1140 	__be16 vport;
1141 	unsigned int flags;
1142 
1143 	EnterFunction(12);
1144 	vaddr = &svc->addr;
1145 	vport = svc->port;
1146 	daddr = &iph->saddr;
1147 	caddr = &iph->daddr;
1148 
1149 	/* check pre-requisites are satisfied */
1150 	if (svc->fwmark)
1151 		return NULL;
1152 	if (!vport || !dport)
1153 		return NULL;
1154 
1155 	/* for persistent service first create connection template */
1156 	if (svc->flags & IP_VS_SVC_F_PERSISTENT) {
1157 		/* apply netmask the same way ingress-side does */
1158 #ifdef CONFIG_IP_VS_IPV6
1159 		if (svc->af == AF_INET6)
1160 			ipv6_addr_prefix(&snet.in6, &caddr->in6,
1161 					 (__force __u32)svc->netmask);
1162 		else
1163 #endif
1164 			snet.ip = caddr->ip & svc->netmask;
1165 		/* fill params and create template if not existent */
1166 		if (ip_vs_conn_fill_param_persist(svc, skb, iph->protocol,
1167 						  &snet, 0, vaddr,
1168 						  vport, &param) < 0)
1169 			return NULL;
1170 		ct = ip_vs_ct_in_get(&param);
1171 		/* check if template exists and points to the same dest */
1172 		if (!ct || !ip_vs_check_template(ct, dest)) {
1173 			ct = ip_vs_conn_new(&param, dest->af, daddr, dport,
1174 					    IP_VS_CONN_F_TEMPLATE, dest, 0);
1175 			if (!ct) {
1176 				kfree(param.pe_data);
1177 				return NULL;
1178 			}
1179 			ct->timeout = svc->timeout;
1180 		} else {
1181 			kfree(param.pe_data);
1182 		}
1183 	}
1184 
1185 	/* connection flags */
1186 	flags = ((svc->flags & IP_VS_SVC_F_ONEPACKET) &&
1187 		 iph->protocol == IPPROTO_UDP) ? IP_VS_CONN_F_ONE_PACKET : 0;
1188 	/* create connection */
1189 	ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
1190 			      caddr, cport, vaddr, vport, &param);
1191 	cp = ip_vs_conn_new(&param, dest->af, daddr, dport, flags, dest, 0);
1192 	if (!cp) {
1193 		if (ct)
1194 			ip_vs_conn_put(ct);
1195 		return NULL;
1196 	}
1197 	if (ct) {
1198 		ip_vs_control_add(cp, ct);
1199 		ip_vs_conn_put(ct);
1200 	}
1201 	ip_vs_conn_stats(cp, svc);
1202 
1203 	/* return connection (will be used to handle outgoing packet) */
1204 	IP_VS_DBG_BUF(6, "New connection RS-initiated:%c c:%s:%u v:%s:%u "
1205 		      "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
1206 		      ip_vs_fwd_tag(cp),
1207 		      IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
1208 		      IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
1209 		      IP_VS_DBG_ADDR(cp->af, &cp->daddr), ntohs(cp->dport),
1210 		      cp->flags, refcount_read(&cp->refcnt));
1211 	LeaveFunction(12);
1212 	return cp;
1213 }
1214 
1215 /* Handle outgoing packets which are considered requests initiated by
1216  * real servers, so that subsequent responses from external client can be
1217  * routed to the right real server.
1218  * Used also for outgoing responses in OPS mode.
1219  *
1220  * Connection management is handled by persistent-engine specific callback.
1221  */
__ip_vs_rs_conn_out(unsigned int hooknum,struct netns_ipvs * ipvs,int af,struct sk_buff * skb,const struct ip_vs_iphdr * iph)1222 static struct ip_vs_conn *__ip_vs_rs_conn_out(unsigned int hooknum,
1223 					      struct netns_ipvs *ipvs,
1224 					      int af, struct sk_buff *skb,
1225 					      const struct ip_vs_iphdr *iph)
1226 {
1227 	struct ip_vs_dest *dest;
1228 	struct ip_vs_conn *cp = NULL;
1229 	__be16 _ports[2], *pptr;
1230 
1231 	if (hooknum == NF_INET_LOCAL_IN)
1232 		return NULL;
1233 
1234 	pptr = frag_safe_skb_hp(skb, iph->len,
1235 				sizeof(_ports), _ports);
1236 	if (!pptr)
1237 		return NULL;
1238 
1239 	dest = ip_vs_find_real_service(ipvs, af, iph->protocol,
1240 				       &iph->saddr, pptr[0]);
1241 	if (dest) {
1242 		struct ip_vs_service *svc;
1243 		struct ip_vs_pe *pe;
1244 
1245 		svc = rcu_dereference(dest->svc);
1246 		if (svc) {
1247 			pe = rcu_dereference(svc->pe);
1248 			if (pe && pe->conn_out)
1249 				cp = pe->conn_out(svc, dest, skb, iph,
1250 						  pptr[0], pptr[1]);
1251 		}
1252 	}
1253 
1254 	return cp;
1255 }
1256 
1257 /* Handle response packets: rewrite addresses and send away...
1258  */
1259 static unsigned int
handle_response(int af,struct sk_buff * skb,struct ip_vs_proto_data * pd,struct ip_vs_conn * cp,struct ip_vs_iphdr * iph,unsigned int hooknum)1260 handle_response(int af, struct sk_buff *skb, struct ip_vs_proto_data *pd,
1261 		struct ip_vs_conn *cp, struct ip_vs_iphdr *iph,
1262 		unsigned int hooknum)
1263 {
1264 	struct ip_vs_protocol *pp = pd->pp;
1265 
1266 	if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
1267 		goto after_nat;
1268 
1269 	IP_VS_DBG_PKT(11, af, pp, skb, iph->off, "Outgoing packet");
1270 
1271 	if (skb_ensure_writable(skb, iph->len))
1272 		goto drop;
1273 
1274 	/* mangle the packet */
1275 	if (pp->snat_handler &&
1276 	    !SNAT_CALL(pp->snat_handler, skb, pp, cp, iph))
1277 		goto drop;
1278 
1279 #ifdef CONFIG_IP_VS_IPV6
1280 	if (af == AF_INET6)
1281 		ipv6_hdr(skb)->saddr = cp->vaddr.in6;
1282 	else
1283 #endif
1284 	{
1285 		ip_hdr(skb)->saddr = cp->vaddr.ip;
1286 		ip_send_check(ip_hdr(skb));
1287 	}
1288 
1289 	/*
1290 	 * nf_iterate does not expect change in the skb->dst->dev.
1291 	 * It looks like it is not fatal to enable this code for hooks
1292 	 * where our handlers are at the end of the chain list and
1293 	 * when all next handlers use skb->dst->dev and not outdev.
1294 	 * It will definitely route properly the inout NAT traffic
1295 	 * when multiple paths are used.
1296 	 */
1297 
1298 	/* For policy routing, packets originating from this
1299 	 * machine itself may be routed differently to packets
1300 	 * passing through.  We want this packet to be routed as
1301 	 * if it came from this machine itself.  So re-compute
1302 	 * the routing information.
1303 	 */
1304 	if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum))
1305 		goto drop;
1306 
1307 	IP_VS_DBG_PKT(10, af, pp, skb, iph->off, "After SNAT");
1308 
1309 after_nat:
1310 	ip_vs_out_stats(cp, skb);
1311 	ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd);
1312 	skb->ipvs_property = 1;
1313 	if (!(cp->flags & IP_VS_CONN_F_NFCT))
1314 		ip_vs_notrack(skb);
1315 	else
1316 		ip_vs_update_conntrack(skb, cp, 0);
1317 	ip_vs_conn_put(cp);
1318 
1319 	LeaveFunction(11);
1320 	return NF_ACCEPT;
1321 
1322 drop:
1323 	ip_vs_conn_put(cp);
1324 	kfree_skb(skb);
1325 	LeaveFunction(11);
1326 	return NF_STOLEN;
1327 }
1328 
1329 /*
1330  *	Check if outgoing packet belongs to the established ip_vs_conn.
1331  */
1332 static unsigned int
ip_vs_out_hook(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)1333 ip_vs_out_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state)
1334 {
1335 	struct netns_ipvs *ipvs = net_ipvs(state->net);
1336 	unsigned int hooknum = state->hook;
1337 	struct ip_vs_iphdr iph;
1338 	struct ip_vs_protocol *pp;
1339 	struct ip_vs_proto_data *pd;
1340 	struct ip_vs_conn *cp;
1341 	int af = state->pf;
1342 	struct sock *sk;
1343 
1344 	EnterFunction(11);
1345 
1346 	/* Already marked as IPVS request or reply? */
1347 	if (skb->ipvs_property)
1348 		return NF_ACCEPT;
1349 
1350 	sk = skb_to_full_sk(skb);
1351 	/* Bad... Do not break raw sockets */
1352 	if (unlikely(sk && hooknum == NF_INET_LOCAL_OUT &&
1353 		     af == AF_INET)) {
1354 
1355 		if (sk->sk_family == PF_INET && inet_sk(sk)->nodefrag)
1356 			return NF_ACCEPT;
1357 	}
1358 
1359 	if (unlikely(!skb_dst(skb)))
1360 		return NF_ACCEPT;
1361 
1362 	if (!ipvs->enable)
1363 		return NF_ACCEPT;
1364 
1365 	ip_vs_fill_iph_skb(af, skb, false, &iph);
1366 #ifdef CONFIG_IP_VS_IPV6
1367 	if (af == AF_INET6) {
1368 		if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
1369 			int related;
1370 			int verdict = ip_vs_out_icmp_v6(ipvs, skb, &related,
1371 							hooknum, &iph);
1372 
1373 			if (related)
1374 				return verdict;
1375 		}
1376 	} else
1377 #endif
1378 		if (unlikely(iph.protocol == IPPROTO_ICMP)) {
1379 			int related;
1380 			int verdict = ip_vs_out_icmp(ipvs, skb, &related, hooknum);
1381 
1382 			if (related)
1383 				return verdict;
1384 		}
1385 
1386 	pd = ip_vs_proto_data_get(ipvs, iph.protocol);
1387 	if (unlikely(!pd))
1388 		return NF_ACCEPT;
1389 	pp = pd->pp;
1390 
1391 	/* reassemble IP fragments */
1392 #ifdef CONFIG_IP_VS_IPV6
1393 	if (af == AF_INET)
1394 #endif
1395 		if (unlikely(ip_is_fragment(ip_hdr(skb)) && !pp->dont_defrag)) {
1396 			if (ip_vs_gather_frags(ipvs, skb,
1397 					       ip_vs_defrag_user(hooknum)))
1398 				return NF_STOLEN;
1399 
1400 			ip_vs_fill_iph_skb(AF_INET, skb, false, &iph);
1401 		}
1402 
1403 	/*
1404 	 * Check if the packet belongs to an existing entry
1405 	 */
1406 	cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto,
1407 			     ipvs, af, skb, &iph);
1408 
1409 	if (likely(cp))
1410 		return handle_response(af, skb, pd, cp, &iph, hooknum);
1411 
1412 	/* Check for real-server-started requests */
1413 	if (atomic_read(&ipvs->conn_out_counter)) {
1414 		/* Currently only for UDP:
1415 		 * connection oriented protocols typically use
1416 		 * ephemeral ports for outgoing connections, so
1417 		 * related incoming responses would not match any VS
1418 		 */
1419 		if (pp->protocol == IPPROTO_UDP) {
1420 			cp = __ip_vs_rs_conn_out(hooknum, ipvs, af, skb, &iph);
1421 			if (likely(cp))
1422 				return handle_response(af, skb, pd, cp, &iph,
1423 						       hooknum);
1424 		}
1425 	}
1426 
1427 	if (sysctl_nat_icmp_send(ipvs) &&
1428 	    (pp->protocol == IPPROTO_TCP ||
1429 	     pp->protocol == IPPROTO_UDP ||
1430 	     pp->protocol == IPPROTO_SCTP)) {
1431 		__be16 _ports[2], *pptr;
1432 
1433 		pptr = frag_safe_skb_hp(skb, iph.len,
1434 					 sizeof(_ports), _ports);
1435 		if (pptr == NULL)
1436 			return NF_ACCEPT;	/* Not for me */
1437 		if (ip_vs_has_real_service(ipvs, af, iph.protocol, &iph.saddr,
1438 					   pptr[0])) {
1439 			/*
1440 			 * Notify the real server: there is no
1441 			 * existing entry if it is not RST
1442 			 * packet or not TCP packet.
1443 			 */
1444 			if ((iph.protocol != IPPROTO_TCP &&
1445 			     iph.protocol != IPPROTO_SCTP)
1446 			     || ((iph.protocol == IPPROTO_TCP
1447 				  && !is_tcp_reset(skb, iph.len))
1448 				 || (iph.protocol == IPPROTO_SCTP
1449 					&& !is_sctp_abort(skb,
1450 						iph.len)))) {
1451 #ifdef CONFIG_IP_VS_IPV6
1452 				if (af == AF_INET6) {
1453 					if (!skb->dev)
1454 						skb->dev = ipvs->net->loopback_dev;
1455 					icmpv6_send(skb,
1456 						    ICMPV6_DEST_UNREACH,
1457 						    ICMPV6_PORT_UNREACH,
1458 						    0);
1459 				} else
1460 #endif
1461 					icmp_send(skb,
1462 						  ICMP_DEST_UNREACH,
1463 						  ICMP_PORT_UNREACH, 0);
1464 				return NF_DROP;
1465 			}
1466 		}
1467 	}
1468 
1469 	IP_VS_DBG_PKT(12, af, pp, skb, iph.off,
1470 		      "ip_vs_out: packet continues traversal as normal");
1471 	return NF_ACCEPT;
1472 }
1473 
1474 static unsigned int
ip_vs_try_to_schedule(struct netns_ipvs * ipvs,int af,struct sk_buff * skb,struct ip_vs_proto_data * pd,int * verdict,struct ip_vs_conn ** cpp,struct ip_vs_iphdr * iph)1475 ip_vs_try_to_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb,
1476 		      struct ip_vs_proto_data *pd,
1477 		      int *verdict, struct ip_vs_conn **cpp,
1478 		      struct ip_vs_iphdr *iph)
1479 {
1480 	struct ip_vs_protocol *pp = pd->pp;
1481 
1482 	if (!iph->fragoffs) {
1483 		/* No (second) fragments need to enter here, as nf_defrag_ipv6
1484 		 * replayed fragment zero will already have created the cp
1485 		 */
1486 
1487 		/* Schedule and create new connection entry into cpp */
1488 		if (!pp->conn_schedule(ipvs, af, skb, pd, verdict, cpp, iph))
1489 			return 0;
1490 	}
1491 
1492 	if (unlikely(!*cpp)) {
1493 		/* sorry, all this trouble for a no-hit :) */
1494 		IP_VS_DBG_PKT(12, af, pp, skb, iph->off,
1495 			      "ip_vs_in: packet continues traversal as normal");
1496 
1497 		/* Fragment couldn't be mapped to a conn entry */
1498 		if (iph->fragoffs)
1499 			IP_VS_DBG_PKT(7, af, pp, skb, iph->off,
1500 				      "unhandled fragment");
1501 
1502 		*verdict = NF_ACCEPT;
1503 		return 0;
1504 	}
1505 
1506 	return 1;
1507 }
1508 
1509 /* Check the UDP tunnel and return its header length */
ipvs_udp_decap(struct netns_ipvs * ipvs,struct sk_buff * skb,unsigned int offset,__u16 af,const union nf_inet_addr * daddr,__u8 * proto)1510 static int ipvs_udp_decap(struct netns_ipvs *ipvs, struct sk_buff *skb,
1511 			  unsigned int offset, __u16 af,
1512 			  const union nf_inet_addr *daddr, __u8 *proto)
1513 {
1514 	struct udphdr _udph, *udph;
1515 	struct ip_vs_dest *dest;
1516 
1517 	udph = skb_header_pointer(skb, offset, sizeof(_udph), &_udph);
1518 	if (!udph)
1519 		goto unk;
1520 	offset += sizeof(struct udphdr);
1521 	dest = ip_vs_find_tunnel(ipvs, af, daddr, udph->dest);
1522 	if (!dest)
1523 		goto unk;
1524 	if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1525 		struct guehdr _gueh, *gueh;
1526 
1527 		gueh = skb_header_pointer(skb, offset, sizeof(_gueh), &_gueh);
1528 		if (!gueh)
1529 			goto unk;
1530 		if (gueh->control != 0 || gueh->version != 0)
1531 			goto unk;
1532 		/* Later we can support also IPPROTO_IPV6 */
1533 		if (gueh->proto_ctype != IPPROTO_IPIP)
1534 			goto unk;
1535 		*proto = gueh->proto_ctype;
1536 		return sizeof(struct udphdr) + sizeof(struct guehdr) +
1537 		       (gueh->hlen << 2);
1538 	}
1539 
1540 unk:
1541 	return 0;
1542 }
1543 
1544 /* Check the GRE tunnel and return its header length */
ipvs_gre_decap(struct netns_ipvs * ipvs,struct sk_buff * skb,unsigned int offset,__u16 af,const union nf_inet_addr * daddr,__u8 * proto)1545 static int ipvs_gre_decap(struct netns_ipvs *ipvs, struct sk_buff *skb,
1546 			  unsigned int offset, __u16 af,
1547 			  const union nf_inet_addr *daddr, __u8 *proto)
1548 {
1549 	struct gre_base_hdr _greh, *greh;
1550 	struct ip_vs_dest *dest;
1551 
1552 	greh = skb_header_pointer(skb, offset, sizeof(_greh), &_greh);
1553 	if (!greh)
1554 		goto unk;
1555 	dest = ip_vs_find_tunnel(ipvs, af, daddr, 0);
1556 	if (!dest)
1557 		goto unk;
1558 	if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1559 		__be16 type;
1560 
1561 		/* Only support version 0 and C (csum) */
1562 		if ((greh->flags & ~GRE_CSUM) != 0)
1563 			goto unk;
1564 		type = greh->protocol;
1565 		/* Later we can support also IPPROTO_IPV6 */
1566 		if (type != htons(ETH_P_IP))
1567 			goto unk;
1568 		*proto = IPPROTO_IPIP;
1569 		return gre_calc_hlen(gre_flags_to_tnl_flags(greh->flags));
1570 	}
1571 
1572 unk:
1573 	return 0;
1574 }
1575 
1576 /*
1577  *	Handle ICMP messages in the outside-to-inside direction (incoming).
1578  *	Find any that might be relevant, check against existing connections,
1579  *	forward to the right destination host if relevant.
1580  *	Currently handles error types - unreachable, quench, ttl exceeded.
1581  */
1582 static int
ip_vs_in_icmp(struct netns_ipvs * ipvs,struct sk_buff * skb,int * related,unsigned int hooknum)1583 ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related,
1584 	      unsigned int hooknum)
1585 {
1586 	struct iphdr *iph;
1587 	struct icmphdr	_icmph, *ic;
1588 	struct iphdr	_ciph, *cih;	/* The ip header contained within the ICMP */
1589 	struct ip_vs_iphdr ciph;
1590 	struct ip_vs_conn *cp;
1591 	struct ip_vs_protocol *pp;
1592 	struct ip_vs_proto_data *pd;
1593 	unsigned int offset, offset2, ihl, verdict;
1594 	bool tunnel, new_cp = false;
1595 	union nf_inet_addr *raddr;
1596 	char *outer_proto = "IPIP";
1597 
1598 	*related = 1;
1599 
1600 	/* reassemble IP fragments */
1601 	if (ip_is_fragment(ip_hdr(skb))) {
1602 		if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum)))
1603 			return NF_STOLEN;
1604 	}
1605 
1606 	iph = ip_hdr(skb);
1607 	offset = ihl = iph->ihl * 4;
1608 	ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
1609 	if (ic == NULL)
1610 		return NF_DROP;
1611 
1612 	IP_VS_DBG(12, "Incoming ICMP (%d,%d) %pI4->%pI4\n",
1613 		  ic->type, ntohs(icmp_id(ic)),
1614 		  &iph->saddr, &iph->daddr);
1615 
1616 	/*
1617 	 * Work through seeing if this is for us.
1618 	 * These checks are supposed to be in an order that means easy
1619 	 * things are checked first to speed up processing.... however
1620 	 * this means that some packets will manage to get a long way
1621 	 * down this stack and then be rejected, but that's life.
1622 	 */
1623 	if ((ic->type != ICMP_DEST_UNREACH) &&
1624 	    (ic->type != ICMP_SOURCE_QUENCH) &&
1625 	    (ic->type != ICMP_TIME_EXCEEDED)) {
1626 		*related = 0;
1627 		return NF_ACCEPT;
1628 	}
1629 
1630 	/* Now find the contained IP header */
1631 	offset += sizeof(_icmph);
1632 	cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1633 	if (cih == NULL)
1634 		return NF_ACCEPT; /* The packet looks wrong, ignore */
1635 	raddr = (union nf_inet_addr *)&cih->daddr;
1636 
1637 	/* Special case for errors for IPIP/UDP/GRE tunnel packets */
1638 	tunnel = false;
1639 	if (cih->protocol == IPPROTO_IPIP) {
1640 		struct ip_vs_dest *dest;
1641 
1642 		if (unlikely(cih->frag_off & htons(IP_OFFSET)))
1643 			return NF_ACCEPT;
1644 		/* Error for our IPIP must arrive at LOCAL_IN */
1645 		if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL))
1646 			return NF_ACCEPT;
1647 		dest = ip_vs_find_tunnel(ipvs, AF_INET, raddr, 0);
1648 		/* Only for known tunnel */
1649 		if (!dest || dest->tun_type != IP_VS_CONN_F_TUNNEL_TYPE_IPIP)
1650 			return NF_ACCEPT;
1651 		offset += cih->ihl * 4;
1652 		cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1653 		if (cih == NULL)
1654 			return NF_ACCEPT; /* The packet looks wrong, ignore */
1655 		tunnel = true;
1656 	} else if ((cih->protocol == IPPROTO_UDP ||	/* Can be UDP encap */
1657 		    cih->protocol == IPPROTO_GRE) &&	/* Can be GRE encap */
1658 		   /* Error for our tunnel must arrive at LOCAL_IN */
1659 		   (skb_rtable(skb)->rt_flags & RTCF_LOCAL)) {
1660 		__u8 iproto;
1661 		int ulen;
1662 
1663 		/* Non-first fragment has no UDP/GRE header */
1664 		if (unlikely(cih->frag_off & htons(IP_OFFSET)))
1665 			return NF_ACCEPT;
1666 		offset2 = offset + cih->ihl * 4;
1667 		if (cih->protocol == IPPROTO_UDP) {
1668 			ulen = ipvs_udp_decap(ipvs, skb, offset2, AF_INET,
1669 					      raddr, &iproto);
1670 			outer_proto = "UDP";
1671 		} else {
1672 			ulen = ipvs_gre_decap(ipvs, skb, offset2, AF_INET,
1673 					      raddr, &iproto);
1674 			outer_proto = "GRE";
1675 		}
1676 		if (ulen > 0) {
1677 			/* Skip IP and UDP/GRE tunnel headers */
1678 			offset = offset2 + ulen;
1679 			/* Now we should be at the original IP header */
1680 			cih = skb_header_pointer(skb, offset, sizeof(_ciph),
1681 						 &_ciph);
1682 			if (cih && cih->version == 4 && cih->ihl >= 5 &&
1683 			    iproto == IPPROTO_IPIP)
1684 				tunnel = true;
1685 			else
1686 				return NF_ACCEPT;
1687 		}
1688 	}
1689 
1690 	pd = ip_vs_proto_data_get(ipvs, cih->protocol);
1691 	if (!pd)
1692 		return NF_ACCEPT;
1693 	pp = pd->pp;
1694 
1695 	/* Is the embedded protocol header present? */
1696 	if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
1697 		     pp->dont_defrag))
1698 		return NF_ACCEPT;
1699 
1700 	IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
1701 		      "Checking incoming ICMP for");
1702 
1703 	offset2 = offset;
1704 	ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !tunnel, &ciph);
1705 	offset = ciph.len;
1706 
1707 	/* The embedded headers contain source and dest in reverse order.
1708 	 * For IPIP/UDP/GRE tunnel this is error for request, not for reply.
1709 	 */
1710 	cp = INDIRECT_CALL_1(pp->conn_in_get, ip_vs_conn_in_get_proto,
1711 			     ipvs, AF_INET, skb, &ciph);
1712 
1713 	if (!cp) {
1714 		int v;
1715 
1716 		if (tunnel || !sysctl_schedule_icmp(ipvs))
1717 			return NF_ACCEPT;
1718 
1719 		if (!ip_vs_try_to_schedule(ipvs, AF_INET, skb, pd, &v, &cp, &ciph))
1720 			return v;
1721 		new_cp = true;
1722 	}
1723 
1724 	verdict = NF_DROP;
1725 
1726 	/* Ensure the checksum is correct */
1727 	if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
1728 		/* Failed checksum! */
1729 		IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n",
1730 			  &iph->saddr);
1731 		goto out;
1732 	}
1733 
1734 	if (tunnel) {
1735 		__be32 info = ic->un.gateway;
1736 		__u8 type = ic->type;
1737 		__u8 code = ic->code;
1738 
1739 		/* Update the MTU */
1740 		if (ic->type == ICMP_DEST_UNREACH &&
1741 		    ic->code == ICMP_FRAG_NEEDED) {
1742 			struct ip_vs_dest *dest = cp->dest;
1743 			u32 mtu = ntohs(ic->un.frag.mtu);
1744 			__be16 frag_off = cih->frag_off;
1745 
1746 			/* Strip outer IP and ICMP, go to IPIP/UDP/GRE header */
1747 			if (pskb_pull(skb, ihl + sizeof(_icmph)) == NULL)
1748 				goto ignore_tunnel;
1749 			offset2 -= ihl + sizeof(_icmph);
1750 			skb_reset_network_header(skb);
1751 			IP_VS_DBG(12, "ICMP for %s %pI4->%pI4: mtu=%u\n",
1752 				  outer_proto, &ip_hdr(skb)->saddr,
1753 				  &ip_hdr(skb)->daddr, mtu);
1754 			ipv4_update_pmtu(skb, ipvs->net, mtu, 0, 0);
1755 			/* Client uses PMTUD? */
1756 			if (!(frag_off & htons(IP_DF)))
1757 				goto ignore_tunnel;
1758 			/* Prefer the resulting PMTU */
1759 			if (dest) {
1760 				struct ip_vs_dest_dst *dest_dst;
1761 
1762 				dest_dst = rcu_dereference(dest->dest_dst);
1763 				if (dest_dst)
1764 					mtu = dst_mtu(dest_dst->dst_cache);
1765 			}
1766 			if (mtu > 68 + sizeof(struct iphdr))
1767 				mtu -= sizeof(struct iphdr);
1768 			info = htonl(mtu);
1769 		}
1770 		/* Strip outer IP, ICMP and IPIP/UDP/GRE, go to IP header of
1771 		 * original request.
1772 		 */
1773 		if (pskb_pull(skb, offset2) == NULL)
1774 			goto ignore_tunnel;
1775 		skb_reset_network_header(skb);
1776 		IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n",
1777 			&ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1778 			type, code, ntohl(info));
1779 		icmp_send(skb, type, code, info);
1780 		/* ICMP can be shorter but anyways, account it */
1781 		ip_vs_out_stats(cp, skb);
1782 
1783 ignore_tunnel:
1784 		consume_skb(skb);
1785 		verdict = NF_STOLEN;
1786 		goto out;
1787 	}
1788 
1789 	/* do the statistics and put it back */
1790 	ip_vs_in_stats(cp, skb);
1791 	if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol ||
1792 	    IPPROTO_SCTP == cih->protocol)
1793 		offset += 2 * sizeof(__u16);
1794 	verdict = ip_vs_icmp_xmit(skb, cp, pp, offset, hooknum, &ciph);
1795 
1796 out:
1797 	if (likely(!new_cp))
1798 		__ip_vs_conn_put(cp);
1799 	else
1800 		ip_vs_conn_put(cp);
1801 
1802 	return verdict;
1803 }
1804 
1805 #ifdef CONFIG_IP_VS_IPV6
ip_vs_in_icmp_v6(struct netns_ipvs * ipvs,struct sk_buff * skb,int * related,unsigned int hooknum,struct ip_vs_iphdr * iph)1806 static int ip_vs_in_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb,
1807 			    int *related, unsigned int hooknum,
1808 			    struct ip_vs_iphdr *iph)
1809 {
1810 	struct icmp6hdr	_icmph, *ic;
1811 	struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};/*Contained IP */
1812 	struct ip_vs_conn *cp;
1813 	struct ip_vs_protocol *pp;
1814 	struct ip_vs_proto_data *pd;
1815 	unsigned int offset, verdict;
1816 	bool new_cp = false;
1817 
1818 	*related = 1;
1819 
1820 	ic = frag_safe_skb_hp(skb, iph->len, sizeof(_icmph), &_icmph);
1821 	if (ic == NULL)
1822 		return NF_DROP;
1823 
1824 	/*
1825 	 * Work through seeing if this is for us.
1826 	 * These checks are supposed to be in an order that means easy
1827 	 * things are checked first to speed up processing.... however
1828 	 * this means that some packets will manage to get a long way
1829 	 * down this stack and then be rejected, but that's life.
1830 	 */
1831 	if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
1832 		*related = 0;
1833 		return NF_ACCEPT;
1834 	}
1835 	/* Fragment header that is before ICMP header tells us that:
1836 	 * it's not an error message since they can't be fragmented.
1837 	 */
1838 	if (iph->flags & IP6_FH_F_FRAG)
1839 		return NF_DROP;
1840 
1841 	IP_VS_DBG(8, "Incoming ICMPv6 (%d,%d) %pI6c->%pI6c\n",
1842 		  ic->icmp6_type, ntohs(icmpv6_id(ic)),
1843 		  &iph->saddr, &iph->daddr);
1844 
1845 	offset = iph->len + sizeof(_icmph);
1846 	if (!ip_vs_fill_iph_skb_icmp(AF_INET6, skb, offset, true, &ciph))
1847 		return NF_ACCEPT;
1848 
1849 	pd = ip_vs_proto_data_get(ipvs, ciph.protocol);
1850 	if (!pd)
1851 		return NF_ACCEPT;
1852 	pp = pd->pp;
1853 
1854 	/* Cannot handle fragmented embedded protocol */
1855 	if (ciph.fragoffs)
1856 		return NF_ACCEPT;
1857 
1858 	IP_VS_DBG_PKT(11, AF_INET6, pp, skb, offset,
1859 		      "Checking incoming ICMPv6 for");
1860 
1861 	/* The embedded headers contain source and dest in reverse order
1862 	 * if not from localhost
1863 	 */
1864 	cp = INDIRECT_CALL_1(pp->conn_in_get, ip_vs_conn_in_get_proto,
1865 			     ipvs, AF_INET6, skb, &ciph);
1866 
1867 	if (!cp) {
1868 		int v;
1869 
1870 		if (!sysctl_schedule_icmp(ipvs))
1871 			return NF_ACCEPT;
1872 
1873 		if (!ip_vs_try_to_schedule(ipvs, AF_INET6, skb, pd, &v, &cp, &ciph))
1874 			return v;
1875 
1876 		new_cp = true;
1877 	}
1878 
1879 	/* VS/TUN, VS/DR and LOCALNODE just let it go */
1880 	if ((hooknum == NF_INET_LOCAL_OUT) &&
1881 	    (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)) {
1882 		verdict = NF_ACCEPT;
1883 		goto out;
1884 	}
1885 
1886 	/* do the statistics and put it back */
1887 	ip_vs_in_stats(cp, skb);
1888 
1889 	/* Need to mangle contained IPv6 header in ICMPv6 packet */
1890 	offset = ciph.len;
1891 	if (IPPROTO_TCP == ciph.protocol || IPPROTO_UDP == ciph.protocol ||
1892 	    IPPROTO_SCTP == ciph.protocol)
1893 		offset += 2 * sizeof(__u16); /* Also mangle ports */
1894 
1895 	verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, offset, hooknum, &ciph);
1896 
1897 out:
1898 	if (likely(!new_cp))
1899 		__ip_vs_conn_put(cp);
1900 	else
1901 		ip_vs_conn_put(cp);
1902 
1903 	return verdict;
1904 }
1905 #endif
1906 
1907 
1908 /*
1909  *	Check if it's for virtual services, look it up,
1910  *	and send it on its way...
1911  */
1912 static unsigned int
ip_vs_in_hook(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)1913 ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state)
1914 {
1915 	struct netns_ipvs *ipvs = net_ipvs(state->net);
1916 	unsigned int hooknum = state->hook;
1917 	struct ip_vs_iphdr iph;
1918 	struct ip_vs_protocol *pp;
1919 	struct ip_vs_proto_data *pd;
1920 	struct ip_vs_conn *cp;
1921 	int ret, pkts;
1922 	struct sock *sk;
1923 	int af = state->pf;
1924 
1925 	/* Already marked as IPVS request or reply? */
1926 	if (skb->ipvs_property)
1927 		return NF_ACCEPT;
1928 
1929 	/*
1930 	 *	Big tappo:
1931 	 *	- remote client: only PACKET_HOST
1932 	 *	- route: used for struct net when skb->dev is unset
1933 	 */
1934 	if (unlikely((skb->pkt_type != PACKET_HOST &&
1935 		      hooknum != NF_INET_LOCAL_OUT) ||
1936 		     !skb_dst(skb))) {
1937 		ip_vs_fill_iph_skb(af, skb, false, &iph);
1938 		IP_VS_DBG_BUF(12, "packet type=%d proto=%d daddr=%s"
1939 			      " ignored in hook %u\n",
1940 			      skb->pkt_type, iph.protocol,
1941 			      IP_VS_DBG_ADDR(af, &iph.daddr), hooknum);
1942 		return NF_ACCEPT;
1943 	}
1944 	/* ipvs enabled in this netns ? */
1945 	if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
1946 		return NF_ACCEPT;
1947 
1948 	ip_vs_fill_iph_skb(af, skb, false, &iph);
1949 
1950 	/* Bad... Do not break raw sockets */
1951 	sk = skb_to_full_sk(skb);
1952 	if (unlikely(sk && hooknum == NF_INET_LOCAL_OUT &&
1953 		     af == AF_INET)) {
1954 
1955 		if (sk->sk_family == PF_INET && inet_sk(sk)->nodefrag)
1956 			return NF_ACCEPT;
1957 	}
1958 
1959 #ifdef CONFIG_IP_VS_IPV6
1960 	if (af == AF_INET6) {
1961 		if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
1962 			int related;
1963 			int verdict = ip_vs_in_icmp_v6(ipvs, skb, &related,
1964 						       hooknum, &iph);
1965 
1966 			if (related)
1967 				return verdict;
1968 		}
1969 	} else
1970 #endif
1971 		if (unlikely(iph.protocol == IPPROTO_ICMP)) {
1972 			int related;
1973 			int verdict = ip_vs_in_icmp(ipvs, skb, &related,
1974 						    hooknum);
1975 
1976 			if (related)
1977 				return verdict;
1978 		}
1979 
1980 	/* Protocol supported? */
1981 	pd = ip_vs_proto_data_get(ipvs, iph.protocol);
1982 	if (unlikely(!pd)) {
1983 		/* The only way we'll see this packet again is if it's
1984 		 * encapsulated, so mark it with ipvs_property=1 so we
1985 		 * skip it if we're ignoring tunneled packets
1986 		 */
1987 		if (sysctl_ignore_tunneled(ipvs))
1988 			skb->ipvs_property = 1;
1989 
1990 		return NF_ACCEPT;
1991 	}
1992 	pp = pd->pp;
1993 	/*
1994 	 * Check if the packet belongs to an existing connection entry
1995 	 */
1996 	cp = INDIRECT_CALL_1(pp->conn_in_get, ip_vs_conn_in_get_proto,
1997 			     ipvs, af, skb, &iph);
1998 
1999 	if (!iph.fragoffs && is_new_conn(skb, &iph) && cp) {
2000 		int conn_reuse_mode = sysctl_conn_reuse_mode(ipvs);
2001 		bool old_ct = false, resched = false;
2002 
2003 		if (unlikely(sysctl_expire_nodest_conn(ipvs)) && cp->dest &&
2004 		    unlikely(!atomic_read(&cp->dest->weight))) {
2005 			resched = true;
2006 			old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
2007 		} else if (conn_reuse_mode &&
2008 			   is_new_conn_expected(cp, conn_reuse_mode)) {
2009 			old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
2010 			if (!atomic_read(&cp->n_control)) {
2011 				resched = true;
2012 			} else {
2013 				/* Do not reschedule controlling connection
2014 				 * that uses conntrack while it is still
2015 				 * referenced by controlled connection(s).
2016 				 */
2017 				resched = !old_ct;
2018 			}
2019 		}
2020 
2021 		if (resched) {
2022 			if (!old_ct)
2023 				cp->flags &= ~IP_VS_CONN_F_NFCT;
2024 			if (!atomic_read(&cp->n_control))
2025 				ip_vs_conn_expire_now(cp);
2026 			__ip_vs_conn_put(cp);
2027 			if (old_ct)
2028 				return NF_DROP;
2029 			cp = NULL;
2030 		}
2031 	}
2032 
2033 	/* Check the server status */
2034 	if (cp && cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
2035 		/* the destination server is not available */
2036 		if (sysctl_expire_nodest_conn(ipvs)) {
2037 			bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
2038 
2039 			if (!old_ct)
2040 				cp->flags &= ~IP_VS_CONN_F_NFCT;
2041 
2042 			ip_vs_conn_expire_now(cp);
2043 			__ip_vs_conn_put(cp);
2044 			if (old_ct)
2045 				return NF_DROP;
2046 			cp = NULL;
2047 		} else {
2048 			__ip_vs_conn_put(cp);
2049 			return NF_DROP;
2050 		}
2051 	}
2052 
2053 	if (unlikely(!cp)) {
2054 		int v;
2055 
2056 		if (!ip_vs_try_to_schedule(ipvs, af, skb, pd, &v, &cp, &iph))
2057 			return v;
2058 	}
2059 
2060 	IP_VS_DBG_PKT(11, af, pp, skb, iph.off, "Incoming packet");
2061 
2062 	ip_vs_in_stats(cp, skb);
2063 	ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);
2064 	if (cp->packet_xmit)
2065 		ret = cp->packet_xmit(skb, cp, pp, &iph);
2066 		/* do not touch skb anymore */
2067 	else {
2068 		IP_VS_DBG_RL("warning: packet_xmit is null");
2069 		ret = NF_ACCEPT;
2070 	}
2071 
2072 	/* Increase its packet counter and check if it is needed
2073 	 * to be synchronized
2074 	 *
2075 	 * Sync connection if it is about to close to
2076 	 * encorage the standby servers to update the connections timeout
2077 	 *
2078 	 * For ONE_PKT let ip_vs_sync_conn() do the filter work.
2079 	 */
2080 
2081 	if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
2082 		pkts = sysctl_sync_threshold(ipvs);
2083 	else
2084 		pkts = atomic_inc_return(&cp->in_pkts);
2085 
2086 	if (ipvs->sync_state & IP_VS_STATE_MASTER)
2087 		ip_vs_sync_conn(ipvs, cp, pkts);
2088 	else if ((cp->flags & IP_VS_CONN_F_ONE_PACKET) && cp->control)
2089 		/* increment is done inside ip_vs_sync_conn too */
2090 		atomic_inc(&cp->control->in_pkts);
2091 
2092 	ip_vs_conn_put(cp);
2093 	return ret;
2094 }
2095 
2096 /*
2097  *	It is hooked at the NF_INET_FORWARD chain, in order to catch ICMP
2098  *      related packets destined for 0.0.0.0/0.
2099  *      When fwmark-based virtual service is used, such as transparent
2100  *      cache cluster, TCP packets can be marked and routed to ip_vs_in,
2101  *      but ICMP destined for 0.0.0.0/0 cannot not be easily marked and
2102  *      sent to ip_vs_in_icmp. So, catch them at the NF_INET_FORWARD chain
2103  *      and send them to ip_vs_in_icmp.
2104  */
2105 static unsigned int
ip_vs_forward_icmp(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)2106 ip_vs_forward_icmp(void *priv, struct sk_buff *skb,
2107 		   const struct nf_hook_state *state)
2108 {
2109 	struct netns_ipvs *ipvs = net_ipvs(state->net);
2110 	int r;
2111 
2112 	/* ipvs enabled in this netns ? */
2113 	if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
2114 		return NF_ACCEPT;
2115 
2116 	if (state->pf == NFPROTO_IPV4) {
2117 		if (ip_hdr(skb)->protocol != IPPROTO_ICMP)
2118 			return NF_ACCEPT;
2119 #ifdef CONFIG_IP_VS_IPV6
2120 	} else {
2121 		struct ip_vs_iphdr iphdr;
2122 
2123 		ip_vs_fill_iph_skb(AF_INET6, skb, false, &iphdr);
2124 
2125 		if (iphdr.protocol != IPPROTO_ICMPV6)
2126 			return NF_ACCEPT;
2127 
2128 		return ip_vs_in_icmp_v6(ipvs, skb, &r, state->hook, &iphdr);
2129 #endif
2130 	}
2131 
2132 	return ip_vs_in_icmp(ipvs, skb, &r, state->hook);
2133 }
2134 
2135 static const struct nf_hook_ops ip_vs_ops4[] = {
2136 	/* After packet filtering, change source only for VS/NAT */
2137 	{
2138 		.hook		= ip_vs_out_hook,
2139 		.pf		= NFPROTO_IPV4,
2140 		.hooknum	= NF_INET_LOCAL_IN,
2141 		.priority	= NF_IP_PRI_NAT_SRC - 2,
2142 	},
2143 	/* After packet filtering, forward packet through VS/DR, VS/TUN,
2144 	 * or VS/NAT(change destination), so that filtering rules can be
2145 	 * applied to IPVS. */
2146 	{
2147 		.hook		= ip_vs_in_hook,
2148 		.pf		= NFPROTO_IPV4,
2149 		.hooknum	= NF_INET_LOCAL_IN,
2150 		.priority	= NF_IP_PRI_NAT_SRC - 1,
2151 	},
2152 	/* Before ip_vs_in, change source only for VS/NAT */
2153 	{
2154 		.hook		= ip_vs_out_hook,
2155 		.pf		= NFPROTO_IPV4,
2156 		.hooknum	= NF_INET_LOCAL_OUT,
2157 		.priority	= NF_IP_PRI_NAT_DST + 1,
2158 	},
2159 	/* After mangle, schedule and forward local requests */
2160 	{
2161 		.hook		= ip_vs_in_hook,
2162 		.pf		= NFPROTO_IPV4,
2163 		.hooknum	= NF_INET_LOCAL_OUT,
2164 		.priority	= NF_IP_PRI_NAT_DST + 2,
2165 	},
2166 	/* After packet filtering (but before ip_vs_out_icmp), catch icmp
2167 	 * destined for 0.0.0.0/0, which is for incoming IPVS connections */
2168 	{
2169 		.hook		= ip_vs_forward_icmp,
2170 		.pf		= NFPROTO_IPV4,
2171 		.hooknum	= NF_INET_FORWARD,
2172 		.priority	= 99,
2173 	},
2174 	/* After packet filtering, change source only for VS/NAT */
2175 	{
2176 		.hook		= ip_vs_out_hook,
2177 		.pf		= NFPROTO_IPV4,
2178 		.hooknum	= NF_INET_FORWARD,
2179 		.priority	= 100,
2180 	},
2181 };
2182 
2183 #ifdef CONFIG_IP_VS_IPV6
2184 static const struct nf_hook_ops ip_vs_ops6[] = {
2185 	/* After packet filtering, change source only for VS/NAT */
2186 	{
2187 		.hook		= ip_vs_out_hook,
2188 		.pf		= NFPROTO_IPV6,
2189 		.hooknum	= NF_INET_LOCAL_IN,
2190 		.priority	= NF_IP6_PRI_NAT_SRC - 2,
2191 	},
2192 	/* After packet filtering, forward packet through VS/DR, VS/TUN,
2193 	 * or VS/NAT(change destination), so that filtering rules can be
2194 	 * applied to IPVS. */
2195 	{
2196 		.hook		= ip_vs_in_hook,
2197 		.pf		= NFPROTO_IPV6,
2198 		.hooknum	= NF_INET_LOCAL_IN,
2199 		.priority	= NF_IP6_PRI_NAT_SRC - 1,
2200 	},
2201 	/* Before ip_vs_in, change source only for VS/NAT */
2202 	{
2203 		.hook		= ip_vs_out_hook,
2204 		.pf		= NFPROTO_IPV6,
2205 		.hooknum	= NF_INET_LOCAL_OUT,
2206 		.priority	= NF_IP6_PRI_NAT_DST + 1,
2207 	},
2208 	/* After mangle, schedule and forward local requests */
2209 	{
2210 		.hook		= ip_vs_in_hook,
2211 		.pf		= NFPROTO_IPV6,
2212 		.hooknum	= NF_INET_LOCAL_OUT,
2213 		.priority	= NF_IP6_PRI_NAT_DST + 2,
2214 	},
2215 	/* After packet filtering (but before ip_vs_out_icmp), catch icmp
2216 	 * destined for 0.0.0.0/0, which is for incoming IPVS connections */
2217 	{
2218 		.hook		= ip_vs_forward_icmp,
2219 		.pf		= NFPROTO_IPV6,
2220 		.hooknum	= NF_INET_FORWARD,
2221 		.priority	= 99,
2222 	},
2223 	/* After packet filtering, change source only for VS/NAT */
2224 	{
2225 		.hook		= ip_vs_out_hook,
2226 		.pf		= NFPROTO_IPV6,
2227 		.hooknum	= NF_INET_FORWARD,
2228 		.priority	= 100,
2229 	},
2230 };
2231 #endif
2232 
ip_vs_register_hooks(struct netns_ipvs * ipvs,unsigned int af)2233 int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af)
2234 {
2235 	const struct nf_hook_ops *ops;
2236 	unsigned int count;
2237 	unsigned int afmask;
2238 	int ret = 0;
2239 
2240 	if (af == AF_INET6) {
2241 #ifdef CONFIG_IP_VS_IPV6
2242 		ops = ip_vs_ops6;
2243 		count = ARRAY_SIZE(ip_vs_ops6);
2244 		afmask = 2;
2245 #else
2246 		return -EINVAL;
2247 #endif
2248 	} else {
2249 		ops = ip_vs_ops4;
2250 		count = ARRAY_SIZE(ip_vs_ops4);
2251 		afmask = 1;
2252 	}
2253 
2254 	if (!(ipvs->hooks_afmask & afmask)) {
2255 		ret = nf_register_net_hooks(ipvs->net, ops, count);
2256 		if (ret >= 0)
2257 			ipvs->hooks_afmask |= afmask;
2258 	}
2259 	return ret;
2260 }
2261 
ip_vs_unregister_hooks(struct netns_ipvs * ipvs,unsigned int af)2262 void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af)
2263 {
2264 	const struct nf_hook_ops *ops;
2265 	unsigned int count;
2266 	unsigned int afmask;
2267 
2268 	if (af == AF_INET6) {
2269 #ifdef CONFIG_IP_VS_IPV6
2270 		ops = ip_vs_ops6;
2271 		count = ARRAY_SIZE(ip_vs_ops6);
2272 		afmask = 2;
2273 #else
2274 		return;
2275 #endif
2276 	} else {
2277 		ops = ip_vs_ops4;
2278 		count = ARRAY_SIZE(ip_vs_ops4);
2279 		afmask = 1;
2280 	}
2281 
2282 	if (ipvs->hooks_afmask & afmask) {
2283 		nf_unregister_net_hooks(ipvs->net, ops, count);
2284 		ipvs->hooks_afmask &= ~afmask;
2285 	}
2286 }
2287 
2288 /*
2289  *	Initialize IP Virtual Server netns mem.
2290  */
__ip_vs_init(struct net * net)2291 static int __net_init __ip_vs_init(struct net *net)
2292 {
2293 	struct netns_ipvs *ipvs;
2294 
2295 	ipvs = net_generic(net, ip_vs_net_id);
2296 	if (ipvs == NULL)
2297 		return -ENOMEM;
2298 
2299 	/* Hold the beast until a service is registered */
2300 	ipvs->enable = 0;
2301 	ipvs->net = net;
2302 	/* Counters used for creating unique names */
2303 	ipvs->gen = atomic_read(&ipvs_netns_cnt);
2304 	atomic_inc(&ipvs_netns_cnt);
2305 	net->ipvs = ipvs;
2306 
2307 	if (ip_vs_estimator_net_init(ipvs) < 0)
2308 		goto estimator_fail;
2309 
2310 	if (ip_vs_control_net_init(ipvs) < 0)
2311 		goto control_fail;
2312 
2313 	if (ip_vs_protocol_net_init(ipvs) < 0)
2314 		goto protocol_fail;
2315 
2316 	if (ip_vs_app_net_init(ipvs) < 0)
2317 		goto app_fail;
2318 
2319 	if (ip_vs_conn_net_init(ipvs) < 0)
2320 		goto conn_fail;
2321 
2322 	if (ip_vs_sync_net_init(ipvs) < 0)
2323 		goto sync_fail;
2324 
2325 	return 0;
2326 /*
2327  * Error handling
2328  */
2329 
2330 sync_fail:
2331 	ip_vs_conn_net_cleanup(ipvs);
2332 conn_fail:
2333 	ip_vs_app_net_cleanup(ipvs);
2334 app_fail:
2335 	ip_vs_protocol_net_cleanup(ipvs);
2336 protocol_fail:
2337 	ip_vs_control_net_cleanup(ipvs);
2338 control_fail:
2339 	ip_vs_estimator_net_cleanup(ipvs);
2340 estimator_fail:
2341 	net->ipvs = NULL;
2342 	return -ENOMEM;
2343 }
2344 
__ip_vs_cleanup_batch(struct list_head * net_list)2345 static void __net_exit __ip_vs_cleanup_batch(struct list_head *net_list)
2346 {
2347 	struct netns_ipvs *ipvs;
2348 	struct net *net;
2349 
2350 	ip_vs_service_nets_cleanup(net_list);	/* ip_vs_flush() with locks */
2351 	list_for_each_entry(net, net_list, exit_list) {
2352 		ipvs = net_ipvs(net);
2353 		ip_vs_conn_net_cleanup(ipvs);
2354 		ip_vs_app_net_cleanup(ipvs);
2355 		ip_vs_protocol_net_cleanup(ipvs);
2356 		ip_vs_control_net_cleanup(ipvs);
2357 		ip_vs_estimator_net_cleanup(ipvs);
2358 		IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen);
2359 		net->ipvs = NULL;
2360 	}
2361 }
2362 
__ip_vs_dev_cleanup_batch(struct list_head * net_list)2363 static void __net_exit __ip_vs_dev_cleanup_batch(struct list_head *net_list)
2364 {
2365 	struct netns_ipvs *ipvs;
2366 	struct net *net;
2367 
2368 	EnterFunction(2);
2369 	list_for_each_entry(net, net_list, exit_list) {
2370 		ipvs = net_ipvs(net);
2371 		ip_vs_unregister_hooks(ipvs, AF_INET);
2372 		ip_vs_unregister_hooks(ipvs, AF_INET6);
2373 		ipvs->enable = 0;	/* Disable packet reception */
2374 		smp_wmb();
2375 		ip_vs_sync_net_cleanup(ipvs);
2376 	}
2377 	LeaveFunction(2);
2378 }
2379 
2380 static struct pernet_operations ipvs_core_ops = {
2381 	.init = __ip_vs_init,
2382 	.exit_batch = __ip_vs_cleanup_batch,
2383 	.id   = &ip_vs_net_id,
2384 	.size = sizeof(struct netns_ipvs),
2385 };
2386 
2387 static struct pernet_operations ipvs_core_dev_ops = {
2388 	.exit_batch = __ip_vs_dev_cleanup_batch,
2389 };
2390 
2391 /*
2392  *	Initialize IP Virtual Server
2393  */
ip_vs_init(void)2394 static int __init ip_vs_init(void)
2395 {
2396 	int ret;
2397 
2398 	ret = ip_vs_control_init();
2399 	if (ret < 0) {
2400 		pr_err("can't setup control.\n");
2401 		goto exit;
2402 	}
2403 
2404 	ip_vs_protocol_init();
2405 
2406 	ret = ip_vs_conn_init();
2407 	if (ret < 0) {
2408 		pr_err("can't setup connection table.\n");
2409 		goto cleanup_protocol;
2410 	}
2411 
2412 	ret = register_pernet_subsys(&ipvs_core_ops);	/* Alloc ip_vs struct */
2413 	if (ret < 0)
2414 		goto cleanup_conn;
2415 
2416 	ret = register_pernet_device(&ipvs_core_dev_ops);
2417 	if (ret < 0)
2418 		goto cleanup_sub;
2419 
2420 	ret = ip_vs_register_nl_ioctl();
2421 	if (ret < 0) {
2422 		pr_err("can't register netlink/ioctl.\n");
2423 		goto cleanup_dev;
2424 	}
2425 
2426 	pr_info("ipvs loaded.\n");
2427 
2428 	return ret;
2429 
2430 cleanup_dev:
2431 	unregister_pernet_device(&ipvs_core_dev_ops);
2432 cleanup_sub:
2433 	unregister_pernet_subsys(&ipvs_core_ops);
2434 cleanup_conn:
2435 	ip_vs_conn_cleanup();
2436 cleanup_protocol:
2437 	ip_vs_protocol_cleanup();
2438 	ip_vs_control_cleanup();
2439 exit:
2440 	return ret;
2441 }
2442 
ip_vs_cleanup(void)2443 static void __exit ip_vs_cleanup(void)
2444 {
2445 	ip_vs_unregister_nl_ioctl();
2446 	unregister_pernet_device(&ipvs_core_dev_ops);
2447 	unregister_pernet_subsys(&ipvs_core_ops);	/* free ip_vs struct */
2448 	ip_vs_conn_cleanup();
2449 	ip_vs_protocol_cleanup();
2450 	ip_vs_control_cleanup();
2451 	pr_info("ipvs unloaded.\n");
2452 }
2453 
2454 module_init(ip_vs_init);
2455 module_exit(ip_vs_cleanup);
2456 MODULE_LICENSE("GPL");
2457