1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2 /* Copyright (c) 2018 Facebook */
3 
4 #include <stdlib.h>
5 #include <memory.h>
6 #include <unistd.h>
7 #include <arpa/inet.h>
8 #include <linux/bpf.h>
9 #include <linux/if_ether.h>
10 #include <linux/pkt_cls.h>
11 #include <linux/rtnetlink.h>
12 #include <linux/netdev.h>
13 #include <sys/socket.h>
14 #include <errno.h>
15 #include <time.h>
16 
17 #include "bpf.h"
18 #include "libbpf.h"
19 #include "libbpf_internal.h"
20 #include "nlattr.h"
21 
22 #ifndef SOL_NETLINK
23 #define SOL_NETLINK 270
24 #endif
25 
26 typedef int (*libbpf_dump_nlmsg_t)(void *cookie, void *msg, struct nlattr **tb);
27 
28 typedef int (*__dump_nlmsg_t)(struct nlmsghdr *nlmsg, libbpf_dump_nlmsg_t,
29 			      void *cookie);
30 
31 struct xdp_link_info {
32 	__u32 prog_id;
33 	__u32 drv_prog_id;
34 	__u32 hw_prog_id;
35 	__u32 skb_prog_id;
36 	__u8 attach_mode;
37 };
38 
39 struct xdp_id_md {
40 	int ifindex;
41 	__u32 flags;
42 	struct xdp_link_info info;
43 	__u64 feature_flags;
44 };
45 
46 struct xdp_features_md {
47 	int ifindex;
48 	__u32 xdp_zc_max_segs;
49 	__u64 flags;
50 };
51 
libbpf_netlink_open(__u32 * nl_pid,int proto)52 static int libbpf_netlink_open(__u32 *nl_pid, int proto)
53 {
54 	struct sockaddr_nl sa;
55 	socklen_t addrlen;
56 	int one = 1, ret;
57 	int sock;
58 
59 	memset(&sa, 0, sizeof(sa));
60 	sa.nl_family = AF_NETLINK;
61 
62 	sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, proto);
63 	if (sock < 0)
64 		return -errno;
65 
66 	if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK,
67 		       &one, sizeof(one)) < 0) {
68 		pr_warn("Netlink error reporting not supported\n");
69 	}
70 
71 	if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
72 		ret = -errno;
73 		goto cleanup;
74 	}
75 
76 	addrlen = sizeof(sa);
77 	if (getsockname(sock, (struct sockaddr *)&sa, &addrlen) < 0) {
78 		ret = -errno;
79 		goto cleanup;
80 	}
81 
82 	if (addrlen != sizeof(sa)) {
83 		ret = -LIBBPF_ERRNO__INTERNAL;
84 		goto cleanup;
85 	}
86 
87 	*nl_pid = sa.nl_pid;
88 	return sock;
89 
90 cleanup:
91 	close(sock);
92 	return ret;
93 }
94 
libbpf_netlink_close(int sock)95 static void libbpf_netlink_close(int sock)
96 {
97 	close(sock);
98 }
99 
100 enum {
101 	NL_CONT,
102 	NL_NEXT,
103 	NL_DONE,
104 };
105 
netlink_recvmsg(int sock,struct msghdr * mhdr,int flags)106 static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags)
107 {
108 	int len;
109 
110 	do {
111 		len = recvmsg(sock, mhdr, flags);
112 	} while (len < 0 && (errno == EINTR || errno == EAGAIN));
113 
114 	if (len < 0)
115 		return -errno;
116 	return len;
117 }
118 
alloc_iov(struct iovec * iov,int len)119 static int alloc_iov(struct iovec *iov, int len)
120 {
121 	void *nbuf;
122 
123 	nbuf = realloc(iov->iov_base, len);
124 	if (!nbuf)
125 		return -ENOMEM;
126 
127 	iov->iov_base = nbuf;
128 	iov->iov_len = len;
129 	return 0;
130 }
131 
libbpf_netlink_recv(int sock,__u32 nl_pid,int seq,__dump_nlmsg_t _fn,libbpf_dump_nlmsg_t fn,void * cookie)132 static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq,
133 			       __dump_nlmsg_t _fn, libbpf_dump_nlmsg_t fn,
134 			       void *cookie)
135 {
136 	struct iovec iov = {};
137 	struct msghdr mhdr = {
138 		.msg_iov = &iov,
139 		.msg_iovlen = 1,
140 	};
141 	bool multipart = true;
142 	struct nlmsgerr *err;
143 	struct nlmsghdr *nh;
144 	int len, ret;
145 
146 	ret = alloc_iov(&iov, 4096);
147 	if (ret)
148 		goto done;
149 
150 	while (multipart) {
151 start:
152 		multipart = false;
153 		len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC);
154 		if (len < 0) {
155 			ret = len;
156 			goto done;
157 		}
158 
159 		if (len > iov.iov_len) {
160 			ret = alloc_iov(&iov, len);
161 			if (ret)
162 				goto done;
163 		}
164 
165 		len = netlink_recvmsg(sock, &mhdr, 0);
166 		if (len < 0) {
167 			ret = len;
168 			goto done;
169 		}
170 
171 		if (len == 0)
172 			break;
173 
174 		for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len);
175 		     nh = NLMSG_NEXT(nh, len)) {
176 			if (nh->nlmsg_pid != nl_pid) {
177 				ret = -LIBBPF_ERRNO__WRNGPID;
178 				goto done;
179 			}
180 			if (nh->nlmsg_seq != seq) {
181 				ret = -LIBBPF_ERRNO__INVSEQ;
182 				goto done;
183 			}
184 			if (nh->nlmsg_flags & NLM_F_MULTI)
185 				multipart = true;
186 			switch (nh->nlmsg_type) {
187 			case NLMSG_ERROR:
188 				err = (struct nlmsgerr *)NLMSG_DATA(nh);
189 				if (!err->error)
190 					continue;
191 				ret = err->error;
192 				libbpf_nla_dump_errormsg(nh);
193 				goto done;
194 			case NLMSG_DONE:
195 				ret = 0;
196 				goto done;
197 			default:
198 				break;
199 			}
200 			if (_fn) {
201 				ret = _fn(nh, fn, cookie);
202 				switch (ret) {
203 				case NL_CONT:
204 					break;
205 				case NL_NEXT:
206 					goto start;
207 				case NL_DONE:
208 					ret = 0;
209 					goto done;
210 				default:
211 					goto done;
212 				}
213 			}
214 		}
215 	}
216 	ret = 0;
217 done:
218 	free(iov.iov_base);
219 	return ret;
220 }
221 
libbpf_netlink_send_recv(struct libbpf_nla_req * req,int proto,__dump_nlmsg_t parse_msg,libbpf_dump_nlmsg_t parse_attr,void * cookie)222 static int libbpf_netlink_send_recv(struct libbpf_nla_req *req,
223 				    int proto, __dump_nlmsg_t parse_msg,
224 				    libbpf_dump_nlmsg_t parse_attr,
225 				    void *cookie)
226 {
227 	__u32 nl_pid = 0;
228 	int sock, ret;
229 
230 	sock = libbpf_netlink_open(&nl_pid, proto);
231 	if (sock < 0)
232 		return sock;
233 
234 	req->nh.nlmsg_pid = 0;
235 	req->nh.nlmsg_seq = time(NULL);
236 
237 	if (send(sock, req, req->nh.nlmsg_len, 0) < 0) {
238 		ret = -errno;
239 		goto out;
240 	}
241 
242 	ret = libbpf_netlink_recv(sock, nl_pid, req->nh.nlmsg_seq,
243 				  parse_msg, parse_attr, cookie);
244 out:
245 	libbpf_netlink_close(sock);
246 	return ret;
247 }
248 
parse_genl_family_id(struct nlmsghdr * nh,libbpf_dump_nlmsg_t fn,void * cookie)249 static int parse_genl_family_id(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
250 				void *cookie)
251 {
252 	struct genlmsghdr *gnl = NLMSG_DATA(nh);
253 	struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN);
254 	struct nlattr *tb[CTRL_ATTR_FAMILY_ID + 1];
255 	__u16 *id = cookie;
256 
257 	libbpf_nla_parse(tb, CTRL_ATTR_FAMILY_ID, na,
258 			 NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL);
259 	if (!tb[CTRL_ATTR_FAMILY_ID])
260 		return NL_CONT;
261 
262 	*id = libbpf_nla_getattr_u16(tb[CTRL_ATTR_FAMILY_ID]);
263 	return NL_DONE;
264 }
265 
libbpf_netlink_resolve_genl_family_id(const char * name,__u16 len,__u16 * id)266 static int libbpf_netlink_resolve_genl_family_id(const char *name,
267 						 __u16 len, __u16 *id)
268 {
269 	struct libbpf_nla_req req = {
270 		.nh.nlmsg_len	= NLMSG_LENGTH(GENL_HDRLEN),
271 		.nh.nlmsg_type	= GENL_ID_CTRL,
272 		.nh.nlmsg_flags	= NLM_F_REQUEST,
273 		.gnl.cmd	= CTRL_CMD_GETFAMILY,
274 		.gnl.version	= 2,
275 	};
276 	int err;
277 
278 	err = nlattr_add(&req, CTRL_ATTR_FAMILY_NAME, name, len);
279 	if (err < 0)
280 		return err;
281 
282 	return libbpf_netlink_send_recv(&req, NETLINK_GENERIC,
283 					parse_genl_family_id, NULL, id);
284 }
285 
__bpf_set_link_xdp_fd_replace(int ifindex,int fd,int old_fd,__u32 flags)286 static int __bpf_set_link_xdp_fd_replace(int ifindex, int fd, int old_fd,
287 					 __u32 flags)
288 {
289 	struct nlattr *nla;
290 	int ret;
291 	struct libbpf_nla_req req;
292 
293 	memset(&req, 0, sizeof(req));
294 	req.nh.nlmsg_len      = NLMSG_LENGTH(sizeof(struct ifinfomsg));
295 	req.nh.nlmsg_flags    = NLM_F_REQUEST | NLM_F_ACK;
296 	req.nh.nlmsg_type     = RTM_SETLINK;
297 	req.ifinfo.ifi_family = AF_UNSPEC;
298 	req.ifinfo.ifi_index  = ifindex;
299 
300 	nla = nlattr_begin_nested(&req, IFLA_XDP);
301 	if (!nla)
302 		return -EMSGSIZE;
303 	ret = nlattr_add(&req, IFLA_XDP_FD, &fd, sizeof(fd));
304 	if (ret < 0)
305 		return ret;
306 	if (flags) {
307 		ret = nlattr_add(&req, IFLA_XDP_FLAGS, &flags, sizeof(flags));
308 		if (ret < 0)
309 			return ret;
310 	}
311 	if (flags & XDP_FLAGS_REPLACE) {
312 		ret = nlattr_add(&req, IFLA_XDP_EXPECTED_FD, &old_fd,
313 				 sizeof(old_fd));
314 		if (ret < 0)
315 			return ret;
316 	}
317 	nlattr_end_nested(&req, nla);
318 
319 	return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL);
320 }
321 
bpf_xdp_attach(int ifindex,int prog_fd,__u32 flags,const struct bpf_xdp_attach_opts * opts)322 int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, const struct bpf_xdp_attach_opts *opts)
323 {
324 	int old_prog_fd, err;
325 
326 	if (!OPTS_VALID(opts, bpf_xdp_attach_opts))
327 		return libbpf_err(-EINVAL);
328 
329 	old_prog_fd = OPTS_GET(opts, old_prog_fd, 0);
330 	if (old_prog_fd)
331 		flags |= XDP_FLAGS_REPLACE;
332 	else
333 		old_prog_fd = -1;
334 
335 	err = __bpf_set_link_xdp_fd_replace(ifindex, prog_fd, old_prog_fd, flags);
336 	return libbpf_err(err);
337 }
338 
bpf_xdp_detach(int ifindex,__u32 flags,const struct bpf_xdp_attach_opts * opts)339 int bpf_xdp_detach(int ifindex, __u32 flags, const struct bpf_xdp_attach_opts *opts)
340 {
341 	return bpf_xdp_attach(ifindex, -1, flags, opts);
342 }
343 
__dump_link_nlmsg(struct nlmsghdr * nlh,libbpf_dump_nlmsg_t dump_link_nlmsg,void * cookie)344 static int __dump_link_nlmsg(struct nlmsghdr *nlh,
345 			     libbpf_dump_nlmsg_t dump_link_nlmsg, void *cookie)
346 {
347 	struct nlattr *tb[IFLA_MAX + 1], *attr;
348 	struct ifinfomsg *ifi = NLMSG_DATA(nlh);
349 	int len;
350 
351 	len = nlh->nlmsg_len - NLMSG_LENGTH(sizeof(*ifi));
352 	attr = (struct nlattr *) ((void *) ifi + NLMSG_ALIGN(sizeof(*ifi)));
353 
354 	if (libbpf_nla_parse(tb, IFLA_MAX, attr, len, NULL) != 0)
355 		return -LIBBPF_ERRNO__NLPARSE;
356 
357 	return dump_link_nlmsg(cookie, ifi, tb);
358 }
359 
get_xdp_info(void * cookie,void * msg,struct nlattr ** tb)360 static int get_xdp_info(void *cookie, void *msg, struct nlattr **tb)
361 {
362 	struct nlattr *xdp_tb[IFLA_XDP_MAX + 1];
363 	struct xdp_id_md *xdp_id = cookie;
364 	struct ifinfomsg *ifinfo = msg;
365 	int ret;
366 
367 	if (xdp_id->ifindex && xdp_id->ifindex != ifinfo->ifi_index)
368 		return 0;
369 
370 	if (!tb[IFLA_XDP])
371 		return 0;
372 
373 	ret = libbpf_nla_parse_nested(xdp_tb, IFLA_XDP_MAX, tb[IFLA_XDP], NULL);
374 	if (ret)
375 		return ret;
376 
377 	if (!xdp_tb[IFLA_XDP_ATTACHED])
378 		return 0;
379 
380 	xdp_id->info.attach_mode = libbpf_nla_getattr_u8(
381 		xdp_tb[IFLA_XDP_ATTACHED]);
382 
383 	if (xdp_id->info.attach_mode == XDP_ATTACHED_NONE)
384 		return 0;
385 
386 	if (xdp_tb[IFLA_XDP_PROG_ID])
387 		xdp_id->info.prog_id = libbpf_nla_getattr_u32(
388 			xdp_tb[IFLA_XDP_PROG_ID]);
389 
390 	if (xdp_tb[IFLA_XDP_SKB_PROG_ID])
391 		xdp_id->info.skb_prog_id = libbpf_nla_getattr_u32(
392 			xdp_tb[IFLA_XDP_SKB_PROG_ID]);
393 
394 	if (xdp_tb[IFLA_XDP_DRV_PROG_ID])
395 		xdp_id->info.drv_prog_id = libbpf_nla_getattr_u32(
396 			xdp_tb[IFLA_XDP_DRV_PROG_ID]);
397 
398 	if (xdp_tb[IFLA_XDP_HW_PROG_ID])
399 		xdp_id->info.hw_prog_id = libbpf_nla_getattr_u32(
400 			xdp_tb[IFLA_XDP_HW_PROG_ID]);
401 
402 	return 0;
403 }
404 
parse_xdp_features(struct nlmsghdr * nh,libbpf_dump_nlmsg_t fn,void * cookie)405 static int parse_xdp_features(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
406 			      void *cookie)
407 {
408 	struct genlmsghdr *gnl = NLMSG_DATA(nh);
409 	struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN);
410 	struct nlattr *tb[NETDEV_CMD_MAX + 1];
411 	struct xdp_features_md *md = cookie;
412 	__u32 ifindex;
413 
414 	libbpf_nla_parse(tb, NETDEV_CMD_MAX, na,
415 			 NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL);
416 
417 	if (!tb[NETDEV_A_DEV_IFINDEX] || !tb[NETDEV_A_DEV_XDP_FEATURES])
418 		return NL_CONT;
419 
420 	ifindex = libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_IFINDEX]);
421 	if (ifindex != md->ifindex)
422 		return NL_CONT;
423 
424 	md->flags = libbpf_nla_getattr_u64(tb[NETDEV_A_DEV_XDP_FEATURES]);
425 	if (tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS])
426 		md->xdp_zc_max_segs =
427 			libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]);
428 	return NL_DONE;
429 }
430 
bpf_xdp_query(int ifindex,int xdp_flags,struct bpf_xdp_query_opts * opts)431 int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts)
432 {
433 	struct libbpf_nla_req req = {
434 		.nh.nlmsg_len      = NLMSG_LENGTH(sizeof(struct ifinfomsg)),
435 		.nh.nlmsg_type     = RTM_GETLINK,
436 		.nh.nlmsg_flags    = NLM_F_DUMP | NLM_F_REQUEST,
437 		.ifinfo.ifi_family = AF_PACKET,
438 	};
439 	struct xdp_id_md xdp_id = {};
440 	struct xdp_features_md md = {
441 		.ifindex = ifindex,
442 	};
443 	__u16 id;
444 	int err;
445 
446 	if (!OPTS_VALID(opts, bpf_xdp_query_opts))
447 		return libbpf_err(-EINVAL);
448 
449 	if (xdp_flags & ~XDP_FLAGS_MASK)
450 		return libbpf_err(-EINVAL);
451 
452 	/* Check whether the single {HW,DRV,SKB} mode is set */
453 	xdp_flags &= XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE;
454 	if (xdp_flags & (xdp_flags - 1))
455 		return libbpf_err(-EINVAL);
456 
457 	xdp_id.ifindex = ifindex;
458 	xdp_id.flags = xdp_flags;
459 
460 	err = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, __dump_link_nlmsg,
461 				       get_xdp_info, &xdp_id);
462 	if (err)
463 		return libbpf_err(err);
464 
465 	OPTS_SET(opts, prog_id, xdp_id.info.prog_id);
466 	OPTS_SET(opts, drv_prog_id, xdp_id.info.drv_prog_id);
467 	OPTS_SET(opts, hw_prog_id, xdp_id.info.hw_prog_id);
468 	OPTS_SET(opts, skb_prog_id, xdp_id.info.skb_prog_id);
469 	OPTS_SET(opts, attach_mode, xdp_id.info.attach_mode);
470 
471 	if (!OPTS_HAS(opts, feature_flags))
472 		return 0;
473 
474 	err = libbpf_netlink_resolve_genl_family_id("netdev", sizeof("netdev"), &id);
475 	if (err < 0) {
476 		if (err == -ENOENT) {
477 			opts->feature_flags = 0;
478 			goto skip_feature_flags;
479 		}
480 		return libbpf_err(err);
481 	}
482 
483 	memset(&req, 0, sizeof(req));
484 	req.nh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
485 	req.nh.nlmsg_flags = NLM_F_REQUEST;
486 	req.nh.nlmsg_type = id;
487 	req.gnl.cmd = NETDEV_CMD_DEV_GET;
488 	req.gnl.version = 2;
489 
490 	err = nlattr_add(&req, NETDEV_A_DEV_IFINDEX, &ifindex, sizeof(ifindex));
491 	if (err < 0)
492 		return libbpf_err(err);
493 
494 	err = libbpf_netlink_send_recv(&req, NETLINK_GENERIC,
495 				       parse_xdp_features, NULL, &md);
496 	if (err)
497 		return libbpf_err(err);
498 
499 	opts->feature_flags = md.flags;
500 	opts->xdp_zc_max_segs = md.xdp_zc_max_segs;
501 
502 skip_feature_flags:
503 	return 0;
504 }
505 
bpf_xdp_query_id(int ifindex,int flags,__u32 * prog_id)506 int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id)
507 {
508 	LIBBPF_OPTS(bpf_xdp_query_opts, opts);
509 	int ret;
510 
511 	ret = bpf_xdp_query(ifindex, flags, &opts);
512 	if (ret)
513 		return libbpf_err(ret);
514 
515 	flags &= XDP_FLAGS_MODES;
516 
517 	if (opts.attach_mode != XDP_ATTACHED_MULTI && !flags)
518 		*prog_id = opts.prog_id;
519 	else if (flags & XDP_FLAGS_DRV_MODE)
520 		*prog_id = opts.drv_prog_id;
521 	else if (flags & XDP_FLAGS_HW_MODE)
522 		*prog_id = opts.hw_prog_id;
523 	else if (flags & XDP_FLAGS_SKB_MODE)
524 		*prog_id = opts.skb_prog_id;
525 	else
526 		*prog_id = 0;
527 
528 	return 0;
529 }
530 
531 
532 typedef int (*qdisc_config_t)(struct libbpf_nla_req *req);
533 
clsact_config(struct libbpf_nla_req * req)534 static int clsact_config(struct libbpf_nla_req *req)
535 {
536 	req->tc.tcm_parent = TC_H_CLSACT;
537 	req->tc.tcm_handle = TC_H_MAKE(TC_H_CLSACT, 0);
538 
539 	return nlattr_add(req, TCA_KIND, "clsact", sizeof("clsact"));
540 }
541 
attach_point_to_config(struct bpf_tc_hook * hook,qdisc_config_t * config)542 static int attach_point_to_config(struct bpf_tc_hook *hook,
543 				  qdisc_config_t *config)
544 {
545 	switch (OPTS_GET(hook, attach_point, 0)) {
546 	case BPF_TC_INGRESS:
547 	case BPF_TC_EGRESS:
548 	case BPF_TC_INGRESS | BPF_TC_EGRESS:
549 		if (OPTS_GET(hook, parent, 0))
550 			return -EINVAL;
551 		*config = &clsact_config;
552 		return 0;
553 	case BPF_TC_CUSTOM:
554 		return -EOPNOTSUPP;
555 	default:
556 		return -EINVAL;
557 	}
558 }
559 
tc_get_tcm_parent(enum bpf_tc_attach_point attach_point,__u32 * parent)560 static int tc_get_tcm_parent(enum bpf_tc_attach_point attach_point,
561 			     __u32 *parent)
562 {
563 	switch (attach_point) {
564 	case BPF_TC_INGRESS:
565 	case BPF_TC_EGRESS:
566 		if (*parent)
567 			return -EINVAL;
568 		*parent = TC_H_MAKE(TC_H_CLSACT,
569 				    attach_point == BPF_TC_INGRESS ?
570 				    TC_H_MIN_INGRESS : TC_H_MIN_EGRESS);
571 		break;
572 	case BPF_TC_CUSTOM:
573 		if (!*parent)
574 			return -EINVAL;
575 		break;
576 	default:
577 		return -EINVAL;
578 	}
579 	return 0;
580 }
581 
tc_qdisc_modify(struct bpf_tc_hook * hook,int cmd,int flags)582 static int tc_qdisc_modify(struct bpf_tc_hook *hook, int cmd, int flags)
583 {
584 	qdisc_config_t config;
585 	int ret;
586 	struct libbpf_nla_req req;
587 
588 	ret = attach_point_to_config(hook, &config);
589 	if (ret < 0)
590 		return ret;
591 
592 	memset(&req, 0, sizeof(req));
593 	req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
594 	req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | flags;
595 	req.nh.nlmsg_type  = cmd;
596 	req.tc.tcm_family  = AF_UNSPEC;
597 	req.tc.tcm_ifindex = OPTS_GET(hook, ifindex, 0);
598 
599 	ret = config(&req);
600 	if (ret < 0)
601 		return ret;
602 
603 	return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL);
604 }
605 
tc_qdisc_create_excl(struct bpf_tc_hook * hook)606 static int tc_qdisc_create_excl(struct bpf_tc_hook *hook)
607 {
608 	return tc_qdisc_modify(hook, RTM_NEWQDISC, NLM_F_CREATE | NLM_F_EXCL);
609 }
610 
tc_qdisc_delete(struct bpf_tc_hook * hook)611 static int tc_qdisc_delete(struct bpf_tc_hook *hook)
612 {
613 	return tc_qdisc_modify(hook, RTM_DELQDISC, 0);
614 }
615 
bpf_tc_hook_create(struct bpf_tc_hook * hook)616 int bpf_tc_hook_create(struct bpf_tc_hook *hook)
617 {
618 	int ret;
619 
620 	if (!hook || !OPTS_VALID(hook, bpf_tc_hook) ||
621 	    OPTS_GET(hook, ifindex, 0) <= 0)
622 		return libbpf_err(-EINVAL);
623 
624 	ret = tc_qdisc_create_excl(hook);
625 	return libbpf_err(ret);
626 }
627 
628 static int __bpf_tc_detach(const struct bpf_tc_hook *hook,
629 			   const struct bpf_tc_opts *opts,
630 			   const bool flush);
631 
bpf_tc_hook_destroy(struct bpf_tc_hook * hook)632 int bpf_tc_hook_destroy(struct bpf_tc_hook *hook)
633 {
634 	if (!hook || !OPTS_VALID(hook, bpf_tc_hook) ||
635 	    OPTS_GET(hook, ifindex, 0) <= 0)
636 		return libbpf_err(-EINVAL);
637 
638 	switch (OPTS_GET(hook, attach_point, 0)) {
639 	case BPF_TC_INGRESS:
640 	case BPF_TC_EGRESS:
641 		return libbpf_err(__bpf_tc_detach(hook, NULL, true));
642 	case BPF_TC_INGRESS | BPF_TC_EGRESS:
643 		return libbpf_err(tc_qdisc_delete(hook));
644 	case BPF_TC_CUSTOM:
645 		return libbpf_err(-EOPNOTSUPP);
646 	default:
647 		return libbpf_err(-EINVAL);
648 	}
649 }
650 
651 struct bpf_cb_ctx {
652 	struct bpf_tc_opts *opts;
653 	bool processed;
654 };
655 
__get_tc_info(void * cookie,struct tcmsg * tc,struct nlattr ** tb,bool unicast)656 static int __get_tc_info(void *cookie, struct tcmsg *tc, struct nlattr **tb,
657 			 bool unicast)
658 {
659 	struct nlattr *tbb[TCA_BPF_MAX + 1];
660 	struct bpf_cb_ctx *info = cookie;
661 
662 	if (!info || !info->opts)
663 		return -EINVAL;
664 	if (unicast && info->processed)
665 		return -EINVAL;
666 	if (!tb[TCA_OPTIONS])
667 		return NL_CONT;
668 
669 	libbpf_nla_parse_nested(tbb, TCA_BPF_MAX, tb[TCA_OPTIONS], NULL);
670 	if (!tbb[TCA_BPF_ID])
671 		return -EINVAL;
672 
673 	OPTS_SET(info->opts, prog_id, libbpf_nla_getattr_u32(tbb[TCA_BPF_ID]));
674 	OPTS_SET(info->opts, handle, tc->tcm_handle);
675 	OPTS_SET(info->opts, priority, TC_H_MAJ(tc->tcm_info) >> 16);
676 
677 	info->processed = true;
678 	return unicast ? NL_NEXT : NL_DONE;
679 }
680 
get_tc_info(struct nlmsghdr * nh,libbpf_dump_nlmsg_t fn,void * cookie)681 static int get_tc_info(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
682 		       void *cookie)
683 {
684 	struct tcmsg *tc = NLMSG_DATA(nh);
685 	struct nlattr *tb[TCA_MAX + 1];
686 
687 	libbpf_nla_parse(tb, TCA_MAX,
688 			 (struct nlattr *)((void *)tc + NLMSG_ALIGN(sizeof(*tc))),
689 			 NLMSG_PAYLOAD(nh, sizeof(*tc)), NULL);
690 	if (!tb[TCA_KIND])
691 		return NL_CONT;
692 	return __get_tc_info(cookie, tc, tb, nh->nlmsg_flags & NLM_F_ECHO);
693 }
694 
tc_add_fd_and_name(struct libbpf_nla_req * req,int fd)695 static int tc_add_fd_and_name(struct libbpf_nla_req *req, int fd)
696 {
697 	struct bpf_prog_info info;
698 	__u32 info_len = sizeof(info);
699 	char name[256];
700 	int len, ret;
701 
702 	memset(&info, 0, info_len);
703 	ret = bpf_prog_get_info_by_fd(fd, &info, &info_len);
704 	if (ret < 0)
705 		return ret;
706 
707 	ret = nlattr_add(req, TCA_BPF_FD, &fd, sizeof(fd));
708 	if (ret < 0)
709 		return ret;
710 	len = snprintf(name, sizeof(name), "%s:[%u]", info.name, info.id);
711 	if (len < 0)
712 		return -errno;
713 	if (len >= sizeof(name))
714 		return -ENAMETOOLONG;
715 	return nlattr_add(req, TCA_BPF_NAME, name, len + 1);
716 }
717 
bpf_tc_attach(const struct bpf_tc_hook * hook,struct bpf_tc_opts * opts)718 int bpf_tc_attach(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts)
719 {
720 	__u32 protocol, bpf_flags, handle, priority, parent, prog_id, flags;
721 	int ret, ifindex, attach_point, prog_fd;
722 	struct bpf_cb_ctx info = {};
723 	struct libbpf_nla_req req;
724 	struct nlattr *nla;
725 
726 	if (!hook || !opts ||
727 	    !OPTS_VALID(hook, bpf_tc_hook) ||
728 	    !OPTS_VALID(opts, bpf_tc_opts))
729 		return libbpf_err(-EINVAL);
730 
731 	ifindex      = OPTS_GET(hook, ifindex, 0);
732 	parent       = OPTS_GET(hook, parent, 0);
733 	attach_point = OPTS_GET(hook, attach_point, 0);
734 
735 	handle       = OPTS_GET(opts, handle, 0);
736 	priority     = OPTS_GET(opts, priority, 0);
737 	prog_fd      = OPTS_GET(opts, prog_fd, 0);
738 	prog_id      = OPTS_GET(opts, prog_id, 0);
739 	flags        = OPTS_GET(opts, flags, 0);
740 
741 	if (ifindex <= 0 || !prog_fd || prog_id)
742 		return libbpf_err(-EINVAL);
743 	if (priority > UINT16_MAX)
744 		return libbpf_err(-EINVAL);
745 	if (flags & ~BPF_TC_F_REPLACE)
746 		return libbpf_err(-EINVAL);
747 
748 	flags = (flags & BPF_TC_F_REPLACE) ? NLM_F_REPLACE : NLM_F_EXCL;
749 	protocol = ETH_P_ALL;
750 
751 	memset(&req, 0, sizeof(req));
752 	req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
753 	req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE |
754 			     NLM_F_ECHO | flags;
755 	req.nh.nlmsg_type  = RTM_NEWTFILTER;
756 	req.tc.tcm_family  = AF_UNSPEC;
757 	req.tc.tcm_ifindex = ifindex;
758 	req.tc.tcm_handle  = handle;
759 	req.tc.tcm_info    = TC_H_MAKE(priority << 16, htons(protocol));
760 
761 	ret = tc_get_tcm_parent(attach_point, &parent);
762 	if (ret < 0)
763 		return libbpf_err(ret);
764 	req.tc.tcm_parent = parent;
765 
766 	ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
767 	if (ret < 0)
768 		return libbpf_err(ret);
769 	nla = nlattr_begin_nested(&req, TCA_OPTIONS);
770 	if (!nla)
771 		return libbpf_err(-EMSGSIZE);
772 	ret = tc_add_fd_and_name(&req, prog_fd);
773 	if (ret < 0)
774 		return libbpf_err(ret);
775 	bpf_flags = TCA_BPF_FLAG_ACT_DIRECT;
776 	ret = nlattr_add(&req, TCA_BPF_FLAGS, &bpf_flags, sizeof(bpf_flags));
777 	if (ret < 0)
778 		return libbpf_err(ret);
779 	nlattr_end_nested(&req, nla);
780 
781 	info.opts = opts;
782 
783 	ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL,
784 				       &info);
785 	if (ret < 0)
786 		return libbpf_err(ret);
787 	if (!info.processed)
788 		return libbpf_err(-ENOENT);
789 	return ret;
790 }
791 
__bpf_tc_detach(const struct bpf_tc_hook * hook,const struct bpf_tc_opts * opts,const bool flush)792 static int __bpf_tc_detach(const struct bpf_tc_hook *hook,
793 			   const struct bpf_tc_opts *opts,
794 			   const bool flush)
795 {
796 	__u32 protocol = 0, handle, priority, parent, prog_id, flags;
797 	int ret, ifindex, attach_point, prog_fd;
798 	struct libbpf_nla_req req;
799 
800 	if (!hook ||
801 	    !OPTS_VALID(hook, bpf_tc_hook) ||
802 	    !OPTS_VALID(opts, bpf_tc_opts))
803 		return -EINVAL;
804 
805 	ifindex      = OPTS_GET(hook, ifindex, 0);
806 	parent       = OPTS_GET(hook, parent, 0);
807 	attach_point = OPTS_GET(hook, attach_point, 0);
808 
809 	handle       = OPTS_GET(opts, handle, 0);
810 	priority     = OPTS_GET(opts, priority, 0);
811 	prog_fd      = OPTS_GET(opts, prog_fd, 0);
812 	prog_id      = OPTS_GET(opts, prog_id, 0);
813 	flags        = OPTS_GET(opts, flags, 0);
814 
815 	if (ifindex <= 0 || flags || prog_fd || prog_id)
816 		return -EINVAL;
817 	if (priority > UINT16_MAX)
818 		return -EINVAL;
819 	if (!flush) {
820 		if (!handle || !priority)
821 			return -EINVAL;
822 		protocol = ETH_P_ALL;
823 	} else {
824 		if (handle || priority)
825 			return -EINVAL;
826 	}
827 
828 	memset(&req, 0, sizeof(req));
829 	req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
830 	req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
831 	req.nh.nlmsg_type  = RTM_DELTFILTER;
832 	req.tc.tcm_family  = AF_UNSPEC;
833 	req.tc.tcm_ifindex = ifindex;
834 	if (!flush) {
835 		req.tc.tcm_handle = handle;
836 		req.tc.tcm_info   = TC_H_MAKE(priority << 16, htons(protocol));
837 	}
838 
839 	ret = tc_get_tcm_parent(attach_point, &parent);
840 	if (ret < 0)
841 		return ret;
842 	req.tc.tcm_parent = parent;
843 
844 	if (!flush) {
845 		ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
846 		if (ret < 0)
847 			return ret;
848 	}
849 
850 	return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL);
851 }
852 
bpf_tc_detach(const struct bpf_tc_hook * hook,const struct bpf_tc_opts * opts)853 int bpf_tc_detach(const struct bpf_tc_hook *hook,
854 		  const struct bpf_tc_opts *opts)
855 {
856 	int ret;
857 
858 	if (!opts)
859 		return libbpf_err(-EINVAL);
860 
861 	ret = __bpf_tc_detach(hook, opts, false);
862 	return libbpf_err(ret);
863 }
864 
bpf_tc_query(const struct bpf_tc_hook * hook,struct bpf_tc_opts * opts)865 int bpf_tc_query(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts)
866 {
867 	__u32 protocol, handle, priority, parent, prog_id, flags;
868 	int ret, ifindex, attach_point, prog_fd;
869 	struct bpf_cb_ctx info = {};
870 	struct libbpf_nla_req req;
871 
872 	if (!hook || !opts ||
873 	    !OPTS_VALID(hook, bpf_tc_hook) ||
874 	    !OPTS_VALID(opts, bpf_tc_opts))
875 		return libbpf_err(-EINVAL);
876 
877 	ifindex      = OPTS_GET(hook, ifindex, 0);
878 	parent       = OPTS_GET(hook, parent, 0);
879 	attach_point = OPTS_GET(hook, attach_point, 0);
880 
881 	handle       = OPTS_GET(opts, handle, 0);
882 	priority     = OPTS_GET(opts, priority, 0);
883 	prog_fd      = OPTS_GET(opts, prog_fd, 0);
884 	prog_id      = OPTS_GET(opts, prog_id, 0);
885 	flags        = OPTS_GET(opts, flags, 0);
886 
887 	if (ifindex <= 0 || flags || prog_fd || prog_id ||
888 	    !handle || !priority)
889 		return libbpf_err(-EINVAL);
890 	if (priority > UINT16_MAX)
891 		return libbpf_err(-EINVAL);
892 
893 	protocol = ETH_P_ALL;
894 
895 	memset(&req, 0, sizeof(req));
896 	req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
897 	req.nh.nlmsg_flags = NLM_F_REQUEST;
898 	req.nh.nlmsg_type  = RTM_GETTFILTER;
899 	req.tc.tcm_family  = AF_UNSPEC;
900 	req.tc.tcm_ifindex = ifindex;
901 	req.tc.tcm_handle  = handle;
902 	req.tc.tcm_info    = TC_H_MAKE(priority << 16, htons(protocol));
903 
904 	ret = tc_get_tcm_parent(attach_point, &parent);
905 	if (ret < 0)
906 		return libbpf_err(ret);
907 	req.tc.tcm_parent = parent;
908 
909 	ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
910 	if (ret < 0)
911 		return libbpf_err(ret);
912 
913 	info.opts = opts;
914 
915 	ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL,
916 				       &info);
917 	if (ret < 0)
918 		return libbpf_err(ret);
919 	if (!info.processed)
920 		return libbpf_err(-ENOENT);
921 	return ret;
922 }
923