1 // SPDX-License-Identifier: GPL-2.0-only
2 #define _GNU_SOURCE
3 
4 #include <arpa/inet.h>
5 #include <bpf/bpf.h>
6 #include <bpf/libbpf.h>
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <getopt.h>
10 #include <linux/ethtool.h>
11 #include <linux/hashtable.h>
12 #include <linux/if_link.h>
13 #include <linux/jhash.h>
14 #include <linux/limits.h>
15 #include <linux/list.h>
16 #include <linux/sockios.h>
17 #include <locale.h>
18 #include <math.h>
19 #include <net/if.h>
20 #include <poll.h>
21 #include <signal.h>
22 #include <stdbool.h>
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26 #include <sys/ioctl.h>
27 #include <sys/mman.h>
28 #include <sys/signalfd.h>
29 #include <sys/sysinfo.h>
30 #include <sys/timerfd.h>
31 #include <sys/utsname.h>
32 #include <time.h>
33 #include <unistd.h>
34 
35 #include "bpf_util.h"
36 #include "xdp_sample_user.h"
37 
38 #define __sample_print(fmt, cond, ...)                                         \
39 	({                                                                     \
40 		if (cond)                                                      \
41 			printf(fmt, ##__VA_ARGS__);                            \
42 	})
43 
44 #define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__)
45 #define print_default(fmt, ...)                                                \
46 	__sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__)
47 #define __print_err(err, fmt, ...)                                             \
48 	({                                                                     \
49 		__sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT,  \
50 			       ##__VA_ARGS__);                                 \
51 		sample_err_exp = sample_err_exp ? true : err > 0;              \
52 	})
53 #define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__)
54 
55 #define __COLUMN(x) "%'10" x " %-13s"
56 #define FMT_COLUMNf __COLUMN(".0f")
57 #define FMT_COLUMNd __COLUMN("d")
58 #define FMT_COLUMNl __COLUMN("llu")
59 #define RX(rx) rx, "rx/s"
60 #define PPS(pps) pps, "pkt/s"
61 #define DROP(drop) drop, "drop/s"
62 #define ERR(err) err, "error/s"
63 #define HITS(hits) hits, "hit/s"
64 #define XMIT(xmit) xmit, "xmit/s"
65 #define PASS(pass) pass, "pass/s"
66 #define REDIR(redir) redir, "redir/s"
67 #define NANOSEC_PER_SEC 1000000000 /* 10^9 */
68 
69 #define XDP_UNKNOWN (XDP_REDIRECT + 1)
70 #define XDP_ACTION_MAX (XDP_UNKNOWN + 1)
71 #define XDP_REDIRECT_ERR_MAX 7
72 
73 enum map_type {
74 	MAP_RX,
75 	MAP_REDIRECT_ERR,
76 	MAP_CPUMAP_ENQUEUE,
77 	MAP_CPUMAP_KTHREAD,
78 	MAP_EXCEPTION,
79 	MAP_DEVMAP_XMIT,
80 	MAP_DEVMAP_XMIT_MULTI,
81 	NUM_MAP,
82 };
83 
84 enum log_level {
85 	LL_DEFAULT = 1U << 0,
86 	LL_SIMPLE = 1U << 1,
87 	LL_DEBUG = 1U << 2,
88 };
89 
90 struct record {
91 	__u64 timestamp;
92 	struct datarec total;
93 	struct datarec *cpu;
94 };
95 
96 struct map_entry {
97 	struct hlist_node node;
98 	__u64 pair;
99 	struct record val;
100 };
101 
102 struct stats_record {
103 	struct record rx_cnt;
104 	struct record redir_err[XDP_REDIRECT_ERR_MAX];
105 	struct record kthread;
106 	struct record exception[XDP_ACTION_MAX];
107 	struct record devmap_xmit;
108 	DECLARE_HASHTABLE(xmit_map, 5);
109 	struct record enq[];
110 };
111 
112 struct sample_output {
113 	struct {
114 		__u64 rx;
115 		__u64 redir;
116 		__u64 drop;
117 		__u64 drop_xmit;
118 		__u64 err;
119 		__u64 xmit;
120 	} totals;
121 	struct {
122 		union {
123 			__u64 pps;
124 			__u64 num;
125 		};
126 		__u64 drop;
127 		__u64 err;
128 	} rx_cnt;
129 	struct {
130 		__u64 suc;
131 		__u64 err;
132 	} redir_cnt;
133 	struct {
134 		__u64 hits;
135 	} except_cnt;
136 	struct {
137 		__u64 pps;
138 		__u64 drop;
139 		__u64 err;
140 		double bavg;
141 	} xmit_cnt;
142 };
143 
144 struct xdp_desc {
145 	int ifindex;
146 	__u32 prog_id;
147 	int flags;
148 } sample_xdp_progs[32];
149 
150 struct datarec *sample_mmap[NUM_MAP];
151 struct bpf_map *sample_map[NUM_MAP];
152 size_t sample_map_count[NUM_MAP];
153 enum log_level sample_log_level;
154 struct sample_output sample_out;
155 unsigned long sample_interval;
156 bool sample_err_exp;
157 int sample_xdp_cnt;
158 int sample_n_cpus;
159 int sample_sig_fd;
160 int sample_mask;
161 
162 static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = {
163 	/* Key=1 keeps unknown errors */
164 	"Success",
165 	"Unknown",
166 	"EINVAL",
167 	"ENETDOWN",
168 	"EMSGSIZE",
169 	"EOPNOTSUPP",
170 	"ENOSPC",
171 };
172 
173 /* Keyed from Unknown */
174 static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = {
175 	"Unknown error",
176 	"Invalid redirection",
177 	"Device being redirected to is down",
178 	"Packet length too large for device",
179 	"Operation not supported",
180 	"No space in ptr_ring of cpumap kthread",
181 };
182 
183 static const char *xdp_action_names[XDP_ACTION_MAX] = {
184 	[XDP_ABORTED]  = "XDP_ABORTED",
185 	[XDP_DROP]     = "XDP_DROP",
186 	[XDP_PASS]     = "XDP_PASS",
187 	[XDP_TX]       = "XDP_TX",
188 	[XDP_REDIRECT] = "XDP_REDIRECT",
189 	[XDP_UNKNOWN]  = "XDP_UNKNOWN",
190 };
191 
gettime(void)192 static __u64 gettime(void)
193 {
194 	struct timespec t;
195 	int res;
196 
197 	res = clock_gettime(CLOCK_MONOTONIC, &t);
198 	if (res < 0) {
199 		fprintf(stderr, "Error with gettimeofday! (%i)\n", res);
200 		return UINT64_MAX;
201 	}
202 	return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;
203 }
204 
action2str(int action)205 static const char *action2str(int action)
206 {
207 	if (action < XDP_ACTION_MAX)
208 		return xdp_action_names[action];
209 	return NULL;
210 }
211 
sample_print_help(int mask)212 static void sample_print_help(int mask)
213 {
214 	printf("Output format description\n\n"
215 	       "By default, redirect success statistics are disabled, use -s to enable.\n"
216 	       "The terse output mode is default, verbose mode can be activated using -v\n"
217 	       "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n"
218 	       "Terse mode displays at most the following fields:\n"
219 	       "  rx/s        Number of packets received per second\n"
220 	       "  redir/s     Number of packets successfully redirected per second\n"
221 	       "  err,drop/s  Aggregated count of errors per second (including dropped packets)\n"
222 	       "  xmit/s      Number of packets transmitted on the output device per second\n\n"
223 	       "Output description for verbose mode:\n"
224 	       "  FIELD                 DESCRIPTION\n");
225 
226 	if (mask & SAMPLE_RX_CNT) {
227 		printf("  receive\t\tDisplays the number of packets received & errors encountered\n"
228 		       " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n"
229 		       " \t\t\tand drop statistics will be expanded inline in terse mode.\n"
230 		       " \t\t\t\tpkt/s     - Packets received per second\n"
231 		       " \t\t\t\tdrop/s    - Packets dropped per second\n"
232 		       " \t\t\t\terror/s   - Errors encountered per second\n\n");
233 	}
234 	if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
235 		printf("  redirect\t\tDisplays the number of packets successfully redirected\n"
236 		       "  \t\t\tErrors encountered are expanded under redirect_err field\n"
237 		       "  \t\t\tNote that passing -s to enable it has a per packet overhead\n"
238 		       "  \t\t\t\tredir/s   - Packets redirected successfully per second\n\n"
239 		       "  redirect_err\t\tDisplays the number of packets that failed redirection\n"
240 		       "  \t\t\tThe errno is expanded under this field with per CPU count\n"
241 		       "  \t\t\tThe recognized errors are:\n");
242 
243 		for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++)
244 			printf("\t\t\t  %s: %s\n", xdp_redirect_err_names[i],
245 			       xdp_redirect_err_help[i - 1]);
246 
247 		printf("  \n\t\t\t\terror/s   - Packets that failed redirection per second\n\n");
248 	}
249 
250 	if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
251 		printf("  enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n"
252 		       "  \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n"
253 		       "  \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n"
254 		       "  \t\t\tpackets to.\n"
255 		       "  \t\t\t\tpkt/s    - Packets enqueued per second from other CPU to CPU N\n"
256 		       "  \t\t\t\tdrop/s   - Packets dropped when trying to enqueue to CPU N\n"
257 		       "  \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
258 	}
259 
260 	if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
261 		printf("  kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n"
262 		       "  \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n"
263 		       "  \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n"
264 		       "  \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n"
265 		       "  \t\t\t\tpkt/s    - Packets consumed per second from ptr_ring\n"
266 		       "  \t\t\t\tdrop/s   - Packets dropped per second in kthread\n"
267 		       "  \t\t\t\tsched    - Number of times kthread called schedule()\n\n"
268 		       "  \t\t\txdp_stats (also expands to per-CPU counts)\n"
269 		       "  \t\t\t\tpass/s  - XDP_PASS count for CPUMAP program execution\n"
270 		       "  \t\t\t\tdrop/s  - XDP_DROP count for CPUMAP program execution\n"
271 		       "  \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n");
272 	}
273 
274 	if (mask & SAMPLE_EXCEPTION_CNT) {
275 		printf("  xdp_exception\t\tDisplays xdp_exception tracepoint events\n"
276 		       "  \t\t\tThis can occur due to internal driver errors, unrecognized\n"
277 		       "  \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n"
278 		       "  \t\t\tEach action is expanded below this field with its count\n"
279 		       "  \t\t\t\thit/s     - Number of times the tracepoint was hit per second\n\n");
280 	}
281 
282 	if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
283 		printf("  devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n"
284 		       "  \t\t\tThis tracepoint is invoked for successful transmissions on output\n"
285 		       "  \t\t\tdevice but these statistics are not available for generic XDP mode,\n"
286 		       "  \t\t\thence they will be omitted from the output when using SKB mode\n"
287 		       "  \t\t\t\txmit/s    - Number of packets that were transmitted per second\n"
288 		       "  \t\t\t\tdrop/s    - Number of packets that failed transmissions per second\n"
289 		       "  \t\t\t\tdrv_err/s - Number of internal driver errors per second\n"
290 		       "  \t\t\t\tbulk-avg  - Average number of packets processed for each event\n\n");
291 	}
292 }
293 
sample_usage(char * argv[],const struct option * long_options,const char * doc,int mask,bool error)294 void sample_usage(char *argv[], const struct option *long_options,
295 		  const char *doc, int mask, bool error)
296 {
297 	int i;
298 
299 	if (!error)
300 		sample_print_help(mask);
301 
302 	printf("\n%s\nOption for %s:\n", doc, argv[0]);
303 	for (i = 0; long_options[i].name != 0; i++) {
304 		printf(" --%-15s", long_options[i].name);
305 		if (long_options[i].flag != NULL)
306 			printf(" flag (internal value: %d)",
307 			       *long_options[i].flag);
308 		else
309 			printf("\t short-option: -%c", long_options[i].val);
310 		printf("\n");
311 	}
312 	printf("\n");
313 }
314 
alloc_record_per_cpu(void)315 static struct datarec *alloc_record_per_cpu(void)
316 {
317 	unsigned int nr_cpus = libbpf_num_possible_cpus();
318 	struct datarec *array;
319 
320 	array = calloc(nr_cpus, sizeof(*array));
321 	if (!array) {
322 		fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n",
323 			nr_cpus);
324 		return NULL;
325 	}
326 	return array;
327 }
328 
map_entry_init(struct map_entry * e,__u64 pair)329 static int map_entry_init(struct map_entry *e, __u64 pair)
330 {
331 	e->pair = pair;
332 	INIT_HLIST_NODE(&e->node);
333 	e->val.timestamp = gettime();
334 	e->val.cpu = alloc_record_per_cpu();
335 	if (!e->val.cpu)
336 		return -ENOMEM;
337 	return 0;
338 }
339 
map_collect_percpu(struct datarec * values,struct record * rec)340 static void map_collect_percpu(struct datarec *values, struct record *rec)
341 {
342 	/* For percpu maps, userspace gets a value per possible CPU */
343 	unsigned int nr_cpus = libbpf_num_possible_cpus();
344 	__u64 sum_xdp_redirect = 0;
345 	__u64 sum_processed = 0;
346 	__u64 sum_xdp_pass = 0;
347 	__u64 sum_xdp_drop = 0;
348 	__u64 sum_dropped = 0;
349 	__u64 sum_issue = 0;
350 	int i;
351 
352 	/* Get time as close as possible to reading map contents */
353 	rec->timestamp = gettime();
354 
355 	/* Record and sum values from each CPU */
356 	for (i = 0; i < nr_cpus; i++) {
357 		rec->cpu[i].processed = READ_ONCE(values[i].processed);
358 		rec->cpu[i].dropped = READ_ONCE(values[i].dropped);
359 		rec->cpu[i].issue = READ_ONCE(values[i].issue);
360 		rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass);
361 		rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop);
362 		rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect);
363 
364 		sum_processed += rec->cpu[i].processed;
365 		sum_dropped += rec->cpu[i].dropped;
366 		sum_issue += rec->cpu[i].issue;
367 		sum_xdp_pass += rec->cpu[i].xdp_pass;
368 		sum_xdp_drop += rec->cpu[i].xdp_drop;
369 		sum_xdp_redirect += rec->cpu[i].xdp_redirect;
370 	}
371 
372 	rec->total.processed = sum_processed;
373 	rec->total.dropped = sum_dropped;
374 	rec->total.issue = sum_issue;
375 	rec->total.xdp_pass = sum_xdp_pass;
376 	rec->total.xdp_drop = sum_xdp_drop;
377 	rec->total.xdp_redirect = sum_xdp_redirect;
378 }
379 
map_collect_percpu_devmap(int map_fd,struct stats_record * rec)380 static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec)
381 {
382 	unsigned int nr_cpus = bpf_num_possible_cpus();
383 	__u32 batch, count = 32;
384 	struct datarec *values;
385 	bool init = false;
386 	__u64 *keys;
387 	int i, ret;
388 
389 	keys = calloc(count, sizeof(__u64));
390 	if (!keys)
391 		return -ENOMEM;
392 	values = calloc(count * nr_cpus, sizeof(struct datarec));
393 	if (!values) {
394 		free(keys);
395 		return -ENOMEM;
396 	}
397 
398 	for (;;) {
399 		bool exit = false;
400 
401 		ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch,
402 					   keys, values, &count, NULL);
403 		if (ret < 0 && errno != ENOENT)
404 			break;
405 		if (errno == ENOENT)
406 			exit = true;
407 
408 		init = true;
409 		for (i = 0; i < count; i++) {
410 			struct map_entry *e, *x = NULL;
411 			__u64 pair = keys[i];
412 			struct datarec *arr;
413 
414 			arr = &values[i * nr_cpus];
415 			hash_for_each_possible(rec->xmit_map, e, node, pair) {
416 				if (e->pair == pair) {
417 					x = e;
418 					break;
419 				}
420 			}
421 			if (!x) {
422 				x = calloc(1, sizeof(*x));
423 				if (!x)
424 					goto cleanup;
425 				if (map_entry_init(x, pair) < 0) {
426 					free(x);
427 					goto cleanup;
428 				}
429 				hash_add(rec->xmit_map, &x->node, pair);
430 			}
431 			map_collect_percpu(arr, &x->val);
432 		}
433 
434 		if (exit)
435 			break;
436 		count = 32;
437 	}
438 
439 	free(values);
440 	free(keys);
441 	return 0;
442 cleanup:
443 	free(values);
444 	free(keys);
445 	return -ENOMEM;
446 }
447 
alloc_stats_record(void)448 static struct stats_record *alloc_stats_record(void)
449 {
450 	struct stats_record *rec;
451 	int i;
452 
453 	rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record));
454 	if (!rec) {
455 		fprintf(stderr, "Failed to allocate memory\n");
456 		return NULL;
457 	}
458 
459 	if (sample_mask & SAMPLE_RX_CNT) {
460 		rec->rx_cnt.cpu = alloc_record_per_cpu();
461 		if (!rec->rx_cnt.cpu) {
462 			fprintf(stderr,
463 				"Failed to allocate rx_cnt per-CPU array\n");
464 			goto end_rec;
465 		}
466 	}
467 	if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
468 		for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) {
469 			rec->redir_err[i].cpu = alloc_record_per_cpu();
470 			if (!rec->redir_err[i].cpu) {
471 				fprintf(stderr,
472 					"Failed to allocate redir_err per-CPU array for "
473 					"\"%s\" case\n",
474 					xdp_redirect_err_names[i]);
475 				while (i--)
476 					free(rec->redir_err[i].cpu);
477 				goto end_rx_cnt;
478 			}
479 		}
480 	}
481 	if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
482 		rec->kthread.cpu = alloc_record_per_cpu();
483 		if (!rec->kthread.cpu) {
484 			fprintf(stderr,
485 				"Failed to allocate kthread per-CPU array\n");
486 			goto end_redir;
487 		}
488 	}
489 	if (sample_mask & SAMPLE_EXCEPTION_CNT) {
490 		for (i = 0; i < XDP_ACTION_MAX; i++) {
491 			rec->exception[i].cpu = alloc_record_per_cpu();
492 			if (!rec->exception[i].cpu) {
493 				fprintf(stderr,
494 					"Failed to allocate exception per-CPU array for "
495 					"\"%s\" case\n",
496 					action2str(i));
497 				while (i--)
498 					free(rec->exception[i].cpu);
499 				goto end_kthread;
500 			}
501 		}
502 	}
503 	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) {
504 		rec->devmap_xmit.cpu = alloc_record_per_cpu();
505 		if (!rec->devmap_xmit.cpu) {
506 			fprintf(stderr,
507 				"Failed to allocate devmap_xmit per-CPU array\n");
508 			goto end_exception;
509 		}
510 	}
511 	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
512 		hash_init(rec->xmit_map);
513 	if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
514 		for (i = 0; i < sample_n_cpus; i++) {
515 			rec->enq[i].cpu = alloc_record_per_cpu();
516 			if (!rec->enq[i].cpu) {
517 				fprintf(stderr,
518 					"Failed to allocate enqueue per-CPU array for "
519 					"CPU %d\n",
520 					i);
521 				while (i--)
522 					free(rec->enq[i].cpu);
523 				goto end_devmap_xmit;
524 			}
525 		}
526 	}
527 
528 	return rec;
529 
530 end_devmap_xmit:
531 	free(rec->devmap_xmit.cpu);
532 end_exception:
533 	for (i = 0; i < XDP_ACTION_MAX; i++)
534 		free(rec->exception[i].cpu);
535 end_kthread:
536 	free(rec->kthread.cpu);
537 end_redir:
538 	for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
539 		free(rec->redir_err[i].cpu);
540 end_rx_cnt:
541 	free(rec->rx_cnt.cpu);
542 end_rec:
543 	free(rec);
544 	return NULL;
545 }
546 
free_stats_record(struct stats_record * r)547 static void free_stats_record(struct stats_record *r)
548 {
549 	struct hlist_node *tmp;
550 	struct map_entry *e;
551 	int i;
552 
553 	for (i = 0; i < sample_n_cpus; i++)
554 		free(r->enq[i].cpu);
555 	hash_for_each_safe(r->xmit_map, i, tmp, e, node) {
556 		hash_del(&e->node);
557 		free(e->val.cpu);
558 		free(e);
559 	}
560 	free(r->devmap_xmit.cpu);
561 	for (i = 0; i < XDP_ACTION_MAX; i++)
562 		free(r->exception[i].cpu);
563 	free(r->kthread.cpu);
564 	for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
565 		free(r->redir_err[i].cpu);
566 	free(r->rx_cnt.cpu);
567 	free(r);
568 }
569 
calc_period(struct record * r,struct record * p)570 static double calc_period(struct record *r, struct record *p)
571 {
572 	double period_ = 0;
573 	__u64 period = 0;
574 
575 	period = r->timestamp - p->timestamp;
576 	if (period > 0)
577 		period_ = ((double)period / NANOSEC_PER_SEC);
578 
579 	return period_;
580 }
581 
sample_round(double val)582 static double sample_round(double val)
583 {
584 	if (val - floor(val) < 0.5)
585 		return floor(val);
586 	return ceil(val);
587 }
588 
calc_pps(struct datarec * r,struct datarec * p,double period_)589 static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_)
590 {
591 	__u64 packets = 0;
592 	__u64 pps = 0;
593 
594 	if (period_ > 0) {
595 		packets = r->processed - p->processed;
596 		pps = sample_round(packets / period_);
597 	}
598 	return pps;
599 }
600 
calc_drop_pps(struct datarec * r,struct datarec * p,double period_)601 static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_)
602 {
603 	__u64 packets = 0;
604 	__u64 pps = 0;
605 
606 	if (period_ > 0) {
607 		packets = r->dropped - p->dropped;
608 		pps = sample_round(packets / period_);
609 	}
610 	return pps;
611 }
612 
calc_errs_pps(struct datarec * r,struct datarec * p,double period_)613 static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_)
614 {
615 	__u64 packets = 0;
616 	__u64 pps = 0;
617 
618 	if (period_ > 0) {
619 		packets = r->issue - p->issue;
620 		pps = sample_round(packets / period_);
621 	}
622 	return pps;
623 }
624 
calc_info_pps(struct datarec * r,struct datarec * p,double period_)625 static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_)
626 {
627 	__u64 packets = 0;
628 	__u64 pps = 0;
629 
630 	if (period_ > 0) {
631 		packets = r->info - p->info;
632 		pps = sample_round(packets / period_);
633 	}
634 	return pps;
635 }
636 
calc_xdp_pps(struct datarec * r,struct datarec * p,double * xdp_pass,double * xdp_drop,double * xdp_redirect,double period_)637 static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass,
638 			 double *xdp_drop, double *xdp_redirect, double period_)
639 {
640 	*xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0;
641 	if (period_ > 0) {
642 		*xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_;
643 		*xdp_pass = (r->xdp_pass - p->xdp_pass) / period_;
644 		*xdp_drop = (r->xdp_drop - p->xdp_drop) / period_;
645 	}
646 }
647 
stats_get_rx_cnt(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus,struct sample_output * out)648 static void stats_get_rx_cnt(struct stats_record *stats_rec,
649 			     struct stats_record *stats_prev,
650 			     unsigned int nr_cpus, struct sample_output *out)
651 {
652 	struct record *rec, *prev;
653 	double t, pps, drop, err;
654 	int i;
655 
656 	rec = &stats_rec->rx_cnt;
657 	prev = &stats_prev->rx_cnt;
658 	t = calc_period(rec, prev);
659 
660 	for (i = 0; i < nr_cpus; i++) {
661 		struct datarec *r = &rec->cpu[i];
662 		struct datarec *p = &prev->cpu[i];
663 		char str[64];
664 
665 		pps = calc_pps(r, p, t);
666 		drop = calc_drop_pps(r, p, t);
667 		err = calc_errs_pps(r, p, t);
668 		if (!pps && !drop && !err)
669 			continue;
670 
671 		snprintf(str, sizeof(str), "cpu:%d", i);
672 		print_default("    %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
673 			      "\n",
674 			      str, PPS(pps), DROP(drop), ERR(err));
675 	}
676 
677 	if (out) {
678 		pps = calc_pps(&rec->total, &prev->total, t);
679 		drop = calc_drop_pps(&rec->total, &prev->total, t);
680 		err = calc_errs_pps(&rec->total, &prev->total, t);
681 
682 		out->rx_cnt.pps = pps;
683 		out->rx_cnt.drop = drop;
684 		out->rx_cnt.err = err;
685 		out->totals.rx += pps;
686 		out->totals.drop += drop;
687 		out->totals.err += err;
688 	}
689 }
690 
stats_get_cpumap_enqueue(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus)691 static void stats_get_cpumap_enqueue(struct stats_record *stats_rec,
692 				     struct stats_record *stats_prev,
693 				     unsigned int nr_cpus)
694 {
695 	struct record *rec, *prev;
696 	double t, pps, drop, err;
697 	int i, to_cpu;
698 
699 	/* cpumap enqueue stats */
700 	for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) {
701 		rec = &stats_rec->enq[to_cpu];
702 		prev = &stats_prev->enq[to_cpu];
703 		t = calc_period(rec, prev);
704 
705 		pps = calc_pps(&rec->total, &prev->total, t);
706 		drop = calc_drop_pps(&rec->total, &prev->total, t);
707 		err = calc_errs_pps(&rec->total, &prev->total, t);
708 
709 		if (pps > 0 || drop > 0) {
710 			char str[64];
711 
712 			snprintf(str, sizeof(str), "enqueue to cpu %d", to_cpu);
713 
714 			if (err > 0)
715 				err = pps / err; /* calc average bulk size */
716 
717 			print_err(drop,
718 				  "  %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
719 					  ".2f") "\n",
720 				  str, PPS(pps), DROP(drop), err, "bulk-avg");
721 		}
722 
723 		for (i = 0; i < nr_cpus; i++) {
724 			struct datarec *r = &rec->cpu[i];
725 			struct datarec *p = &prev->cpu[i];
726 			char str[64];
727 
728 			pps = calc_pps(r, p, t);
729 			drop = calc_drop_pps(r, p, t);
730 			err = calc_errs_pps(r, p, t);
731 			if (!pps && !drop && !err)
732 				continue;
733 
734 			snprintf(str, sizeof(str), "cpu:%d->%d", i, to_cpu);
735 			if (err > 0)
736 				err = pps / err; /* calc average bulk size */
737 			print_default(
738 				"    %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
739 					".2f") "\n",
740 				str, PPS(pps), DROP(drop), err, "bulk-avg");
741 		}
742 	}
743 }
744 
stats_get_cpumap_remote(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus)745 static void stats_get_cpumap_remote(struct stats_record *stats_rec,
746 				    struct stats_record *stats_prev,
747 				    unsigned int nr_cpus)
748 {
749 	double xdp_pass, xdp_drop, xdp_redirect;
750 	struct record *rec, *prev;
751 	double t;
752 	int i;
753 
754 	rec = &stats_rec->kthread;
755 	prev = &stats_prev->kthread;
756 	t = calc_period(rec, prev);
757 
758 	calc_xdp_pps(&rec->total, &prev->total, &xdp_pass, &xdp_drop,
759 		     &xdp_redirect, t);
760 	if (xdp_pass || xdp_drop || xdp_redirect) {
761 		print_err(xdp_drop,
762 			  "    %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
763 			  "xdp_stats", PASS(xdp_pass), DROP(xdp_drop),
764 			  REDIR(xdp_redirect));
765 	}
766 
767 	for (i = 0; i < nr_cpus; i++) {
768 		struct datarec *r = &rec->cpu[i];
769 		struct datarec *p = &prev->cpu[i];
770 		char str[64];
771 
772 		calc_xdp_pps(r, p, &xdp_pass, &xdp_drop, &xdp_redirect, t);
773 		if (!xdp_pass && !xdp_drop && !xdp_redirect)
774 			continue;
775 
776 		snprintf(str, sizeof(str), "cpu:%d", i);
777 		print_default("      %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
778 			      "\n",
779 			      str, PASS(xdp_pass), DROP(xdp_drop),
780 			      REDIR(xdp_redirect));
781 	}
782 }
783 
stats_get_cpumap_kthread(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus)784 static void stats_get_cpumap_kthread(struct stats_record *stats_rec,
785 				     struct stats_record *stats_prev,
786 				     unsigned int nr_cpus)
787 {
788 	struct record *rec, *prev;
789 	double t, pps, drop, err;
790 	int i;
791 
792 	rec = &stats_rec->kthread;
793 	prev = &stats_prev->kthread;
794 	t = calc_period(rec, prev);
795 
796 	pps = calc_pps(&rec->total, &prev->total, t);
797 	drop = calc_drop_pps(&rec->total, &prev->total, t);
798 	err = calc_errs_pps(&rec->total, &prev->total, t);
799 
800 	print_err(drop, "  %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
801 		  pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err,
802 		  "sched");
803 
804 	for (i = 0; i < nr_cpus; i++) {
805 		struct datarec *r = &rec->cpu[i];
806 		struct datarec *p = &prev->cpu[i];
807 		char str[64];
808 
809 		pps = calc_pps(r, p, t);
810 		drop = calc_drop_pps(r, p, t);
811 		err = calc_errs_pps(r, p, t);
812 		if (!pps && !drop && !err)
813 			continue;
814 
815 		snprintf(str, sizeof(str), "cpu:%d", i);
816 		print_default("    %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
817 			      "\n",
818 			      str, PPS(pps), DROP(drop), err, "sched");
819 	}
820 }
821 
stats_get_redirect_cnt(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus,struct sample_output * out)822 static void stats_get_redirect_cnt(struct stats_record *stats_rec,
823 				   struct stats_record *stats_prev,
824 				   unsigned int nr_cpus,
825 				   struct sample_output *out)
826 {
827 	struct record *rec, *prev;
828 	double t, pps;
829 	int i;
830 
831 	rec = &stats_rec->redir_err[0];
832 	prev = &stats_prev->redir_err[0];
833 	t = calc_period(rec, prev);
834 	for (i = 0; i < nr_cpus; i++) {
835 		struct datarec *r = &rec->cpu[i];
836 		struct datarec *p = &prev->cpu[i];
837 		char str[64];
838 
839 		pps = calc_pps(r, p, t);
840 		if (!pps)
841 			continue;
842 
843 		snprintf(str, sizeof(str), "cpu:%d", i);
844 		print_default("    %-18s " FMT_COLUMNf "\n", str, REDIR(pps));
845 	}
846 
847 	if (out) {
848 		pps = calc_pps(&rec->total, &prev->total, t);
849 		out->redir_cnt.suc = pps;
850 		out->totals.redir += pps;
851 	}
852 }
853 
stats_get_redirect_err_cnt(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus,struct sample_output * out)854 static void stats_get_redirect_err_cnt(struct stats_record *stats_rec,
855 				       struct stats_record *stats_prev,
856 				       unsigned int nr_cpus,
857 				       struct sample_output *out)
858 {
859 	struct record *rec, *prev;
860 	double t, drop, sum = 0;
861 	int rec_i, i;
862 
863 	for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) {
864 		char str[64];
865 
866 		rec = &stats_rec->redir_err[rec_i];
867 		prev = &stats_prev->redir_err[rec_i];
868 		t = calc_period(rec, prev);
869 
870 		drop = calc_drop_pps(&rec->total, &prev->total, t);
871 		if (drop > 0 && !out) {
872 			snprintf(str, sizeof(str),
873 				 sample_log_level & LL_DEFAULT ? "%s total" :
874 								       "%s",
875 				 xdp_redirect_err_names[rec_i]);
876 			print_err(drop, "    %-18s " FMT_COLUMNf "\n", str,
877 				  ERR(drop));
878 		}
879 
880 		for (i = 0; i < nr_cpus; i++) {
881 			struct datarec *r = &rec->cpu[i];
882 			struct datarec *p = &prev->cpu[i];
883 			double drop;
884 
885 			drop = calc_drop_pps(r, p, t);
886 			if (!drop)
887 				continue;
888 
889 			snprintf(str, sizeof(str), "cpu:%d", i);
890 			print_default("       %-16s" FMT_COLUMNf "\n", str,
891 				      ERR(drop));
892 		}
893 
894 		sum += drop;
895 	}
896 
897 	if (out) {
898 		out->redir_cnt.err = sum;
899 		out->totals.err += sum;
900 	}
901 }
902 
stats_get_exception_cnt(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus,struct sample_output * out)903 static void stats_get_exception_cnt(struct stats_record *stats_rec,
904 				    struct stats_record *stats_prev,
905 				    unsigned int nr_cpus,
906 				    struct sample_output *out)
907 {
908 	double t, drop, sum = 0;
909 	struct record *rec, *prev;
910 	int rec_i, i;
911 
912 	for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) {
913 		rec = &stats_rec->exception[rec_i];
914 		prev = &stats_prev->exception[rec_i];
915 		t = calc_period(rec, prev);
916 
917 		drop = calc_drop_pps(&rec->total, &prev->total, t);
918 		/* Fold out errors after heading */
919 		sum += drop;
920 
921 		if (drop > 0 && !out) {
922 			print_always("    %-18s " FMT_COLUMNf "\n",
923 				     action2str(rec_i), ERR(drop));
924 
925 			for (i = 0; i < nr_cpus; i++) {
926 				struct datarec *r = &rec->cpu[i];
927 				struct datarec *p = &prev->cpu[i];
928 				char str[64];
929 				double drop;
930 
931 				drop = calc_drop_pps(r, p, t);
932 				if (!drop)
933 					continue;
934 
935 				snprintf(str, sizeof(str), "cpu:%d", i);
936 				print_default("       %-16s" FMT_COLUMNf "\n",
937 					      str, ERR(drop));
938 			}
939 		}
940 	}
941 
942 	if (out) {
943 		out->except_cnt.hits = sum;
944 		out->totals.err += sum;
945 	}
946 }
947 
stats_get_devmap_xmit(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus,struct sample_output * out)948 static void stats_get_devmap_xmit(struct stats_record *stats_rec,
949 				  struct stats_record *stats_prev,
950 				  unsigned int nr_cpus,
951 				  struct sample_output *out)
952 {
953 	double pps, drop, info, err;
954 	struct record *rec, *prev;
955 	double t;
956 	int i;
957 
958 	rec = &stats_rec->devmap_xmit;
959 	prev = &stats_prev->devmap_xmit;
960 	t = calc_period(rec, prev);
961 	for (i = 0; i < nr_cpus; i++) {
962 		struct datarec *r = &rec->cpu[i];
963 		struct datarec *p = &prev->cpu[i];
964 		char str[64];
965 
966 		pps = calc_pps(r, p, t);
967 		drop = calc_drop_pps(r, p, t);
968 		err = calc_errs_pps(r, p, t);
969 
970 		if (!pps && !drop && !err)
971 			continue;
972 
973 		snprintf(str, sizeof(str), "cpu:%d", i);
974 		info = calc_info_pps(r, p, t);
975 		if (info > 0)
976 			info = (pps + drop) / info; /* calc avg bulk */
977 		print_default("     %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
978 				      __COLUMN(".2f") "\n",
979 			      str, XMIT(pps), DROP(drop), err, "drv_err/s",
980 			      info, "bulk-avg");
981 	}
982 	if (out) {
983 		pps = calc_pps(&rec->total, &prev->total, t);
984 		drop = calc_drop_pps(&rec->total, &prev->total, t);
985 		info = calc_info_pps(&rec->total, &prev->total, t);
986 		if (info > 0)
987 			info = (pps + drop) / info; /* calc avg bulk */
988 		err = calc_errs_pps(&rec->total, &prev->total, t);
989 
990 		out->xmit_cnt.pps = pps;
991 		out->xmit_cnt.drop = drop;
992 		out->xmit_cnt.bavg = info;
993 		out->xmit_cnt.err = err;
994 		out->totals.xmit += pps;
995 		out->totals.drop_xmit += drop;
996 		out->totals.err += err;
997 	}
998 }
999 
stats_get_devmap_xmit_multi(struct stats_record * stats_rec,struct stats_record * stats_prev,unsigned int nr_cpus,struct sample_output * out,bool xmit_total)1000 static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec,
1001 					struct stats_record *stats_prev,
1002 					unsigned int nr_cpus,
1003 					struct sample_output *out,
1004 					bool xmit_total)
1005 {
1006 	double pps, drop, info, err;
1007 	struct map_entry *entry;
1008 	struct record *r, *p;
1009 	double t;
1010 	int bkt;
1011 
1012 	hash_for_each(stats_rec->xmit_map, bkt, entry, node) {
1013 		struct map_entry *e, *x = NULL;
1014 		char ifname_from[IFNAMSIZ];
1015 		char ifname_to[IFNAMSIZ];
1016 		const char *fstr, *tstr;
1017 		unsigned long prev_time;
1018 		struct record beg = {};
1019 		__u32 from_idx, to_idx;
1020 		char str[128];
1021 		__u64 pair;
1022 		int i;
1023 
1024 		prev_time = sample_interval * NANOSEC_PER_SEC;
1025 
1026 		pair = entry->pair;
1027 		from_idx = pair >> 32;
1028 		to_idx = pair & 0xFFFFFFFF;
1029 
1030 		r = &entry->val;
1031 		beg.timestamp = r->timestamp - prev_time;
1032 
1033 		/* Find matching entry from stats_prev map */
1034 		hash_for_each_possible(stats_prev->xmit_map, e, node, pair) {
1035 			if (e->pair == pair) {
1036 				x = e;
1037 				break;
1038 			}
1039 		}
1040 		if (x)
1041 			p = &x->val;
1042 		else
1043 			p = &beg;
1044 		t = calc_period(r, p);
1045 		pps = calc_pps(&r->total, &p->total, t);
1046 		drop = calc_drop_pps(&r->total, &p->total, t);
1047 		info = calc_info_pps(&r->total, &p->total, t);
1048 		if (info > 0)
1049 			info = (pps + drop) / info; /* calc avg bulk */
1050 		err = calc_errs_pps(&r->total, &p->total, t);
1051 
1052 		if (out) {
1053 			/* We are responsible for filling out totals */
1054 			out->totals.xmit += pps;
1055 			out->totals.drop_xmit += drop;
1056 			out->totals.err += err;
1057 			continue;
1058 		}
1059 
1060 		fstr = tstr = NULL;
1061 		if (if_indextoname(from_idx, ifname_from))
1062 			fstr = ifname_from;
1063 		if (if_indextoname(to_idx, ifname_to))
1064 			tstr = ifname_to;
1065 
1066 		snprintf(str, sizeof(str), "xmit %s->%s", fstr ?: "?",
1067 			 tstr ?: "?");
1068 		/* Skip idle streams of redirection */
1069 		if (pps || drop || err) {
1070 			print_err(drop,
1071 				  "  %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1072 				  __COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop),
1073 				  err, "drv_err/s", info, "bulk-avg");
1074 		}
1075 
1076 		for (i = 0; i < nr_cpus; i++) {
1077 			struct datarec *rc = &r->cpu[i];
1078 			struct datarec *pc, p_beg = {};
1079 			char str[64];
1080 
1081 			pc = p == &beg ? &p_beg : &p->cpu[i];
1082 
1083 			pps = calc_pps(rc, pc, t);
1084 			drop = calc_drop_pps(rc, pc, t);
1085 			err = calc_errs_pps(rc, pc, t);
1086 
1087 			if (!pps && !drop && !err)
1088 				continue;
1089 
1090 			snprintf(str, sizeof(str), "cpu:%d", i);
1091 			info = calc_info_pps(rc, pc, t);
1092 			if (info > 0)
1093 				info = (pps + drop) / info; /* calc avg bulk */
1094 
1095 			print_default("     %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1096 				      __COLUMN(".2f") "\n", str, XMIT(pps),
1097 				      DROP(drop), err, "drv_err/s", info, "bulk-avg");
1098 		}
1099 	}
1100 }
1101 
stats_print(const char * prefix,int mask,struct stats_record * r,struct stats_record * p,struct sample_output * out)1102 static void stats_print(const char *prefix, int mask, struct stats_record *r,
1103 			struct stats_record *p, struct sample_output *out)
1104 {
1105 	int nr_cpus = libbpf_num_possible_cpus();
1106 	const char *str;
1107 
1108 	print_always("%-23s", prefix ?: "Summary");
1109 	if (mask & SAMPLE_RX_CNT)
1110 		print_always(FMT_COLUMNl, RX(out->totals.rx));
1111 	if (mask & SAMPLE_REDIRECT_CNT)
1112 		print_always(FMT_COLUMNl, REDIR(out->totals.redir));
1113 	printf(FMT_COLUMNl,
1114 	       out->totals.err + out->totals.drop + out->totals.drop_xmit,
1115 	       "err,drop/s");
1116 	if (mask & SAMPLE_DEVMAP_XMIT_CNT ||
1117 	    mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1118 		printf(FMT_COLUMNl, XMIT(out->totals.xmit));
1119 	printf("\n");
1120 
1121 	if (mask & SAMPLE_RX_CNT) {
1122 		str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ?
1123 				    "receive total" :
1124 				    "receive";
1125 		print_err((out->rx_cnt.err || out->rx_cnt.drop),
1126 			  "  %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n",
1127 			  str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop),
1128 			  ERR(out->rx_cnt.err));
1129 
1130 		stats_get_rx_cnt(r, p, nr_cpus, NULL);
1131 	}
1132 
1133 	if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1134 		stats_get_cpumap_enqueue(r, p, nr_cpus);
1135 
1136 	if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
1137 		stats_get_cpumap_kthread(r, p, nr_cpus);
1138 		stats_get_cpumap_remote(r, p, nr_cpus);
1139 	}
1140 
1141 	if (mask & SAMPLE_REDIRECT_CNT) {
1142 		str = out->redir_cnt.suc ? "redirect total" : "redirect";
1143 		print_default("  %-20s " FMT_COLUMNl "\n", str,
1144 			      REDIR(out->redir_cnt.suc));
1145 
1146 		stats_get_redirect_cnt(r, p, nr_cpus, NULL);
1147 	}
1148 
1149 	if (mask & SAMPLE_REDIRECT_ERR_CNT) {
1150 		str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ?
1151 				    "redirect_err total" :
1152 				    "redirect_err";
1153 		print_err(out->redir_cnt.err, "  %-20s " FMT_COLUMNl "\n", str,
1154 			  ERR(out->redir_cnt.err));
1155 
1156 		stats_get_redirect_err_cnt(r, p, nr_cpus, NULL);
1157 	}
1158 
1159 	if (mask & SAMPLE_EXCEPTION_CNT) {
1160 		str = out->except_cnt.hits ? "xdp_exception total" :
1161 						   "xdp_exception";
1162 
1163 		print_err(out->except_cnt.hits, "  %-20s " FMT_COLUMNl "\n", str,
1164 			  HITS(out->except_cnt.hits));
1165 
1166 		stats_get_exception_cnt(r, p, nr_cpus, NULL);
1167 	}
1168 
1169 	if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
1170 		str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ?
1171 				    "devmap_xmit total" :
1172 				    "devmap_xmit";
1173 
1174 		print_err(out->xmit_cnt.err || out->xmit_cnt.drop,
1175 			  "  %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl
1176 				  __COLUMN(".2f") "\n",
1177 			  str, XMIT(out->xmit_cnt.pps),
1178 			  DROP(out->xmit_cnt.drop), out->xmit_cnt.err,
1179 			  "drv_err/s", out->xmit_cnt.bavg, "bulk-avg");
1180 
1181 		stats_get_devmap_xmit(r, p, nr_cpus, NULL);
1182 	}
1183 
1184 	if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1185 		stats_get_devmap_xmit_multi(r, p, nr_cpus, NULL,
1186 					    mask & SAMPLE_DEVMAP_XMIT_CNT);
1187 
1188 	if (sample_log_level & LL_DEFAULT ||
1189 	    ((sample_log_level & LL_SIMPLE) && sample_err_exp)) {
1190 		sample_err_exp = false;
1191 		printf("\n");
1192 	}
1193 }
1194 
sample_setup_maps(struct bpf_map ** maps)1195 int sample_setup_maps(struct bpf_map **maps)
1196 {
1197 	sample_n_cpus = libbpf_num_possible_cpus();
1198 
1199 	for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1200 		sample_map[i] = maps[i];
1201 
1202 		switch (i) {
1203 		case MAP_RX:
1204 		case MAP_CPUMAP_KTHREAD:
1205 		case MAP_DEVMAP_XMIT:
1206 			sample_map_count[i] = sample_n_cpus;
1207 			break;
1208 		case MAP_REDIRECT_ERR:
1209 			sample_map_count[i] =
1210 				XDP_REDIRECT_ERR_MAX * sample_n_cpus;
1211 			break;
1212 		case MAP_EXCEPTION:
1213 			sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus;
1214 		case MAP_CPUMAP_ENQUEUE:
1215 			sample_map_count[i] = sample_n_cpus * sample_n_cpus;
1216 			break;
1217 		default:
1218 			return -EINVAL;
1219 		}
1220 		if (bpf_map__set_max_entries(sample_map[i], sample_map_count[i]) < 0)
1221 			return -errno;
1222 	}
1223 	sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI];
1224 	return 0;
1225 }
1226 
sample_setup_maps_mappings(void)1227 static int sample_setup_maps_mappings(void)
1228 {
1229 	for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1230 		size_t size = sample_map_count[i] * sizeof(struct datarec);
1231 
1232 		sample_mmap[i] = mmap(NULL, size, PROT_READ | PROT_WRITE,
1233 				      MAP_SHARED, bpf_map__fd(sample_map[i]), 0);
1234 		if (sample_mmap[i] == MAP_FAILED)
1235 			return -errno;
1236 	}
1237 	return 0;
1238 }
1239 
__sample_init(int mask)1240 int __sample_init(int mask)
1241 {
1242 	sigset_t st;
1243 
1244 	sigemptyset(&st);
1245 	sigaddset(&st, SIGQUIT);
1246 	sigaddset(&st, SIGINT);
1247 	sigaddset(&st, SIGTERM);
1248 
1249 	if (sigprocmask(SIG_BLOCK, &st, NULL) < 0)
1250 		return -errno;
1251 
1252 	sample_sig_fd = signalfd(-1, &st, SFD_CLOEXEC | SFD_NONBLOCK);
1253 	if (sample_sig_fd < 0)
1254 		return -errno;
1255 
1256 	sample_mask = mask;
1257 
1258 	return sample_setup_maps_mappings();
1259 }
1260 
__sample_remove_xdp(int ifindex,__u32 prog_id,int xdp_flags)1261 static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags)
1262 {
1263 	__u32 cur_prog_id = 0;
1264 	int ret;
1265 
1266 	if (prog_id) {
1267 		ret = bpf_xdp_query_id(ifindex, xdp_flags, &cur_prog_id);
1268 		if (ret < 0)
1269 			return -errno;
1270 
1271 		if (prog_id != cur_prog_id) {
1272 			print_always(
1273 				"Program on ifindex %d does not match installed "
1274 				"program, skipping unload\n",
1275 				ifindex);
1276 			return -ENOENT;
1277 		}
1278 	}
1279 
1280 	return bpf_xdp_detach(ifindex, xdp_flags, NULL);
1281 }
1282 
sample_install_xdp(struct bpf_program * xdp_prog,int ifindex,bool generic,bool force)1283 int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic,
1284 		       bool force)
1285 {
1286 	int ret, xdp_flags = 0;
1287 	__u32 prog_id = 0;
1288 
1289 	if (sample_xdp_cnt == 32) {
1290 		fprintf(stderr,
1291 			"Total limit for installed XDP programs in a sample reached\n");
1292 		return -ENOTSUP;
1293 	}
1294 
1295 	xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0;
1296 	xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
1297 	ret = bpf_xdp_attach(ifindex, bpf_program__fd(xdp_prog), xdp_flags, NULL);
1298 	if (ret < 0) {
1299 		ret = -errno;
1300 		fprintf(stderr,
1301 			"Failed to install program \"%s\" on ifindex %d, mode = %s, "
1302 			"force = %s: %s\n",
1303 			bpf_program__name(xdp_prog), ifindex,
1304 			generic ? "skb" : "native", force ? "true" : "false",
1305 			strerror(-ret));
1306 		return ret;
1307 	}
1308 
1309 	ret = bpf_xdp_query_id(ifindex, xdp_flags, &prog_id);
1310 	if (ret < 0) {
1311 		ret = -errno;
1312 		fprintf(stderr,
1313 			"Failed to get XDP program id for ifindex %d, removing program: %s\n",
1314 			ifindex, strerror(errno));
1315 		__sample_remove_xdp(ifindex, 0, xdp_flags);
1316 		return ret;
1317 	}
1318 	sample_xdp_progs[sample_xdp_cnt++] =
1319 		(struct xdp_desc){ ifindex, prog_id, xdp_flags };
1320 
1321 	return 0;
1322 }
1323 
sample_summary_print(void)1324 static void sample_summary_print(void)
1325 {
1326 	double num = sample_out.rx_cnt.num;
1327 
1328 	if (sample_out.totals.rx) {
1329 		double pkts = sample_out.totals.rx;
1330 
1331 		print_always("  Packets received    : %'-10llu\n",
1332 			     sample_out.totals.rx);
1333 		print_always("  Average packets/s   : %'-10.0f\n",
1334 			     sample_round(pkts / num));
1335 	}
1336 	if (sample_out.totals.redir) {
1337 		double pkts = sample_out.totals.redir;
1338 
1339 		print_always("  Packets redirected  : %'-10llu\n",
1340 			     sample_out.totals.redir);
1341 		print_always("  Average redir/s     : %'-10.0f\n",
1342 			     sample_round(pkts / num));
1343 	}
1344 	if (sample_out.totals.drop)
1345 		print_always("  Rx dropped          : %'-10llu\n",
1346 			     sample_out.totals.drop);
1347 	if (sample_out.totals.drop_xmit)
1348 		print_always("  Tx dropped          : %'-10llu\n",
1349 			     sample_out.totals.drop_xmit);
1350 	if (sample_out.totals.err)
1351 		print_always("  Errors recorded     : %'-10llu\n",
1352 			     sample_out.totals.err);
1353 	if (sample_out.totals.xmit) {
1354 		double pkts = sample_out.totals.xmit;
1355 
1356 		print_always("  Packets transmitted : %'-10llu\n",
1357 			     sample_out.totals.xmit);
1358 		print_always("  Average transmit/s  : %'-10.0f\n",
1359 			     sample_round(pkts / num));
1360 	}
1361 }
1362 
sample_exit(int status)1363 void sample_exit(int status)
1364 {
1365 	size_t size;
1366 
1367 	for (int i = 0; i < NUM_MAP; i++) {
1368 		size = sample_map_count[i] * sizeof(**sample_mmap);
1369 		munmap(sample_mmap[i], size);
1370 	}
1371 	while (sample_xdp_cnt--) {
1372 		int i = sample_xdp_cnt, ifindex, xdp_flags;
1373 		__u32 prog_id;
1374 
1375 		prog_id = sample_xdp_progs[i].prog_id;
1376 		ifindex = sample_xdp_progs[i].ifindex;
1377 		xdp_flags = sample_xdp_progs[i].flags;
1378 
1379 		__sample_remove_xdp(ifindex, prog_id, xdp_flags);
1380 	}
1381 	sample_summary_print();
1382 	close(sample_sig_fd);
1383 	exit(status);
1384 }
1385 
sample_stats_collect(struct stats_record * rec)1386 static int sample_stats_collect(struct stats_record *rec)
1387 {
1388 	int i;
1389 
1390 	if (sample_mask & SAMPLE_RX_CNT)
1391 		map_collect_percpu(sample_mmap[MAP_RX], &rec->rx_cnt);
1392 
1393 	if (sample_mask & SAMPLE_REDIRECT_CNT)
1394 		map_collect_percpu(sample_mmap[MAP_REDIRECT_ERR], &rec->redir_err[0]);
1395 
1396 	if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) {
1397 		for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++)
1398 			map_collect_percpu(&sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus],
1399 					   &rec->redir_err[i]);
1400 	}
1401 
1402 	if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1403 		for (i = 0; i < sample_n_cpus; i++)
1404 			map_collect_percpu(&sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus],
1405 					   &rec->enq[i]);
1406 
1407 	if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT)
1408 		map_collect_percpu(sample_mmap[MAP_CPUMAP_KTHREAD],
1409 				   &rec->kthread);
1410 
1411 	if (sample_mask & SAMPLE_EXCEPTION_CNT)
1412 		for (i = 0; i < XDP_ACTION_MAX; i++)
1413 			map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus],
1414 					   &rec->exception[i]);
1415 
1416 	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT)
1417 		map_collect_percpu(sample_mmap[MAP_DEVMAP_XMIT], &rec->devmap_xmit);
1418 
1419 	if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) {
1420 		if (map_collect_percpu_devmap(bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0)
1421 			return -EINVAL;
1422 	}
1423 	return 0;
1424 }
1425 
sample_summary_update(struct sample_output * out)1426 static void sample_summary_update(struct sample_output *out)
1427 {
1428 	sample_out.totals.rx += out->totals.rx;
1429 	sample_out.totals.redir += out->totals.redir;
1430 	sample_out.totals.drop += out->totals.drop;
1431 	sample_out.totals.drop_xmit += out->totals.drop_xmit;
1432 	sample_out.totals.err += out->totals.err;
1433 	sample_out.totals.xmit += out->totals.xmit;
1434 	sample_out.rx_cnt.num++;
1435 }
1436 
sample_stats_print(int mask,struct stats_record * cur,struct stats_record * prev,char * prog_name)1437 static void sample_stats_print(int mask, struct stats_record *cur,
1438 			       struct stats_record *prev, char *prog_name)
1439 {
1440 	struct sample_output out = {};
1441 
1442 	if (mask & SAMPLE_RX_CNT)
1443 		stats_get_rx_cnt(cur, prev, 0, &out);
1444 	if (mask & SAMPLE_REDIRECT_CNT)
1445 		stats_get_redirect_cnt(cur, prev, 0, &out);
1446 	if (mask & SAMPLE_REDIRECT_ERR_CNT)
1447 		stats_get_redirect_err_cnt(cur, prev, 0, &out);
1448 	if (mask & SAMPLE_EXCEPTION_CNT)
1449 		stats_get_exception_cnt(cur, prev, 0, &out);
1450 	if (mask & SAMPLE_DEVMAP_XMIT_CNT)
1451 		stats_get_devmap_xmit(cur, prev, 0, &out);
1452 	else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1453 		stats_get_devmap_xmit_multi(cur, prev, 0, &out,
1454 					    mask & SAMPLE_DEVMAP_XMIT_CNT);
1455 	sample_summary_update(&out);
1456 
1457 	stats_print(prog_name, mask, cur, prev, &out);
1458 }
1459 
sample_switch_mode(void)1460 void sample_switch_mode(void)
1461 {
1462 	sample_log_level ^= LL_DEBUG - 1;
1463 }
1464 
sample_signal_cb(void)1465 static int sample_signal_cb(void)
1466 {
1467 	struct signalfd_siginfo si;
1468 	int r;
1469 
1470 	r = read(sample_sig_fd, &si, sizeof(si));
1471 	if (r < 0)
1472 		return -errno;
1473 
1474 	switch (si.ssi_signo) {
1475 	case SIGQUIT:
1476 		sample_switch_mode();
1477 		printf("\n");
1478 		break;
1479 	default:
1480 		printf("\n");
1481 		return 1;
1482 	}
1483 
1484 	return 0;
1485 }
1486 
1487 /* Pointer swap trick */
swap(struct stats_record ** a,struct stats_record ** b)1488 static void swap(struct stats_record **a, struct stats_record **b)
1489 {
1490 	struct stats_record *tmp;
1491 
1492 	tmp = *a;
1493 	*a = *b;
1494 	*b = tmp;
1495 }
1496 
sample_timer_cb(int timerfd,struct stats_record ** rec,struct stats_record ** prev)1497 static int sample_timer_cb(int timerfd, struct stats_record **rec,
1498 			   struct stats_record **prev)
1499 {
1500 	char line[64] = "Summary";
1501 	int ret;
1502 	__u64 t;
1503 
1504 	ret = read(timerfd, &t, sizeof(t));
1505 	if (ret < 0)
1506 		return -errno;
1507 
1508 	swap(prev, rec);
1509 	ret = sample_stats_collect(*rec);
1510 	if (ret < 0)
1511 		return ret;
1512 
1513 	if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) {
1514 		char fi[IFNAMSIZ];
1515 		char to[IFNAMSIZ];
1516 		const char *f, *t;
1517 
1518 		f = t = NULL;
1519 		if (if_indextoname(sample_xdp_progs[0].ifindex, fi))
1520 			f = fi;
1521 		if (if_indextoname(sample_xdp_progs[1].ifindex, to))
1522 			t = to;
1523 
1524 		snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?");
1525 	}
1526 
1527 	sample_stats_print(sample_mask, *rec, *prev, line);
1528 	return 0;
1529 }
1530 
sample_run(int interval,void (* post_cb)(void *),void * ctx)1531 int sample_run(int interval, void (*post_cb)(void *), void *ctx)
1532 {
1533 	struct timespec ts = { interval, 0 };
1534 	struct itimerspec its = { ts, ts };
1535 	struct stats_record *rec, *prev;
1536 	struct pollfd pfd[2] = {};
1537 	int timerfd, ret;
1538 
1539 	if (!interval) {
1540 		fprintf(stderr, "Incorrect interval 0\n");
1541 		return -EINVAL;
1542 	}
1543 	sample_interval = interval;
1544 	/* Pretty print numbers */
1545 	setlocale(LC_NUMERIC, "en_US.UTF-8");
1546 
1547 	timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK);
1548 	if (timerfd < 0)
1549 		return -errno;
1550 	timerfd_settime(timerfd, 0, &its, NULL);
1551 
1552 	pfd[0].fd = sample_sig_fd;
1553 	pfd[0].events = POLLIN;
1554 
1555 	pfd[1].fd = timerfd;
1556 	pfd[1].events = POLLIN;
1557 
1558 	ret = -ENOMEM;
1559 	rec = alloc_stats_record();
1560 	if (!rec)
1561 		goto end;
1562 	prev = alloc_stats_record();
1563 	if (!prev)
1564 		goto end_rec;
1565 
1566 	ret = sample_stats_collect(rec);
1567 	if (ret < 0)
1568 		goto end_rec_prev;
1569 
1570 	for (;;) {
1571 		ret = poll(pfd, 2, -1);
1572 		if (ret < 0) {
1573 			if (errno == EINTR)
1574 				continue;
1575 			else
1576 				break;
1577 		}
1578 
1579 		if (pfd[0].revents & POLLIN)
1580 			ret = sample_signal_cb();
1581 		else if (pfd[1].revents & POLLIN)
1582 			ret = sample_timer_cb(timerfd, &rec, &prev);
1583 
1584 		if (ret)
1585 			break;
1586 
1587 		if (post_cb)
1588 			post_cb(ctx);
1589 	}
1590 
1591 end_rec_prev:
1592 	free_stats_record(prev);
1593 end_rec:
1594 	free_stats_record(rec);
1595 end:
1596 	close(timerfd);
1597 
1598 	return ret;
1599 }
1600 
get_driver_name(int ifindex)1601 const char *get_driver_name(int ifindex)
1602 {
1603 	struct ethtool_drvinfo drv = {};
1604 	char ifname[IF_NAMESIZE];
1605 	static char drvname[32];
1606 	struct ifreq ifr = {};
1607 	int fd, r = 0;
1608 
1609 	fd = socket(AF_INET, SOCK_DGRAM, 0);
1610 	if (fd < 0)
1611 		return "[error]";
1612 
1613 	if (!if_indextoname(ifindex, ifname))
1614 		goto end;
1615 
1616 	drv.cmd = ETHTOOL_GDRVINFO;
1617 	safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1618 	ifr.ifr_data = (void *)&drv;
1619 
1620 	r = ioctl(fd, SIOCETHTOOL, &ifr);
1621 	if (r)
1622 		goto end;
1623 
1624 	safe_strncpy(drvname, drv.driver, sizeof(drvname));
1625 
1626 	close(fd);
1627 	return drvname;
1628 
1629 end:
1630 	r = errno;
1631 	close(fd);
1632 	return r == EOPNOTSUPP ? "loopback" : "[error]";
1633 }
1634 
get_mac_addr(int ifindex,void * mac_addr)1635 int get_mac_addr(int ifindex, void *mac_addr)
1636 {
1637 	char ifname[IF_NAMESIZE];
1638 	struct ifreq ifr = {};
1639 	int fd, r;
1640 
1641 	fd = socket(AF_INET, SOCK_DGRAM, 0);
1642 	if (fd < 0)
1643 		return -errno;
1644 
1645 	if (!if_indextoname(ifindex, ifname)) {
1646 		r = -errno;
1647 		goto end;
1648 	}
1649 
1650 	safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1651 
1652 	r = ioctl(fd, SIOCGIFHWADDR, &ifr);
1653 	if (r) {
1654 		r = -errno;
1655 		goto end;
1656 	}
1657 
1658 	memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
1659 
1660 end:
1661 	close(fd);
1662 	return r;
1663 }
1664 
sample_ctor(void)1665 __attribute__((constructor)) static void sample_ctor(void)
1666 {
1667 	if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) {
1668 		fprintf(stderr, "Failed to set libbpf strict mode: %s\n",
1669 			strerror(errno));
1670 		/* Just exit, nothing to cleanup right now */
1671 		exit(EXIT_FAIL_BPF);
1672 	}
1673 }
1674