153 	void (*tester)(int cg_fd, struct sockmap_options *opt);
416 			     struct sockmap_options *opt)  in msg_loop_sendpage()  argument
418 	bool drop = opt->drop_expected;  in msg_loop_sendpage()
553 		    struct sockmap_options *opt)  in msg_loop()  argument
557 	bool drop = opt->drop_expected;  in msg_loop()
558 	bool data = opt->data_test;  in msg_loop()
561 	if (!tx && opt->check_recved_len)  in msg_loop()
582 				if (opt->tx_wait_mem && errno == EACCES) {  in msg_loop()
645 				if (opt->verbose)  in msg_loop()
652 			if (opt->tx_wait_mem) {  in msg_loop()
685 			if (opt->check_recved_len && s->bytes_recvd > total_bytes) {  in msg_loop()
693 				int chunk_sz = opt->sendpage ?  in msg_loop()
737 static int sendmsg_test(struct sockmap_options *opt)  in sendmsg_test()  argument
742 	int iov_count = opt->iov_count;  in sendmsg_test()
743 	int iov_buf = opt->iov_length;  in sendmsg_test()
745 	int cnt = opt->rate;  in sendmsg_test()
749 	if (opt->base)  in sendmsg_test()
760 			err = sockmap_init_ktls(opt->verbose, rx_fd);  in sendmsg_test()
764 		err = sockmap_init_ktls(opt->verbose, c1);  in sendmsg_test()
769 	if (opt->tx_wait_mem) {  in sendmsg_test()
789 		if (opt->drop_expected || txmsg_ktls_skb_drop)  in sendmsg_test()
795 		if (opt->sendpage)  in sendmsg_test()
798 			       cnt, &s, false, opt);  in sendmsg_test()
799 		if (opt->verbose > 1)  in sendmsg_test()
807 		if (opt->verbose > 1)  in sendmsg_test()
821 	if (opt->tx_wait_mem)  in sendmsg_test()
826 		if (opt->sendpage)  in sendmsg_test()
827 			err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);  in sendmsg_test()
830 				       cnt, &s, true, opt);  in sendmsg_test()
840 		if (opt->verbose > 1)  in sendmsg_test()
869 static int forever_ping_pong(int rate, struct sockmap_options *opt)  in forever_ping_pong()  argument
934 		if (opt->verbose) {  in forever_ping_pong()
1376 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)  in __test_exec()  argument
1382 		opt->sendpage = true;  in __test_exec()
1384 		opt->sendpage = false;  in __test_exec()
1387 		opt->drop_expected = true;  in __test_exec()
1389 		opt->drop_expected = false;  in __test_exec()
1393 	if (opt->verbose) {  in __test_exec()
1396 			test_cnt, opt->rate, opt->iov_count, opt->iov_length,  in __test_exec()
1400 	err = run_options(opt, cgrp, test);  in __test_exec()
1401 	if (opt->verbose)  in __test_exec()
1409 static void test_exec(int cgrp, struct sockmap_options *opt)  in test_exec()  argument
1411 	int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);  in test_exec()
1416 		err = __test_exec(cgrp, SENDMSG, opt);  in test_exec()
1421 		err = __test_exec(cgrp, SENDPAGE, opt);  in test_exec()
1427 static void test_send_one(struct sockmap_options *opt, int cgrp)  in test_send_one()  argument
1429 	opt->iov_length = 1;  in test_send_one()
1430 	opt->iov_count = 1;  in test_send_one()
1431 	opt->rate = 1;  in test_send_one()
1432 	test_exec(cgrp, opt);  in test_send_one()
1434 	opt->iov_length = 1;  in test_send_one()
1435 	opt->iov_count = 1024;  in test_send_one()
1436 	opt->rate = 1;  in test_send_one()
1437 	test_exec(cgrp, opt);  in test_send_one()
1439 	opt->iov_length = 1024;  in test_send_one()
1440 	opt->iov_count = 1;  in test_send_one()
1441 	opt->rate = 1;  in test_send_one()
1442 	test_exec(cgrp, opt);  in test_send_one()
1446 static void test_send_many(struct sockmap_options *opt, int cgrp)  in test_send_many()  argument
1448 	opt->iov_length = 3;  in test_send_many()
1449 	opt->iov_count = 1;  in test_send_many()
1450 	opt->rate = 512;  in test_send_many()
1451 	test_exec(cgrp, opt);  in test_send_many()
1453 	opt->rate = 100;  in test_send_many()
1454 	opt->iov_count = 1;  in test_send_many()
1455 	opt->iov_length = 5;  in test_send_many()
1456 	test_exec(cgrp, opt);  in test_send_many()
1459 static void test_send_large(struct sockmap_options *opt, int cgrp)  in test_send_large()  argument
1461 	opt->iov_length = 256;  in test_send_large()
1462 	opt->iov_count = 1024;  in test_send_large()
1463 	opt->rate = 2;  in test_send_large()
1464 	test_exec(cgrp, opt);  in test_send_large()
1467 static void test_send(struct sockmap_options *opt, int cgrp)  in test_send()  argument
1469 	test_send_one(opt, cgrp);  in test_send()
1470 	test_send_many(opt, cgrp);  in test_send()
1471 	test_send_large(opt, cgrp);  in test_send()
1475 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)  in test_txmsg_pass()  argument
1479 	test_send(opt, cgrp);  in test_txmsg_pass()
1482 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)  in test_txmsg_redir()  argument
1485 	test_send(opt, cgrp);  in test_txmsg_redir()
1488 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)  in test_txmsg_redir_wait_sndmem()  argument
1491 	opt->tx_wait_mem = true;  in test_txmsg_redir_wait_sndmem()
1492 	test_send_large(opt, cgrp);  in test_txmsg_redir_wait_sndmem()
1493 	opt->tx_wait_mem = false;  in test_txmsg_redir_wait_sndmem()
1496 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)  in test_txmsg_drop()  argument
1499 	test_send(opt, cgrp);  in test_txmsg_drop()
1502 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)  in test_txmsg_ingress_redir()  argument
1506 	test_send(opt, cgrp);  in test_txmsg_ingress_redir()
1509 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)  in test_txmsg_skb()  argument
1511 	bool data = opt->data_test;  in test_txmsg_skb()
1514 	opt->data_test = true;  in test_txmsg_skb()
1526 	opt->iov_length = 100;  in test_txmsg_skb()
1527 	opt->iov_count = 1;  in test_txmsg_skb()
1528 	opt->rate = 1;  in test_txmsg_skb()
1529 	test_exec(cgrp, opt);  in test_txmsg_skb()
1532 	test_exec(cgrp, opt);  in test_txmsg_skb()
1536 	test_exec(cgrp, opt);  in test_txmsg_skb()
1543 	test_exec(cgrp, opt);  in test_txmsg_skb()
1546 	test_exec(cgrp, opt);  in test_txmsg_skb()
1550 	test_exec(cgrp, opt);  in test_txmsg_skb()
1553 	test_exec(cgrp, opt);  in test_txmsg_skb()
1556 	opt->data_test = data;  in test_txmsg_skb()
1567 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)  in test_txmsg_cork_hangs()  argument
1573 	test_send_large(opt, cgrp);  in test_txmsg_cork_hangs()
1579 	test_send_large(opt, cgrp);  in test_txmsg_cork_hangs()
1585 	test_send_large(opt, cgrp);  in test_txmsg_cork_hangs()
1588 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)  in test_txmsg_pull()  argument
1593 	test_send(opt, cgrp);  in test_txmsg_pull()
1598 	test_send_large(opt, cgrp);  in test_txmsg_pull()
1604 	test_send(opt, cgrp);  in test_txmsg_pull()
1611 	test_send_many(opt, cgrp);  in test_txmsg_pull()
1618 	test_send_many(opt, cgrp);  in test_txmsg_pull()
1621 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)  in test_txmsg_pop()  argument
1626 	test_send_many(opt, cgrp);  in test_txmsg_pop()
1631 	test_send_large(opt, cgrp);  in test_txmsg_pop()
1637 	test_send_many(opt, cgrp);  in test_txmsg_pop()
1644 	test_send_many(opt, cgrp);  in test_txmsg_pop()
1651 	test_send_many(opt, cgrp);  in test_txmsg_pop()
1654 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)  in test_txmsg_push()  argument
1659 	test_send(opt, cgrp);  in test_txmsg_push()
1664 	test_send_large(opt, cgrp);  in test_txmsg_push()
1670 	test_send_many(opt, cgrp);  in test_txmsg_push()
1677 	test_send_many(opt, cgrp);  in test_txmsg_push()
1680 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)  in test_txmsg_push_pop()  argument
1686 	test_send_large(opt, cgrp);  in test_txmsg_push_pop()
1689 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)  in test_txmsg_apply()  argument
1696 	test_send_one(opt, cgrp);  in test_txmsg_apply()
1703 	test_send_one(opt, cgrp);  in test_txmsg_apply()
1710 	test_send_one(opt, cgrp);  in test_txmsg_apply()
1717 	test_send_large(opt, cgrp);  in test_txmsg_apply()
1724 	test_send_large(opt, cgrp);  in test_txmsg_apply()
1731 	test_send_large(opt, cgrp);  in test_txmsg_apply()
1734 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)  in test_txmsg_cork()  argument
1740 	test_send(opt, cgrp);  in test_txmsg_cork()
1746 	test_send(opt, cgrp);  in test_txmsg_cork()
1749 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)  in test_txmsg_ingress_parser()  argument
1755 	opt->iov_length = 256;  in test_txmsg_ingress_parser()
1756 	opt->iov_count = 1;  in test_txmsg_ingress_parser()
1757 	opt->rate = 2;  in test_txmsg_ingress_parser()
1758 	test_exec(cgrp, opt);  in test_txmsg_ingress_parser()
1761 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)  in test_txmsg_ingress_parser2()  argument
1766 	opt->iov_length = 20;  in test_txmsg_ingress_parser2()
1767 	opt->iov_count = 1;  in test_txmsg_ingress_parser2()
1768 	opt->rate = 1;  in test_txmsg_ingress_parser2()
1769 	opt->check_recved_len = true;  in test_txmsg_ingress_parser2()
1770 	test_exec(cgrp, opt);  in test_txmsg_ingress_parser2()
1771 	opt->check_recved_len = false;  in test_txmsg_ingress_parser2()
1877 static int check_whitelist(struct _test *t, struct sockmap_options *opt)  in check_whitelist()  argument
1881 	if (!opt->whitelist)  in check_whitelist()
1883 	ptr = strdup(opt->whitelist);  in check_whitelist()
1888 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||  in check_whitelist()
1889 		    strstr(opt->map, entry) != 0 ||  in check_whitelist()
1897 static int check_blacklist(struct _test *t, struct sockmap_options *opt)  in check_blacklist()  argument
1901 	if (!opt->blacklist)  in check_blacklist()
1903 	ptr = strdup(opt->blacklist);  in check_blacklist()
1908 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||  in check_blacklist()
1909 		    strstr(opt->map, entry) != 0 ||  in check_blacklist()
1917 static int __test_selftests(int cg_fd, struct sockmap_options *opt)  in __test_selftests()  argument
1921 	err = populate_progs(opt->map);  in __test_selftests()
1931 		if (check_whitelist(&t, opt) != 0)  in __test_selftests()
1933 		if (check_blacklist(&t, opt) == 0)  in __test_selftests()
1936 		test_start_subtest(&t, opt);  in __test_selftests()
1937 		t.tester(cg_fd, opt);  in __test_selftests()
1944 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)  in test_selftests_sockmap()  argument
1946 	opt->map = BPF_SOCKMAP_FILENAME;  in test_selftests_sockmap()
1947 	__test_selftests(cg_fd, opt);  in test_selftests_sockmap()
1950 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)  in test_selftests_sockhash()  argument
1952 	opt->map = BPF_SOCKHASH_FILENAME;  in test_selftests_sockhash()
1953 	__test_selftests(cg_fd, opt);  in test_selftests_sockhash()
1956 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)  in test_selftests_ktls()  argument
1958 	opt->map = BPF_SOCKHASH_FILENAME;  in test_selftests_ktls()
1959 	opt->prepend = "ktls";  in test_selftests_ktls()
1961 	__test_selftests(cg_fd, opt);  in test_selftests_ktls()
1965 static int test_selftest(int cg_fd, struct sockmap_options *opt)  in test_selftest()  argument
1968 	test_selftests_sockmap(cg_fd, opt);  in test_selftest()
1969 	test_selftests_sockhash(cg_fd, opt);  in test_selftest()
1970 	test_selftests_ktls(cg_fd, opt);  in test_selftest()
1979 	int opt, longindex, err, cg_fd = 0;  in main()  local
1984 	while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",  in main()
1986 		switch (opt) {  in main()