229 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)  in gen_eth_hdr()  argument
231 	memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);  in gen_eth_hdr()
232 	memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);  in gen_eth_hdr()
236 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)  in gen_ip_hdr()  argument
246 	ip_hdr->saddr = ifobject->src_ip;  in gen_ip_hdr()
247 	ip_hdr->daddr = ifobject->dst_ip;  in gen_ip_hdr()
251 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject,  in gen_udp_hdr()  argument
254 	udp_hdr->source = htons(ifobject->src_port);  in gen_udp_hdr()
255 	udp_hdr->dest = htons(ifobject->dst_port);  in gen_udp_hdr()
260 static bool is_umem_valid(struct ifobject *ifobj)  in is_umem_valid()
316 				  struct ifobject *ifobject, bool shared)  in __xsk_configure_socket()  argument
326 	cfg.xdp_flags = ifobject->xdp_flags;  in __xsk_configure_socket()
327 	cfg.bind_flags = ifobject->bind_flags;  in __xsk_configure_socket()
331 	txr = ifobject->tx_on ? &xsk->tx : NULL;  in __xsk_configure_socket()
332 	rxr = ifobject->rx_on ? &xsk->rx : NULL;  in __xsk_configure_socket()
333 	return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);  in __xsk_configure_socket()
336 static bool ifobj_zc_avail(struct ifobject *ifobject)  in ifobj_zc_avail()  argument
363 	ifobject->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;  in ifobj_zc_avail()
364 	ifobject->xdp_flags |= XDP_FLAGS_DRV_MODE;  in ifobj_zc_avail()
365 	ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;  in ifobj_zc_avail()
366 	ifobject->rx_on = true;  in ifobj_zc_avail()
368 	ret = __xsk_configure_socket(xsk, umem, ifobject, false);  in ifobj_zc_avail()
424 static bool validate_interface(struct ifobject *ifobj)  in validate_interface()
431 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,  in parse_command_line()
434 	struct ifobject *ifobj;  in parse_command_line()
480 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,  in __test_spec_init()
481 			     struct ifobject *ifobj_rx)  in __test_spec_init()
486 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;  in __test_spec_init()
525 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,  in test_spec_init()
526 			   struct ifobject *ifobj_rx, enum test_mode mode)  in test_spec_init()
539 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;  in test_spec_init()
678 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,  in __pkt_stream_replace_half()
712 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)  in pkt_generate()  argument
714 	struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);  in pkt_generate()
725 	data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);  in pkt_generate()
730 	gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr);  in pkt_generate()
731 	gen_ip_hdr(ifobject, ip_hdr);  in pkt_generate()
733 	gen_eth_hdr(ifobject, eth_hdr);  in pkt_generate()
738 static void __pkt_stream_generate_custom(struct ifobject *ifobj,  in __pkt_stream_generate_custom()
915 	struct ifobject *ifobj = test->ifobj_rx;  in receive_pkts()
1004 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds,  in __send_pkts()  argument
1007 	struct xsk_socket_info *xsk = ifobject->xsk;  in __send_pkts()
1008 	bool use_poll = ifobject->use_poll;  in __send_pkts()
1036 		struct pkt *pkt = pkt_generate(ifobject, *pkt_nb);  in __send_pkts()
1051 	if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) {  in __send_pkts()
1088 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)  in send_pkts()  argument
1094 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);  in send_pkts()
1097 	while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {  in send_pkts()
1098 		ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout);  in send_pkts()
1105 	wait_for_tx_completion(ifobject->xsk);  in send_pkts()
1132 static int validate_rx_dropped(struct ifobject *ifobject)  in validate_rx_dropped()  argument
1134 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_rx_dropped()
1138 	kick_rx(ifobject->xsk);  in validate_rx_dropped()
1144 	if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2)  in validate_rx_dropped()
1150 static int validate_rx_full(struct ifobject *ifobject)  in validate_rx_full()  argument
1152 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_rx_full()
1157 	kick_rx(ifobject->xsk);  in validate_rx_full()
1169 static int validate_fill_empty(struct ifobject *ifobject)  in validate_fill_empty()  argument
1171 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_fill_empty()
1176 	kick_rx(ifobject->xsk);  in validate_fill_empty()
1188 static int validate_tx_invalid_descs(struct ifobject *ifobject)  in validate_tx_invalid_descs()  argument
1190 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_tx_invalid_descs()
1204 	if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {  in validate_tx_invalid_descs()
1206 			       __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);  in validate_tx_invalid_descs()
1213 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,  in xsk_configure_socket()  argument
1219 		bool shared = (ifobject->shared_umem && tx) ? true : !!i;  in xsk_configure_socket()
1223 			ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,  in xsk_configure_socket()
1224 						     ifobject, shared);  in xsk_configure_socket()
1233 		if (ifobject->busy_poll)  in xsk_configure_socket()
1234 			enable_busy_poll(&ifobject->xsk_arr[i]);  in xsk_configure_socket()
1238 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)  in thread_common_ops_tx()  argument
1240 	xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);  in thread_common_ops_tx()
1241 	ifobject->xsk = &ifobject->xsk_arr[0];  in thread_common_ops_tx()
1242 	ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd;  in thread_common_ops_tx()
1243 	memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));  in thread_common_ops_tx()
1277 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)  in thread_common_ops()  argument
1279 	u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;  in thread_common_ops()
1285 	ifobject->ns_fd = switch_namespace(ifobject->nsname);  in thread_common_ops()
1287 	if (ifobject->umem->unaligned_mode)  in thread_common_ops()
1290 	if (ifobject->shared_umem)  in thread_common_ops()
1297 	ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz);  in thread_common_ops()
1301 	xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);  in thread_common_ops()
1303 	xsk_configure_socket(test, ifobject, ifobject->umem, false);  in thread_common_ops()
1305 	ifobject->xsk = &ifobject->xsk_arr[0];  in thread_common_ops()
1307 	if (!ifobject->rx_on)  in thread_common_ops()
1310 	ifindex = if_nametoindex(ifobject->ifname);  in thread_common_ops()
1314 	ret = xsk_setup_xdp_prog_xsk(ifobject->xsk->xsk, &ifobject->xsk_map_fd);  in thread_common_ops()
1318 	ret = bpf_xdp_query(ifindex, ifobject->xdp_flags, &opts);  in thread_common_ops()
1322 	if (ifobject->xdp_flags & XDP_FLAGS_SKB_MODE) {  in thread_common_ops()
1327 	} else if (ifobject->xdp_flags & XDP_FLAGS_DRV_MODE) {  in thread_common_ops()
1334 	ret = xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);  in thread_common_ops()
1342 	struct ifobject *ifobject = test->ifobj_tx;  in worker_testapp_validate_tx()  local
1346 		if (!ifobject->shared_umem)  in worker_testapp_validate_tx()
1347 			thread_common_ops(test, ifobject);  in worker_testapp_validate_tx()
1349 			thread_common_ops_tx(test, ifobject);  in worker_testapp_validate_tx()
1352 	print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,  in worker_testapp_validate_tx()
1353 		      ifobject->ifname);  in worker_testapp_validate_tx()
1354 	err = send_pkts(test, ifobject);  in worker_testapp_validate_tx()
1356 	if (!err && ifobject->validation_func)  in worker_testapp_validate_tx()
1357 		err = ifobject->validation_func(ifobject);  in worker_testapp_validate_tx()
1367 	struct ifobject *ifobject = test->ifobj_rx;  in worker_testapp_validate_rx()  local
1373 		thread_common_ops(test, ifobject);  in worker_testapp_validate_rx()
1375 		bpf_map_delete_elem(ifobject->xsk_map_fd, &id);  in worker_testapp_validate_rx()
1376 		xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);  in worker_testapp_validate_rx()
1379 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);  in worker_testapp_validate_rx()
1386 	if (!err && ifobject->validation_func)  in worker_testapp_validate_rx()
1387 		err = ifobject->validation_func(ifobject);  in worker_testapp_validate_rx()
1398 static void testapp_clean_xsk_umem(struct ifobject *ifobj)  in testapp_clean_xsk_umem()
1414 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj,  in testapp_validate_traffic_single_thread()
1457 	struct ifobject *ifobj_tx = test->ifobj_tx;  in testapp_validate_traffic()
1458 	struct ifobject *ifobj_rx = test->ifobj_rx;  in testapp_validate_traffic()
1504 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)  in swap_directions()
1507 	struct ifobject *tmp_ifobj = (*ifobj1);  in swap_directions()
1532 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)  in swap_xsk_resources()
1620 static bool hugepages_present(struct ifobject *ifobject)  in hugepages_present()  argument
1622 	const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size;  in hugepages_present()
1703 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,  in init_iface()
1829 static struct ifobject *ifobject_create(void)  in ifobject_create()
1831 	struct ifobject *ifobj;  in ifobject_create()
1833 	ifobj = calloc(1, sizeof(struct ifobject));  in ifobject_create()
1856 static void ifobject_delete(struct ifobject *ifobj)  in ifobject_delete()
1865 static bool is_xdp_supported(struct ifobject *ifobject)  in is_xdp_supported()  argument
1874 	int ifindex = if_nametoindex(ifobject->ifname);  in is_xdp_supported()
1898 	struct ifobject *ifobj_tx, *ifobj_rx;  in main()