157 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)  in gen_eth_hdr()  argument
159 	memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);  in gen_eth_hdr()
160 	memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);  in gen_eth_hdr()
164 static bool is_umem_valid(struct ifobject *ifobj)  in is_umem_valid()
179 static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,  in xsk_configure_umem()
246 				  struct ifobject *ifobject, bool shared)  in __xsk_configure_socket()  argument
255 	cfg.bind_flags = ifobject->bind_flags;  in __xsk_configure_socket()
258 	if (ifobject->pkt_stream && ifobject->mtu > MAX_ETH_PKT_SIZE)  in __xsk_configure_socket()
261 	txr = ifobject->tx_on ? &xsk->tx : NULL;  in __xsk_configure_socket()
262 	rxr = ifobject->rx_on ? &xsk->rx : NULL;  in __xsk_configure_socket()
263 	return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);  in __xsk_configure_socket()
266 static bool ifobj_zc_avail(struct ifobject *ifobject)  in ifobj_zc_avail()  argument
286 	ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);  in ifobj_zc_avail()
293 	ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;  in ifobj_zc_avail()
294 	ifobject->rx_on = true;  in ifobj_zc_avail()
296 	ret = __xsk_configure_socket(xsk, umem, ifobject, false);  in ifobj_zc_avail()
328 static bool validate_interface(struct ifobject *ifobj)  in validate_interface()
335 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,  in parse_command_line()
338 	struct ifobject *ifobj;  in parse_command_line()
381 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,  in __test_spec_init()
382 			     struct ifobject *ifobj_rx)  in __test_spec_init()
387 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;  in __test_spec_init()
429 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,  in test_spec_init()
430 			   struct ifobject *ifobj_rx, enum test_mode mode)  in test_spec_init()
443 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;  in test_spec_init()
651 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,  in __pkt_stream_replace_half()
696 static void pkt_generate(struct ifobject *ifobject, u64 addr, u32 len, u32 pkt_nb,  in pkt_generate()  argument
699 	void *data = xsk_umem__get_data(ifobject->umem->buffer, addr);  in pkt_generate()
705 		gen_eth_hdr(ifobject, data);  in pkt_generate()
716 static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames,  in __pkt_stream_generate_custom()
973 	struct ifobject *ifobj = test->ifobj_rx;  in receive_pkts()
1095 static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeout)  in __send_pkts()  argument
1098 	struct pkt_stream *pkt_stream = ifobject->pkt_stream;  in __send_pkts()
1099 	struct xsk_socket_info *xsk = ifobject->xsk;  in __send_pkts()
1100 	struct xsk_umem_info *umem = ifobject->umem;  in __send_pkts()
1101 	bool use_poll = ifobject->use_poll;  in __send_pkts()
1152 			tx_desc->addr = pkt_get_addr(pkt, ifobject->umem);  in __send_pkts()
1164 				pkt_generate(ifobject, tx_desc->addr, tx_desc->len, pkt->pkt_nb,  in __send_pkts()
1216 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)  in send_pkts()  argument
1218 	struct pkt_stream *pkt_stream = ifobject->pkt_stream;  in send_pkts()
1223 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);  in send_pkts()
1227 		ret = __send_pkts(ifobject, &fds, timeout);  in send_pkts()
1236 	wait_for_tx_completion(ifobject->xsk);  in send_pkts()
1263 static int validate_rx_dropped(struct ifobject *ifobject)  in validate_rx_dropped()  argument
1265 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_rx_dropped()
1269 	kick_rx(ifobject->xsk);  in validate_rx_dropped()
1281 	if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 ||  in validate_rx_dropped()
1282 	    stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 - 1)  in validate_rx_dropped()
1288 static int validate_rx_full(struct ifobject *ifobject)  in validate_rx_full()  argument
1290 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_rx_full()
1295 	kick_rx(ifobject->xsk);  in validate_rx_full()
1307 static int validate_fill_empty(struct ifobject *ifobject)  in validate_fill_empty()  argument
1309 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_fill_empty()
1314 	kick_rx(ifobject->xsk);  in validate_fill_empty()
1326 static int validate_tx_invalid_descs(struct ifobject *ifobject)  in validate_tx_invalid_descs()  argument
1328 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in validate_tx_invalid_descs()
1342 	if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {  in validate_tx_invalid_descs()
1344 			       __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);  in validate_tx_invalid_descs()
1351 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,  in xsk_configure_socket()  argument
1357 		bool shared = (ifobject->shared_umem && tx) ? true : !!i;  in xsk_configure_socket()
1361 			ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,  in xsk_configure_socket()
1362 						     ifobject, shared);  in xsk_configure_socket()
1371 		if (ifobject->busy_poll)  in xsk_configure_socket()
1372 			enable_busy_poll(&ifobject->xsk_arr[i]);  in xsk_configure_socket()
1376 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)  in thread_common_ops_tx()  argument
1378 	xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);  in thread_common_ops_tx()
1379 	ifobject->xsk = &ifobject->xsk_arr[0];  in thread_common_ops_tx()
1380 	ifobject->xskmap = test->ifobj_rx->xskmap;  in thread_common_ops_tx()
1381 	memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));  in thread_common_ops_tx()
1382 	ifobject->umem->base_addr = 0;  in thread_common_ops_tx()
1429 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)  in thread_common_ops()  argument
1431 	u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;  in thread_common_ops()
1437 	if (ifobject->umem->unaligned_mode)  in thread_common_ops()
1440 	if (ifobject->shared_umem)  in thread_common_ops()
1447 	ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz);  in thread_common_ops()
1451 	xsk_configure_socket(test, ifobject, ifobject->umem, false);  in thread_common_ops()
1453 	ifobject->xsk = &ifobject->xsk_arr[0];  in thread_common_ops()
1455 	if (!ifobject->rx_on)  in thread_common_ops()
1458 	xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream, ifobject->use_fill_ring);  in thread_common_ops()
1460 	ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);  in thread_common_ops()
1468 	struct ifobject *ifobject = test->ifobj_tx;  in worker_testapp_validate_tx()  local
1472 		if (!ifobject->shared_umem)  in worker_testapp_validate_tx()
1473 			thread_common_ops(test, ifobject);  in worker_testapp_validate_tx()
1475 			thread_common_ops_tx(test, ifobject);  in worker_testapp_validate_tx()
1478 	print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,  in worker_testapp_validate_tx()
1479 		      ifobject->ifname);  in worker_testapp_validate_tx()
1480 	err = send_pkts(test, ifobject);  in worker_testapp_validate_tx()
1482 	if (!err && ifobject->validation_func)  in worker_testapp_validate_tx()
1483 		err = ifobject->validation_func(ifobject);  in worker_testapp_validate_tx()
1493 	struct ifobject *ifobject = test->ifobj_rx;  in worker_testapp_validate_rx()  local
1498 		thread_common_ops(test, ifobject);  in worker_testapp_validate_rx()
1500 		xsk_clear_xskmap(ifobject->xskmap);  in worker_testapp_validate_rx()
1501 		err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);  in worker_testapp_validate_rx()
1508 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);  in worker_testapp_validate_rx()
1515 	if (!err && ifobject->validation_func)  in worker_testapp_validate_rx()
1516 		err = ifobject->validation_func(ifobject);  in worker_testapp_validate_rx()
1528 static void testapp_clean_xsk_umem(struct ifobject *ifobj)  in testapp_clean_xsk_umem()
1547 	struct ifobject *ifobj = test->ifobj_rx;  in xdp_prog_changed_rx()
1554 	struct ifobject *ifobj = test->ifobj_tx;  in xdp_prog_changed_tx()
1559 static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,  in xsk_reattach_xdp()
1582 static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,  in xsk_attach_xdp_progs()
1583 				 struct ifobject *ifobj_tx)  in xsk_attach_xdp_progs()
1595 static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,  in __testapp_validate_traffic()
1596 				      struct ifobject *ifobj2)  in __testapp_validate_traffic()
1663 	struct ifobject *ifobj_rx = test->ifobj_rx;  in testapp_validate_traffic()
1664 	struct ifobject *ifobj_tx = test->ifobj_tx;  in testapp_validate_traffic()
1676 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)  in testapp_validate_traffic_single_thread()
1695 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)  in swap_directions()
1698 	struct ifobject *tmp_ifobj = (*ifobj1);  in swap_directions()
1726 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)  in swap_xsk_resources()
2027 static int xsk_load_xdp_programs(struct ifobject *ifobj)  in xsk_load_xdp_programs()
2036 static void xsk_unload_xdp_programs(struct ifobject *ifobj)  in xsk_unload_xdp_programs()
2057 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,  in init_iface()
2233 static struct ifobject *ifobject_create(void)  in ifobject_create()
2235 	struct ifobject *ifobj;  in ifobject_create()
2237 	ifobj = calloc(1, sizeof(struct ifobject));  in ifobject_create()
2258 static void ifobject_delete(struct ifobject *ifobj)  in ifobject_delete()
2297 	struct ifobject *ifobj_tx, *ifobj_rx;  in main()