1 // SPDX-License-Identifier: GPL-2.0
13  * a unique UMEM. It validates in-order packet delivery and packet content
17  * ------------------
22  *    a. nopoll - soft-irq processing
23  *    b. poll - using poll() syscall
27  *    d. Bi-directional sockets
28  *       Configure sockets as bi-directional tx/rx sockets, sets up fill and
52  * -----
53  * - Single process spawns two threads: Tx and Rx
54  * - Each of these two threads attach to a veth interface within their assigned
56  * - Each thread Creates one AF_XDP socket connected to a unique umem for each
58  * - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy>
59  * - Rx thread verifies if all 10k packets were received and delivered in-order,
63  * --------------------------
64  * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
65  * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
119 			       test_type == TEST_TYPE_BIDI ? "Bi-directional Sockets" : "",\
137  * Linux kernel include/asm-generic/checksum.h
154 	/* add up 32-bit and 32-bit for 32+c bit */  in from64to32()
181  * Linux kernel include/asm-generic/checksum.h
202 	memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);  in gen_eth_hdr()
203 	memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);  in gen_eth_hdr()
204 	eth_hdr->h_proto = htons(ETH_P_IP);  in gen_eth_hdr()
209 	ip_hdr->version = IP_PKT_VER;  in gen_ip_hdr()
210 	ip_hdr->ihl = 0x5;  in gen_ip_hdr()
211 	ip_hdr->tos = IP_PKT_TOS;  in gen_ip_hdr()
212 	ip_hdr->tot_len = htons(IP_PKT_SIZE);  in gen_ip_hdr()
213 	ip_hdr->id = 0;  in gen_ip_hdr()
214 	ip_hdr->frag_off = 0;  in gen_ip_hdr()
215 	ip_hdr->ttl = IPDEFTTL;  in gen_ip_hdr()
216 	ip_hdr->protocol = IPPROTO_UDP;  in gen_ip_hdr()
217 	ip_hdr->saddr = ifobject->src_ip;  in gen_ip_hdr()
218 	ip_hdr->daddr = ifobject->dst_ip;  in gen_ip_hdr()
219 	ip_hdr->check = 0;  in gen_ip_hdr()
225 	udp_hdr->source = htons(ifobject->src_port);  in gen_udp_hdr()
226 	udp_hdr->dest = htons(ifobject->dst_port);  in gen_udp_hdr()
227 	udp_hdr->len = htons(UDP_PKT_SIZE);  in gen_udp_hdr()
233 	udp_hdr->check = 0;  in gen_udp_csum()
234 	udp_hdr->check =  in gen_udp_csum()
235 	    udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);  in gen_udp_csum()
254 	ret = xsk_umem__create(&umem->umem, buffer, size,  in xsk_configure_umem()
255 			       &umem->fq, &umem->cq, &cfg);  in xsk_configure_umem()
257 		exit_with_error(-ret);  in xsk_configure_umem()
259 	umem->buffer = buffer;  in xsk_configure_umem()
261 	data->umem_arr[idx] = umem;  in xsk_configure_umem()
269 	ret = xsk_ring_prod__reserve(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);  in xsk_populate_fill_ring()
271 		exit_with_error(-ret);  in xsk_populate_fill_ring()
273 		*xsk_ring_prod__fill_addr(&umem->fq, idx++) = i * XSK_UMEM__DEFAULT_FRAME_SIZE;  in xsk_populate_fill_ring()
274 	xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);  in xsk_populate_fill_ring()
289 	xsk->umem = ifobject->umem;  in xsk_configure_socket()
297 		rxr = (ifobject->fv.vector == rx) ? &xsk->rx : NULL;  in xsk_configure_socket()
298 		txr = (ifobject->fv.vector == tx) ? &xsk->tx : NULL;  in xsk_configure_socket()
300 		rxr = &xsk->rx;  in xsk_configure_socket()
301 		txr = &xsk->tx;  in xsk_configure_socket()
304 	ret = xsk_socket__create(&xsk->xsk, ifobject->ifname, idx,  in xsk_configure_socket()
305 				 ifobject->umem->umem, rxr, txr, &cfg);  in xsk_configure_socket()
309 	ifobject->xsk_arr[idx] = xsk;  in xsk_configure_socket()
317 	{"dump-pkts", optional_argument, 0, 'D'},
327 		"  -i, --interface      Use interface\n"  in usage()
328 		"  -q, --queue=n        Use queue n (default 0)\n"  in usage()
329 		"  -D, --dump-pkts      Dump packets L2 - L5\n"  in usage()
330 		"  -v, --verbose        Verbose output\n";  in usage()
341 		return -1;  in switch_namespace()
343 	strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1);  in switch_namespace()
346 	if (nsfd == -1)  in switch_namespace()
349 	if (setns(nsfd, 0) == -1)  in switch_namespace()
362 		if (!strcmp(ifdict[i]->ifname, "")) {  in validate_interfaces()
364 			ksft_test_result_fail("ERROR: interfaces: -i <int>,<ns> -i <int>,<ns>.");  in validate_interfaces()
379 		if (c == -1)  in parse_command_line()
389 			memcpy(ifdict[interface_index]->ifname,  in parse_command_line()
393 				memcpy(ifdict[interface_index]->nsname, token,  in parse_command_line()
417 	if (pkt_nb >= pkt_stream->nb_pkts)  in pkt_stream_get_pkt()
420 	return &pkt_stream->pkts[pkt_nb];  in pkt_stream_get_pkt()
432 	pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));  in pkt_stream_generate()
433 	if (!pkt_stream->pkts)  in pkt_stream_generate()
436 	pkt_stream->nb_pkts = nb_pkts;  in pkt_stream_generate()
438 		pkt_stream->pkts[i].addr = (i % num_frames) * XSK_UMEM__DEFAULT_FRAME_SIZE;  in pkt_stream_generate()
439 		pkt_stream->pkts[i].len = pkt_len;  in pkt_stream_generate()
440 		pkt_stream->pkts[i].payload = i;  in pkt_stream_generate()
448 	struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);  in pkt_generate()
457 	data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);  in pkt_generate()
485 		fprintf(stdout, "%02X", ethhdr->h_dest[i]);  in pkt_dump()
489 		fprintf(stdout, "%02X", ethhdr->h_source[i]);  in pkt_dump()
492 	fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);  in pkt_dump()
493 	fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",  in pkt_dump()
494 		inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));  in pkt_dump()
495 	fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",  in pkt_dump()
496 		inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));  in pkt_dump()
498 	fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));  in pkt_dump()
499 	fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));  in pkt_dump()
504 	fprintf(stdout, "---------------------------------------\n");  in pkt_dump()
509 	void *data = xsk_umem__get_data(buffer, desc->addr);  in is_pkt_valid()
517 	if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {  in is_pkt_valid()
523 		if (pkt->len != desc->len) {  in is_pkt_valid()
526 					__func__, pkt->len, desc->len);  in is_pkt_valid()
530 		if (pkt->payload != seqnum) {  in is_pkt_valid()
533 					__func__, pkt->payload, seqnum);  in is_pkt_valid()
538 		ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,  in is_pkt_valid()
539 			       iphdr->tos);  in is_pkt_valid()
550 	ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);  in kick_tx()
561 	if (!xsk->outstanding_tx)  in complete_pkts()
564 	if (xsk_ring_prod__needs_wakeup(&xsk->tx))  in complete_pkts()
567 	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);  in complete_pkts()
569 		xsk_ring_cons__release(&xsk->umem->cq, rcvd);  in complete_pkts()
570 		xsk->outstanding_tx -= rcvd;  in complete_pkts()
583 		rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);  in receive_pkts()
585 			if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {  in receive_pkts()
588 					exit_with_error(-ret);  in receive_pkts()
593 		ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);  in receive_pkts()
596 				exit_with_error(-ret);  in receive_pkts()
597 			if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {  in receive_pkts()
600 					exit_with_error(-ret);  in receive_pkts()
602 			ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);  in receive_pkts()
606 			const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);  in receive_pkts()
607 			u64 addr = desc->addr, orig;  in receive_pkts()
611 			if (!is_pkt_valid(pkt, xsk->umem->buffer, desc))  in receive_pkts()
614 			*xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig;  in receive_pkts()
618 		xsk_ring_prod__submit(&xsk->umem->fq, rcvd);  in receive_pkts()
619 		xsk_ring_cons__release(&xsk->rx, rcvd);  in receive_pkts()
625 	struct xsk_socket_info *xsk = ifobject->xsk;  in __send_pkts()
628 	while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE)  in __send_pkts()
632 		struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);  in __send_pkts()
638 		tx_desc->addr = pkt->addr;  in __send_pkts()
639 		tx_desc->len = pkt->len;  in __send_pkts()
643 	xsk_ring_prod__submit(&xsk->tx, i);  in __send_pkts()
645 		xsk->outstanding_tx += i;  in __send_pkts()
646 	else if (xsk_ring_prod__needs_wakeup(&xsk->tx))  in __send_pkts()
655 	while (xsk->outstanding_tx)  in wait_for_tx_completion()
664 	fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);  in send_pkts()
667 	while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {  in send_pkts()
686 	wait_for_tx_completion(ifobject->xsk);  in send_pkts()
691 	u32 xsk_stat = 0, expected_stat = ifobject->pkt_stream->nb_pkts;  in rx_stats_are_valid()
692 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in rx_stats_are_valid()
702 				      __func__, -err, strerror(-err));  in rx_stats_are_valid()
715 			expected_stat -= RX_FULL_RXQSIZE;  in rx_stats_are_valid()
733 	struct xsk_socket *xsk = ifobject->xsk->xsk;  in tx_stats_validate()
743 				      __func__, -err, strerror(-err));  in tx_stats_validate()
747 	if (stats.tx_invalid_descs == ifobject->pkt_stream->nb_pkts)  in tx_stats_validate()
751 			      __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);  in tx_stats_validate()
762 	ifobject->ns_fd = switch_namespace(ifobject->nsname);  in thread_common_ops()
767 	bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);  in thread_common_ops()
773 		ifobject->umem = ifobject->umem_arr[0];  in thread_common_ops()
781 			exit_with_error(-ret);  in thread_common_ops()
784 	ifobject->umem = ifobject->umem_arr[0];  in thread_common_ops()
785 	ifobject->xsk = ifobject->xsk_arr[0];  in thread_common_ops()
789 		ifobject->umem = ifobject->umem_arr[1];  in thread_common_ops()
793 	ifobject->umem = ifobject->umem_arr[0];  in thread_common_ops()
794 	ifobject->xsk = ifobject->xsk_arr[0];  in thread_common_ops()
796 		      ifobject->ifname, ifobject->fv.vector == tx ? "Tx" : "Rx");  in thread_common_ops()
807 		xsk_socket__delete(ifobj->xsk->xsk);  in testapp_cleanup_xsk_res()
808 		(void)xsk_umem__delete(ifobj->umem->umem);  in testapp_cleanup_xsk_res()
820 	print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,  in worker_testapp_validate_tx()
821 		      ifobject->ifname);  in worker_testapp_validate_tx()
841 		xsk_populate_fill_ring(ifobject->umem);  in worker_testapp_validate_rx()
843 	fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);  in worker_testapp_validate_rx()
852 		receive_pkts(ifobject->pkt_stream, ifobject->xsk, fds);  in worker_testapp_validate_rx()
874 	ifdict_tx->pkt_stream = pkt_stream;  in testapp_validate()
875 	ifdict_rx->pkt_stream = pkt_stream;  in testapp_validate()
878 	pthread_create(&t0, NULL, ifdict_rx->func_ptr, ifdict_rx);  in testapp_validate()
885 	pthread_create(&t1, NULL, ifdict_tx->func_ptr, ifdict_tx);  in testapp_validate()
908 	void *(*tmp_func_ptr)(void *) = ifobj1->func_ptr;  in swap_vectors()
909 	enum fvector tmp_vector = ifobj1->fv.vector;  in swap_vectors()
911 	ifobj1->func_ptr = ifobj2->func_ptr;  in swap_vectors()
912 	ifobj1->fv.vector = ifobj2->fv.vector;  in swap_vectors()
914 	ifobj2->func_ptr = tmp_func_ptr;  in swap_vectors()
915 	ifobj2->fv.vector = tmp_vector;  in swap_vectors()
940 	xsk_socket__delete(ifdict_tx->xsk->xsk);  in swap_xsk_res()
941 	xsk_umem__delete(ifdict_tx->umem->umem);  in swap_xsk_res()
942 	xsk_socket__delete(ifdict_rx->xsk->xsk);  in swap_xsk_res()
943 	xsk_umem__delete(ifdict_rx->umem->umem);  in swap_xsk_res()
944 	ifdict_tx->umem = ifdict_tx->umem_arr[1];  in swap_xsk_res()
945 	ifdict_tx->xsk = ifdict_tx->xsk_arr[1];  in swap_xsk_res()
946 	ifdict_rx->umem = ifdict_rx->umem_arr[1];  in swap_xsk_res()
947 	ifdict_rx->xsk = ifdict_rx->xsk_arr[1];  in swap_xsk_res()
976 			frame_headroom = XSK_UMEM__DEFAULT_FRAME_SIZE -  in testapp_stats()
977 						XDP_PACKET_HEADROOM - 1;  in testapp_stats()
1000 	memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);  in init_iface()
1001 	memcpy(ifobj->src_mac, src_mac, ETH_ALEN);  in init_iface()
1004 	ifobj->dst_ip = ip.s_addr;  in init_iface()
1007 	ifobj->src_ip = ip.s_addr;  in init_iface()
1009 	ifobj->dst_port = dst_port;  in init_iface()
1010 	ifobj->src_port = src_port;  in init_iface()
1013 		ifobj->fv.vector = tx;  in init_iface()
1014 		ifobj->func_ptr = worker_testapp_validate_tx;  in init_iface()
1017 		ifobj->fv.vector = rx;  in init_iface()
1018 		ifobj->func_ptr = worker_testapp_validate_rx;  in init_iface()
1030 	stat_test_type = -1;  in run_pkt_test()
1074 	ifobj->xsk_arr = calloc(2, sizeof(struct xsk_socket_info *));  in ifobject_create()
1075 	if (!ifobj->xsk_arr)  in ifobject_create()
1078 	ifobj->umem_arr = calloc(2, sizeof(struct xsk_umem_info *));  in ifobject_create()
1079 	if (!ifobj->umem_arr)  in ifobject_create()
1085 	free(ifobj->xsk_arr);  in ifobject_create()
1093 	free(ifobj->umem_arr);  in ifobject_delete()
1094 	free(ifobj->xsk_arr);  in ifobject_delete()