277 static inline void *xq_get_data(struct xdpsock *xsk, u64 addr)  in xq_get_data()  argument
279 	return &xsk->umem->frames[addr];  in xq_get_data()
481 	struct xdpsock *xsk;  in xsk_configure()  local
489 	xsk = calloc(1, sizeof(*xsk));  in xsk_configure()
490 	lassert(xsk);  in xsk_configure()
492 	xsk->sfd = sfd;  in xsk_configure()
493 	xsk->outstanding_tx = 0;  in xsk_configure()
497 		xsk->umem = xdp_umem_configure(sfd);  in xsk_configure()
499 		xsk->umem = umem;  in xsk_configure()
511 	xsk->rx.map = mmap(NULL,  in xsk_configure()
517 	lassert(xsk->rx.map != MAP_FAILED);  in xsk_configure()
521 			lassert(umem_fill_to_kernel(&xsk->umem->fq, &i, 1)  in xsk_configure()
526 	xsk->tx.map = mmap(NULL,  in xsk_configure()
532 	lassert(xsk->tx.map != MAP_FAILED);  in xsk_configure()
534 	xsk->rx.mask = NUM_DESCS - 1;  in xsk_configure()
535 	xsk->rx.size = NUM_DESCS;  in xsk_configure()
536 	xsk->rx.producer = xsk->rx.map + off.rx.producer;  in xsk_configure()
537 	xsk->rx.consumer = xsk->rx.map + off.rx.consumer;  in xsk_configure()
538 	xsk->rx.ring = xsk->rx.map + off.rx.desc;  in xsk_configure()
540 	xsk->tx.mask = NUM_DESCS - 1;  in xsk_configure()
541 	xsk->tx.size = NUM_DESCS;  in xsk_configure()
542 	xsk->tx.producer = xsk->tx.map + off.tx.producer;  in xsk_configure()
543 	xsk->tx.consumer = xsk->tx.map + off.tx.consumer;  in xsk_configure()
544 	xsk->tx.ring = xsk->tx.map + off.tx.desc;  in xsk_configure()
545 	xsk->tx.cached_cons = NUM_DESCS;  in xsk_configure()
560 	return xsk;  in xsk_configure()
742 static inline void complete_tx_l2fwd(struct xdpsock *xsk)  in complete_tx_l2fwd()  argument
748 	if (!xsk->outstanding_tx)  in complete_tx_l2fwd()
751 	kick_tx(xsk->sfd);  in complete_tx_l2fwd()
752 	ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE :  in complete_tx_l2fwd()
753 		 xsk->outstanding_tx;  in complete_tx_l2fwd()
756 	rcvd = umem_complete_from_kernel(&xsk->umem->cq, descs, ndescs);  in complete_tx_l2fwd()
758 		umem_fill_to_kernel(&xsk->umem->fq, descs, rcvd);  in complete_tx_l2fwd()
759 		xsk->outstanding_tx -= rcvd;  in complete_tx_l2fwd()
760 		xsk->tx_npkts += rcvd;  in complete_tx_l2fwd()
764 static inline void complete_tx_only(struct xdpsock *xsk)  in complete_tx_only()  argument
769 	if (!xsk->outstanding_tx)  in complete_tx_only()
772 	kick_tx(xsk->sfd);  in complete_tx_only()
774 	rcvd = umem_complete_from_kernel(&xsk->umem->cq, descs, BATCH_SIZE);  in complete_tx_only()
776 		xsk->outstanding_tx -= rcvd;  in complete_tx_only()
777 		xsk->tx_npkts += rcvd;  in complete_tx_only()
781 static void rx_drop(struct xdpsock *xsk)  in rx_drop()  argument
786 	rcvd = xq_deq(&xsk->rx, descs, BATCH_SIZE);  in rx_drop()
791 		char *pkt = xq_get_data(xsk, descs[i].addr);  in rx_drop()
796 	xsk->rx_npkts += rcvd;  in rx_drop()
798 	umem_fill_to_kernel_ex(&xsk->umem->fq, descs, rcvd);  in rx_drop()
826 static void tx_only(struct xdpsock *xsk)  in tx_only()  argument
833 	fds[0].fd = xsk->sfd;  in tx_only()
843 			if (fds[0].fd != xsk->sfd ||  in tx_only()
848 		if (xq_nb_free(&xsk->tx, BATCH_SIZE) >= BATCH_SIZE) {  in tx_only()
849 			lassert(xq_enq_tx_only(&xsk->tx, idx, BATCH_SIZE) == 0);  in tx_only()
851 			xsk->outstanding_tx += BATCH_SIZE;  in tx_only()
856 		complete_tx_only(xsk);  in tx_only()
860 static void l2fwd(struct xdpsock *xsk)  in l2fwd()  argument
868 			complete_tx_l2fwd(xsk);  in l2fwd()
870 			rcvd = xq_deq(&xsk->rx, descs, BATCH_SIZE);  in l2fwd()
876 			char *pkt = xq_get_data(xsk, descs[i].addr);  in l2fwd()
883 		xsk->rx_npkts += rcvd;  in l2fwd()
885 		ret = xq_enq(&xsk->tx, descs, rcvd);  in l2fwd()
887 		xsk->outstanding_tx += rcvd;  in l2fwd()