172 			    struct veth_rq *rq, bool xdp)  in veth_forward_skb()  argument
174 	return __dev_forward_skb(dev, skb) ?: xdp ?  in veth_forward_skb()
365 static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)  in veth_xdp_tx()  argument
367 	struct xdp_frame *frame = convert_to_xdp_frame(xdp);  in veth_xdp_tx()
390 		struct xdp_buff xdp;  in veth_xdp_rcv_one()  local
393 		xdp.data_hard_start = hard_start;  in veth_xdp_rcv_one()
394 		xdp.data = frame->data;  in veth_xdp_rcv_one()
395 		xdp.data_end = frame->data + frame->len;  in veth_xdp_rcv_one()
396 		xdp.data_meta = frame->data - frame->metasize;  in veth_xdp_rcv_one()
397 		xdp.rxq = &rq->xdp_rxq;  in veth_xdp_rcv_one()
399 		act = bpf_prog_run_xdp(xdp_prog, &xdp);  in veth_xdp_rcv_one()
403 			delta = frame->data - xdp.data;  in veth_xdp_rcv_one()
404 			len = xdp.data_end - xdp.data;  in veth_xdp_rcv_one()
408 			xdp.data_hard_start = head;  in veth_xdp_rcv_one()
409 			xdp.rxq->mem = frame->mem;  in veth_xdp_rcv_one()
410 			if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {  in veth_xdp_rcv_one()
420 			xdp.data_hard_start = head;  in veth_xdp_rcv_one()
421 			xdp.rxq->mem = frame->mem;  in veth_xdp_rcv_one()
422 			if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {  in veth_xdp_rcv_one()
464 	struct xdp_buff xdp;  in veth_xdp_rcv_skb()  local
517 	xdp.data_hard_start = skb->head;  in veth_xdp_rcv_skb()
518 	xdp.data = skb_mac_header(skb);  in veth_xdp_rcv_skb()
519 	xdp.data_end = xdp.data + pktlen;  in veth_xdp_rcv_skb()
520 	xdp.data_meta = xdp.data;  in veth_xdp_rcv_skb()
521 	xdp.rxq = &rq->xdp_rxq;  in veth_xdp_rcv_skb()
522 	orig_data = xdp.data;  in veth_xdp_rcv_skb()
523 	orig_data_end = xdp.data_end;  in veth_xdp_rcv_skb()
525 	act = bpf_prog_run_xdp(xdp_prog, &xdp);  in veth_xdp_rcv_skb()
531 		get_page(virt_to_page(xdp.data));  in veth_xdp_rcv_skb()
533 		xdp.rxq->mem = rq->xdp_mem;  in veth_xdp_rcv_skb()
534 		if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {  in veth_xdp_rcv_skb()
542 		get_page(virt_to_page(xdp.data));  in veth_xdp_rcv_skb()
544 		xdp.rxq->mem = rq->xdp_mem;  in veth_xdp_rcv_skb()
545 		if (xdp_do_redirect(rq->dev, &xdp, xdp_prog))  in veth_xdp_rcv_skb()
559 	delta = orig_data - xdp.data;  in veth_xdp_rcv_skb()
566 	off = xdp.data_end - orig_data_end;  in veth_xdp_rcv_skb()
571 	metalen = xdp.data - xdp.data_meta;  in veth_xdp_rcv_skb()
582 	page_frag_free(xdp.data);  in veth_xdp_rcv_skb()
990 static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp)  in veth_xdp()  argument
992 	switch (xdp->command) {  in veth_xdp()
994 		return veth_xdp_set(dev, xdp->prog, xdp->extack);  in veth_xdp()
996 		xdp->prog_id = veth_xdp_query(dev);  in veth_xdp()