331 		struct xdp_frame *xdpf = frames[i];  in dev_map_bpf_prog_run()  local
335 		xdp_convert_frame_to_buff(xdpf, &xdp);  in dev_map_bpf_prog_run()
341 			err = xdp_update_frame_from_buff(&xdp, xdpf);  in dev_map_bpf_prog_run()
343 				xdp_return_frame_rx_napi(xdpf);  in dev_map_bpf_prog_run()
345 				frames[nframes++] = xdpf;  in dev_map_bpf_prog_run()
354 			xdp_return_frame_rx_napi(xdpf);  in dev_map_bpf_prog_run()
373 		struct xdp_frame *xdpf = bq->q[i];  in bq_xmit_all()  local
375 		prefetch(xdpf);  in bq_xmit_all()
442 static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,  in bq_enqueue()  argument
464 	bq->q[bq->count++] = xdpf;  in bq_enqueue()
467 static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,  in __xdp_enqueue()  argument
477 		     xdp_frame_has_frags(xdpf)))  in __xdp_enqueue()
480 	err = xdp_ok_fwd_dev(dev, xdp_get_frame_len(xdpf));  in __xdp_enqueue()
484 	bq_enqueue(dev, xdpf, dev_rx, xdp_prog);  in __xdp_enqueue()
519 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,  in dev_xdp_enqueue()  argument
522 	return __xdp_enqueue(dev, xdpf, dev_rx, NULL);  in dev_xdp_enqueue()
525 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,  in dev_map_enqueue()  argument
530 	return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog);  in dev_map_enqueue()
533 static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf)  in is_valid_dst()  argument
542 		     xdp_frame_has_frags(xdpf)))  in is_valid_dst()
545 	if (xdp_ok_fwd_dev(obj->dev, xdp_get_frame_len(xdpf)))  in is_valid_dst()
553 				 struct xdp_frame *xdpf)  in dev_map_enqueue_clone()  argument
557 	nxdpf = xdpf_clone(xdpf);  in dev_map_enqueue_clone()
591 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,  in dev_map_enqueue_multi()  argument
611 			if (!is_valid_dst(dst, xdpf))  in dev_map_enqueue_multi()
623 			err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);  in dev_map_enqueue_multi()
634 				if (!is_valid_dst(dst, xdpf))  in dev_map_enqueue_multi()
647 				err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);  in dev_map_enqueue_multi()
658 		bq_enqueue(last_dst->dev, xdpf, dev_rx, last_dst->xdp_prog);  in dev_map_enqueue_multi()
660 		xdp_return_frame_rx_napi(xdpf); /* dtab is empty */  in dev_map_enqueue_multi()