158 static struct sk_buff *cpu_map_build_skb(struct xdp_frame *xdpf,  in cpu_map_build_skb()  argument
166 	hard_start_headroom = sizeof(struct xdp_frame) +  xdpf->headroom;  in cpu_map_build_skb()
171 	frame_size = xdpf->frame_sz;  in cpu_map_build_skb()
173 	pkt_data_start = xdpf->data - hard_start_headroom;  in cpu_map_build_skb()
179 	__skb_put(skb, xdpf->len);  in cpu_map_build_skb()
180 	if (xdpf->metasize)  in cpu_map_build_skb()
181 		skb_metadata_set(skb, xdpf->metasize);  in cpu_map_build_skb()
184 	skb->protocol = eth_type_trans(skb, xdpf->dev_rx);  in cpu_map_build_skb()
193 	xdp_release_frame(xdpf);  in cpu_map_build_skb()
196 	xdp_scrub_frame(xdpf);  in cpu_map_build_skb()
208 	struct xdp_frame *xdpf;  in __cpu_map_ring_cleanup()  local
210 	while ((xdpf = ptr_ring_consume(ring)))  in __cpu_map_ring_cleanup()
211 		if (WARN_ON_ONCE(xdpf))  in __cpu_map_ring_cleanup()
212 			xdp_return_frame(xdpf);  in __cpu_map_ring_cleanup()
245 		struct xdp_frame *xdpf = frames[i];  in cpu_map_bpf_prog_run_xdp()  local
249 		rxq.dev = xdpf->dev_rx;  in cpu_map_bpf_prog_run_xdp()
250 		rxq.mem = xdpf->mem;  in cpu_map_bpf_prog_run_xdp()
253 		xdp_convert_frame_to_buff(xdpf, &xdp);  in cpu_map_bpf_prog_run_xdp()
258 			err = xdp_update_frame_from_buff(&xdp, xdpf);  in cpu_map_bpf_prog_run_xdp()
260 				xdp_return_frame(xdpf);  in cpu_map_bpf_prog_run_xdp()
263 				frames[nframes++] = xdpf;  in cpu_map_bpf_prog_run_xdp()
268 			err = xdp_do_redirect(xdpf->dev_rx, &xdp,  in cpu_map_bpf_prog_run_xdp()
271 				xdp_return_frame(xdpf);  in cpu_map_bpf_prog_run_xdp()
281 			xdp_return_frame(xdpf);  in cpu_map_bpf_prog_run_xdp()
363 			struct xdp_frame *xdpf = frames[i];  in cpu_map_kthread_run()  local
367 			skb = cpu_map_build_skb(xdpf, skb);  in cpu_map_kthread_run()
369 				xdp_return_frame(xdpf);  in cpu_map_kthread_run()
685 		struct xdp_frame *xdpf = bq->q[i];  in bq_flush_to_queue()  local
688 		err = __ptr_ring_produce(q, xdpf);  in bq_flush_to_queue()
691 			xdp_return_frame_rx_napi(xdpf);  in bq_flush_to_queue()
707 static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)  in bq_enqueue()  argument
724 	bq->q[bq->count++] = xdpf;  in bq_enqueue()
733 	struct xdp_frame *xdpf;  in cpu_map_enqueue()  local
735 	xdpf = xdp_convert_buff_to_frame(xdp);  in cpu_map_enqueue()
736 	if (unlikely(!xdpf))  in cpu_map_enqueue()
740 	xdpf->dev_rx = dev_rx;  in cpu_map_enqueue()
742 	bq_enqueue(rcpu, xdpf);  in cpu_map_enqueue()