134 	struct xdp_test_data *xdp = arg;  in xdp_test_run_init_page()  local
139 	orig_ctx = xdp->orig_ctx;  in xdp_test_run_init_page()
149 	xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);  in xdp_test_run_init_page()
159 static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)  in xdp_test_run_setup()  argument
166 		.pool_size = xdp->batch_size,  in xdp_test_run_setup()
169 		.init_arg = xdp,  in xdp_test_run_setup()
172 	xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);  in xdp_test_run_setup()
173 	if (!xdp->frames)  in xdp_test_run_setup()
176 	xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);  in xdp_test_run_setup()
177 	if (!xdp->skbs)  in xdp_test_run_setup()
187 	err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp);  in xdp_test_run_setup()
191 	xdp->pp = pp;  in xdp_test_run_setup()
196 	xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0);  in xdp_test_run_setup()
197 	xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;  in xdp_test_run_setup()
198 	xdp->rxq.mem.id = pp->xdp_mem_id;  in xdp_test_run_setup()
199 	xdp->dev = orig_ctx->rxq->dev;  in xdp_test_run_setup()
200 	xdp->orig_ctx = orig_ctx;  in xdp_test_run_setup()
207 	kvfree(xdp->skbs);  in xdp_test_run_setup()
209 	kvfree(xdp->frames);  in xdp_test_run_setup()
213 static void xdp_test_run_teardown(struct xdp_test_data *xdp)  in xdp_test_run_teardown()  argument
215 	xdp_unreg_mem_model(&xdp->mem);  in xdp_test_run_teardown()
216 	page_pool_destroy(xdp->pp);  in xdp_test_run_teardown()
217 	kfree(xdp->frames);  in xdp_test_run_teardown()
218 	kfree(xdp->skbs);  in xdp_test_run_teardown()
281 static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,  in xdp_test_run_batch()  argument
286 	struct xdp_frame **frames = xdp->frames;  in xdp_test_run_batch()
293 	batch_sz = min_t(u32, repeat, xdp->batch_size);  in xdp_test_run_batch()
299 		page = page_pool_dev_alloc_pages(xdp->pp);  in xdp_test_run_batch()
309 		xdp->frame_cnt++;  in xdp_test_run_batch()
328 			ri->tgt_index = xdp->dev->ifindex;  in xdp_test_run_batch()
334 			ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog);  in xdp_test_run_batch()
354 		ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev);  in xdp_test_run_batch()
368 	struct xdp_test_data xdp = { .batch_size = batch_size };  in bpf_test_run_xdp_live()  local
375 	ret = xdp_test_run_setup(&xdp, ctx);  in bpf_test_run_xdp_live()
381 		xdp.frame_cnt = 0;  in bpf_test_run_xdp_live()
382 		ret = xdp_test_run_batch(&xdp, prog, repeat - t.i);  in bpf_test_run_xdp_live()
385 	} while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time));  in bpf_test_run_xdp_live()
388 	xdp_test_run_teardown(&xdp);  in bpf_test_run_xdp_live()
393 			u32 *retval, u32 *time, bool xdp)  in bpf_test_run()  argument
420 		if (xdp)  in bpf_test_run()
1080 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)  in xdp_convert_md_to_buff()  argument
1112 		xdp->rxq = &rxqueue->xdp_rxq;  in xdp_convert_md_to_buff()
1118 	xdp->data = xdp->data_meta + xdp_md->data;  in xdp_convert_md_to_buff()
1126 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)  in xdp_convert_buff_to_md()  argument
1131 	xdp_md->data = xdp->data - xdp->data_meta;  in xdp_convert_buff_to_md()
1132 	xdp_md->data_end = xdp->data_end - xdp->data_meta;  in xdp_convert_buff_to_md()
1135 		dev_put(xdp->rxq->dev);  in xdp_convert_buff_to_md()
1150 	struct xdp_buff xdp = {};  in bpf_prog_test_run_xdp()  local
1207 	xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq);  in bpf_prog_test_run_xdp()
1208 	xdp_prepare_buff(&xdp, data, headroom, size, true);  in bpf_prog_test_run_xdp()
1209 	sinfo = xdp_get_shared_info_from_buff(&xdp);  in bpf_prog_test_run_xdp()
1211 	ret = xdp_convert_md_to_buff(ctx, &xdp);  in bpf_prog_test_run_xdp()
1248 		xdp_buff_set_frags_flag(&xdp);  in bpf_prog_test_run_xdp()
1255 		ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration);  in bpf_prog_test_run_xdp()
1257 		ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);  in bpf_prog_test_run_xdp()
1262 	xdp_convert_buff_to_md(&xdp, ctx);  in bpf_prog_test_run_xdp()
1266 	size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;  in bpf_prog_test_run_xdp()
1267 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size,  in bpf_prog_test_run_xdp()