124 	struct xdp_test_data *xdp = arg;  in xdp_test_run_init_page()  local
129 	orig_ctx = xdp->orig_ctx;  in xdp_test_run_init_page()
139 	xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);  in xdp_test_run_init_page()
149 static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)  in xdp_test_run_setup()  argument
156 		.pool_size = xdp->batch_size,  in xdp_test_run_setup()
159 		.init_arg = xdp,  in xdp_test_run_setup()
162 	xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);  in xdp_test_run_setup()
163 	if (!xdp->frames)  in xdp_test_run_setup()
166 	xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);  in xdp_test_run_setup()
167 	if (!xdp->skbs)  in xdp_test_run_setup()
177 	err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp);  in xdp_test_run_setup()
181 	xdp->pp = pp;  in xdp_test_run_setup()
186 	xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0);  in xdp_test_run_setup()
187 	xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;  in xdp_test_run_setup()
188 	xdp->rxq.mem.id = pp->xdp_mem_id;  in xdp_test_run_setup()
189 	xdp->dev = orig_ctx->rxq->dev;  in xdp_test_run_setup()
190 	xdp->orig_ctx = orig_ctx;  in xdp_test_run_setup()
197 	kvfree(xdp->skbs);  in xdp_test_run_setup()
199 	kvfree(xdp->frames);  in xdp_test_run_setup()
203 static void xdp_test_run_teardown(struct xdp_test_data *xdp)  in xdp_test_run_teardown()  argument
205 	xdp_unreg_mem_model(&xdp->mem);  in xdp_test_run_teardown()
206 	page_pool_destroy(xdp->pp);  in xdp_test_run_teardown()
207 	kfree(xdp->frames);  in xdp_test_run_teardown()
208 	kfree(xdp->skbs);  in xdp_test_run_teardown()
261 static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,  in xdp_test_run_batch()  argument
266 	struct xdp_frame **frames = xdp->frames;  in xdp_test_run_batch()
273 	batch_sz = min_t(u32, repeat, xdp->batch_size);  in xdp_test_run_batch()
279 		page = page_pool_dev_alloc_pages(xdp->pp);  in xdp_test_run_batch()
289 		xdp->frame_cnt++;  in xdp_test_run_batch()
308 			ri->tgt_index = xdp->dev->ifindex;  in xdp_test_run_batch()
314 			ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog);  in xdp_test_run_batch()
334 		ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev);  in xdp_test_run_batch()
348 	struct xdp_test_data xdp = { .batch_size = batch_size };  in bpf_test_run_xdp_live()  local
355 	ret = xdp_test_run_setup(&xdp, ctx);  in bpf_test_run_xdp_live()
361 		xdp.frame_cnt = 0;  in bpf_test_run_xdp_live()
362 		ret = xdp_test_run_batch(&xdp, prog, repeat - t.i);  in bpf_test_run_xdp_live()
365 	} while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time));  in bpf_test_run_xdp_live()
368 	xdp_test_run_teardown(&xdp);  in bpf_test_run_xdp_live()
373 			u32 *retval, u32 *time, bool xdp)  in bpf_test_run()  argument
399 		if (xdp)  in bpf_test_run()
1217 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)  in xdp_convert_md_to_buff()  argument
1249 		xdp->rxq = &rxqueue->xdp_rxq;  in xdp_convert_md_to_buff()
1255 	xdp->data = xdp->data_meta + xdp_md->data;  in xdp_convert_md_to_buff()
1263 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)  in xdp_convert_buff_to_md()  argument
1268 	xdp_md->data = xdp->data - xdp->data_meta;  in xdp_convert_buff_to_md()
1269 	xdp_md->data_end = xdp->data_end - xdp->data_meta;  in xdp_convert_buff_to_md()
1272 		dev_put(xdp->rxq->dev);  in xdp_convert_buff_to_md()
1287 	struct xdp_buff xdp = {};  in bpf_prog_test_run_xdp()  local
1341 	xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq);  in bpf_prog_test_run_xdp()
1342 	xdp_prepare_buff(&xdp, data, headroom, size, true);  in bpf_prog_test_run_xdp()
1343 	sinfo = xdp_get_shared_info_from_buff(&xdp);  in bpf_prog_test_run_xdp()
1345 	ret = xdp_convert_md_to_buff(ctx, &xdp);  in bpf_prog_test_run_xdp()
1383 		xdp_buff_set_frags_flag(&xdp);  in bpf_prog_test_run_xdp()
1390 		ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration);  in bpf_prog_test_run_xdp()
1392 		ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);  in bpf_prog_test_run_xdp()
1397 	xdp_convert_buff_to_md(&xdp, ctx);  in bpf_prog_test_run_xdp()
1401 	size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;  in bpf_prog_test_run_xdp()
1402 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size,  in bpf_prog_test_run_xdp()