435 static int bpf_test_finish(const union bpf_attr *kattr,  in bpf_test_finish()  argument
440 	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);  in bpf_test_finish()
447 	if (kattr->test.data_size_out &&  in bpf_test_finish()
448 	    copy_size > kattr->test.data_size_out) {  in bpf_test_finish()
449 		copy_size = kattr->test.data_size_out;  in bpf_test_finish()
626 static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,  in BTF_ID_FLAGS()
629 	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);  in BTF_ID_FLAGS()
652 			      const union bpf_attr *kattr,  in bpf_prog_test_run_tracing()  argument
660 	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)  in bpf_prog_test_run_tracing()
717 			     const union bpf_attr *kattr,  in bpf_prog_test_run_raw_tp()  argument
720 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);  in bpf_prog_test_run_raw_tp()
721 	__u32 ctx_size_in = kattr->test.ctx_size_in;  in bpf_prog_test_run_raw_tp()
723 	int cpu = kattr->test.cpu, err = 0;  in bpf_prog_test_run_raw_tp()
727 	if (kattr->test.data_in || kattr->test.data_out ||  in bpf_prog_test_run_raw_tp()
728 	    kattr->test.ctx_out || kattr->test.duration ||  in bpf_prog_test_run_raw_tp()
729 	    kattr->test.repeat || kattr->test.batch_size)  in bpf_prog_test_run_raw_tp()
736 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)  in bpf_prog_test_run_raw_tp()
750 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||  in bpf_prog_test_run_raw_tp()
774 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)  in bpf_ctx_init()  argument
776 	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);  in bpf_ctx_init()
777 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);  in bpf_ctx_init()
778 	u32 size = kattr->test.ctx_size_in;  in bpf_ctx_init()
805 static int bpf_ctx_finish(const union bpf_attr *kattr,  in bpf_ctx_finish()  argument
809 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);  in bpf_ctx_finish()
816 	if (copy_size > kattr->test.ctx_size_out) {  in bpf_ctx_finish()
817 		copy_size = kattr->test.ctx_size_out;  in bpf_ctx_finish()
944 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,  in bpf_prog_test_run_skb()  argument
950 	u32 size = kattr->test.data_size_in;  in bpf_prog_test_run_skb()
951 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_skb()
960 	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)  in bpf_prog_test_run_skb()
963 	data = bpf_test_init(kattr, kattr->test.data_size_in,  in bpf_prog_test_run_skb()
969 	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));  in bpf_prog_test_run_skb()
1066 	ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval,  in bpf_prog_test_run_skb()
1069 		ret = bpf_ctx_finish(kattr, uattr, ctx,  in bpf_prog_test_run_skb()
1138 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,  in bpf_prog_test_run_xdp()  argument
1141 	bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES);  in bpf_prog_test_run_xdp()
1143 	u32 batch_size = kattr->test.batch_size;  in bpf_prog_test_run_xdp()
1145 	u32 size = kattr->test.data_size_in;  in bpf_prog_test_run_xdp()
1147 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_xdp()
1159 	if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES)  in bpf_prog_test_run_xdp()
1176 	ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));  in bpf_prog_test_run_xdp()
1185 		    (do_live && (kattr->test.data_out || kattr->test.ctx_out)))  in bpf_prog_test_run_xdp()
1199 	data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom);  in bpf_prog_test_run_xdp()
1215 	if (unlikely(kattr->test.data_size_in > size)) {  in bpf_prog_test_run_xdp()
1216 		void __user *data_in = u64_to_user_ptr(kattr->test.data_in);  in bpf_prog_test_run_xdp()
1218 		while (size < kattr->test.data_size_in) {  in bpf_prog_test_run_xdp()
1236 			data_len = min_t(u32, kattr->test.data_size_in - size,  in bpf_prog_test_run_xdp()
1267 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size,  in bpf_prog_test_run_xdp()
1270 		ret = bpf_ctx_finish(kattr, uattr, ctx,  in bpf_prog_test_run_xdp()
1301 				     const union bpf_attr *kattr,  in bpf_prog_test_run_flow_dissector()  argument
1305 	u32 size = kattr->test.data_size_in;  in bpf_prog_test_run_flow_dissector()
1307 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_flow_dissector()
1316 	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)  in bpf_prog_test_run_flow_dissector()
1322 	data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0);  in bpf_prog_test_run_flow_dissector()
1331 	user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));  in bpf_prog_test_run_flow_dissector()
1357 	ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL,  in bpf_prog_test_run_flow_dissector()
1360 		ret = bpf_ctx_finish(kattr, uattr, user_ctx,  in bpf_prog_test_run_flow_dissector()
1369 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,  in bpf_prog_test_run_sk_lookup()  argument
1375 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_sk_lookup()
1380 	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)  in bpf_prog_test_run_sk_lookup()
1383 	if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||  in bpf_prog_test_run_sk_lookup()
1384 	    kattr->test.data_size_out)  in bpf_prog_test_run_sk_lookup()
1390 	user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));  in bpf_prog_test_run_sk_lookup()
1459 	ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration);  in bpf_prog_test_run_sk_lookup()
1461 		ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));  in bpf_prog_test_run_sk_lookup()
1470 			      const union bpf_attr *kattr,  in bpf_prog_test_run_syscall()  argument
1473 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);  in bpf_prog_test_run_syscall()
1474 	__u32 ctx_size_in = kattr->test.ctx_size_in;  in bpf_prog_test_run_syscall()
1480 	if (kattr->test.data_in || kattr->test.data_out ||  in bpf_prog_test_run_syscall()
1481 	    kattr->test.ctx_out || kattr->test.duration ||  in bpf_prog_test_run_syscall()
1482 	    kattr->test.repeat || kattr->test.flags ||  in bpf_prog_test_run_syscall()
1483 	    kattr->test.batch_size)  in bpf_prog_test_run_syscall()
1568 			 const union bpf_attr *kattr,  in bpf_prog_test_run_nf()  argument
1577 	u32 size = kattr->test.data_size_in;  in bpf_prog_test_run_nf()
1578 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_nf()
1587 	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)  in bpf_prog_test_run_nf()
1593 	data = bpf_test_init(kattr, kattr->test.data_size_in, size,  in bpf_prog_test_run_nf()
1602 	user_ctx = bpf_ctx_init(kattr, sizeof(struct nf_hook_state));  in bpf_prog_test_run_nf()
1659 	ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration);  in bpf_prog_test_run_nf()