130 static int bpf_test_finish(const union bpf_attr *kattr,  in bpf_test_finish()  argument
134 	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);  in bpf_test_finish()
141 	if (kattr->test.data_size_out &&  in bpf_test_finish()
142 	    copy_size > kattr->test.data_size_out) {  in bpf_test_finish()
143 		copy_size = kattr->test.data_size_out;  in bpf_test_finish()
249 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,  in bpf_test_init()  argument
252 	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);  in bpf_test_init()
253 	u32 user_size = kattr->test.data_size_in;  in bpf_test_init()
275 			      const union bpf_attr *kattr,  in bpf_prog_test_run_tracing()  argument
283 	if (kattr->test.flags || kattr->test.cpu)  in bpf_prog_test_run_tracing()
335 			     const union bpf_attr *kattr,  in bpf_prog_test_run_raw_tp()  argument
338 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);  in bpf_prog_test_run_raw_tp()
339 	__u32 ctx_size_in = kattr->test.ctx_size_in;  in bpf_prog_test_run_raw_tp()
341 	int cpu = kattr->test.cpu, err = 0;  in bpf_prog_test_run_raw_tp()
345 	if (kattr->test.data_in || kattr->test.data_out ||  in bpf_prog_test_run_raw_tp()
346 	    kattr->test.ctx_out || kattr->test.duration ||  in bpf_prog_test_run_raw_tp()
347 	    kattr->test.repeat)  in bpf_prog_test_run_raw_tp()
354 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)  in bpf_prog_test_run_raw_tp()
372 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||  in bpf_prog_test_run_raw_tp()
397 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)  in bpf_ctx_init()  argument
399 	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);  in bpf_ctx_init()
400 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);  in bpf_ctx_init()
401 	u32 size = kattr->test.ctx_size_in;  in bpf_ctx_init()
428 static int bpf_ctx_finish(const union bpf_attr *kattr,  in bpf_ctx_finish()  argument
432 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);  in bpf_ctx_finish()
439 	if (copy_size > kattr->test.ctx_size_out) {  in bpf_ctx_finish()
440 		copy_size = kattr->test.ctx_size_out;  in bpf_ctx_finish()
561 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,  in bpf_prog_test_run_skb()  argument
567 	u32 size = kattr->test.data_size_in;  in bpf_prog_test_run_skb()
568 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_skb()
577 	if (kattr->test.flags || kattr->test.cpu)  in bpf_prog_test_run_skb()
580 	data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,  in bpf_prog_test_run_skb()
585 	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));  in bpf_prog_test_run_skb()
682 	ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);  in bpf_prog_test_run_skb()
684 		ret = bpf_ctx_finish(kattr, uattr, ctx,  in bpf_prog_test_run_skb()
753 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,  in bpf_prog_test_run_xdp()  argument
758 	u32 size = kattr->test.data_size_in;  in bpf_prog_test_run_xdp()
759 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_xdp()
772 	ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));  in bpf_prog_test_run_xdp()
789 	data = bpf_test_init(kattr, max_data_sz, headroom, tailroom);  in bpf_prog_test_run_xdp()
818 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, size, retval,  in bpf_prog_test_run_xdp()
821 		ret = bpf_ctx_finish(kattr, uattr, ctx,  in bpf_prog_test_run_xdp()
849 				     const union bpf_attr *kattr,  in bpf_prog_test_run_flow_dissector()  argument
853 	u32 size = kattr->test.data_size_in;  in bpf_prog_test_run_flow_dissector()
855 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_flow_dissector()
867 	if (kattr->test.flags || kattr->test.cpu)  in bpf_prog_test_run_flow_dissector()
873 	data = bpf_test_init(kattr, size, 0, 0);  in bpf_prog_test_run_flow_dissector()
882 	user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));  in bpf_prog_test_run_flow_dissector()
908 	ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),  in bpf_prog_test_run_flow_dissector()
911 		ret = bpf_ctx_finish(kattr, uattr, user_ctx,  in bpf_prog_test_run_flow_dissector()
920 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,  in bpf_prog_test_run_sk_lookup()  argument
926 	u32 repeat = kattr->test.repeat;  in bpf_prog_test_run_sk_lookup()
934 	if (kattr->test.flags || kattr->test.cpu)  in bpf_prog_test_run_sk_lookup()
937 	if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||  in bpf_prog_test_run_sk_lookup()
938 	    kattr->test.data_size_out)  in bpf_prog_test_run_sk_lookup()
944 	user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));  in bpf_prog_test_run_sk_lookup()
1013 	ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration);  in bpf_prog_test_run_sk_lookup()
1015 		ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));  in bpf_prog_test_run_sk_lookup()
1024 			      const union bpf_attr *kattr,  in bpf_prog_test_run_syscall()  argument
1027 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);  in bpf_prog_test_run_syscall()
1028 	__u32 ctx_size_in = kattr->test.ctx_size_in;  in bpf_prog_test_run_syscall()
1034 	if (kattr->test.data_in || kattr->test.data_out ||  in bpf_prog_test_run_syscall()
1035 	    kattr->test.ctx_out || kattr->test.duration ||  in bpf_prog_test_run_syscall()
1036 	    kattr->test.repeat || kattr->test.flags)  in bpf_prog_test_run_syscall()