Lines Matching +full:data +full:- +full:out
1 // SPDX-License-Identifier: GPL-2.0-only
13 #include <linux/error-injection.h>
34 return -ENOMEM; in bpf_test_run()
53 ret = -EINTR; in bpf_test_run()
58 time_spent += ktime_get_ns() - time_start; in bpf_test_run()
69 time_spent += ktime_get_ns() - time_start; in bpf_test_run()
83 union bpf_attr __user *uattr, const void *data, in bpf_test_finish() argument
86 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); in bpf_test_finish()
87 int err = -EFAULT; in bpf_test_finish()
93 if (kattr->test.data_size_out && in bpf_test_finish()
94 copy_size > kattr->test.data_size_out) { in bpf_test_finish()
95 copy_size = kattr->test.data_size_out; in bpf_test_finish()
96 err = -ENOSPC; in bpf_test_finish()
99 if (data_out && copy_to_user(data_out, data, copy_size)) in bpf_test_finish()
100 goto out; in bpf_test_finish()
101 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) in bpf_test_finish()
102 goto out; in bpf_test_finish()
103 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_test_finish()
104 goto out; in bpf_test_finish()
105 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) in bpf_test_finish()
106 goto out; in bpf_test_finish()
107 if (err != -ENOSPC) in bpf_test_finish()
109 out: in bpf_test_finish()
119 __diag_ignore(GCC, 8, "-Wmissing-prototypes",
162 return (long)arg->a; in bpf_fentry_test8()
177 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); in bpf_test_init()
178 u32 user_size = kattr->test.data_size_in; in bpf_test_init()
179 void *data; in bpf_test_init() local
181 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) in bpf_test_init()
182 return ERR_PTR(-EINVAL); in bpf_test_init()
185 return ERR_PTR(-EMSGSIZE); in bpf_test_init()
187 data = kzalloc(size + headroom + tailroom, GFP_USER); in bpf_test_init()
188 if (!data) in bpf_test_init()
189 return ERR_PTR(-ENOMEM); in bpf_test_init()
191 if (copy_from_user(data + headroom, data_in, user_size)) { in bpf_test_init()
192 kfree(data); in bpf_test_init()
193 return ERR_PTR(-EFAULT); in bpf_test_init()
196 return data; in bpf_test_init()
205 int b = 2, err = -EFAULT; in bpf_prog_test_run_tracing()
208 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_tracing()
209 return -EINVAL; in bpf_prog_test_run_tracing()
211 switch (prog->expected_attach_type) { in bpf_prog_test_run_tracing()
222 goto out; in bpf_prog_test_run_tracing()
230 goto out; in bpf_prog_test_run_tracing()
234 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_prog_test_run_tracing()
235 goto out; in bpf_prog_test_run_tracing()
238 out: in bpf_prog_test_run_tracing()
250 __bpf_prog_test_run_raw_tp(void *data) in __bpf_prog_test_run_raw_tp() argument
252 struct bpf_raw_tp_test_run_info *info = data; in __bpf_prog_test_run_raw_tp()
255 info->retval = BPF_PROG_RUN(info->prog, info->ctx); in __bpf_prog_test_run_raw_tp()
263 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_prog_test_run_raw_tp()
264 __u32 ctx_size_in = kattr->test.ctx_size_in; in bpf_prog_test_run_raw_tp()
266 int cpu = kattr->test.cpu, err = 0; in bpf_prog_test_run_raw_tp()
269 /* doesn't support data_in/out, ctx_out, duration, or repeat */ in bpf_prog_test_run_raw_tp()
270 if (kattr->test.data_in || kattr->test.data_out || in bpf_prog_test_run_raw_tp()
271 kattr->test.ctx_out || kattr->test.duration || in bpf_prog_test_run_raw_tp()
272 kattr->test.repeat) in bpf_prog_test_run_raw_tp()
273 return -EINVAL; in bpf_prog_test_run_raw_tp()
275 if (ctx_size_in < prog->aux->max_ctx_offset) in bpf_prog_test_run_raw_tp()
276 return -EINVAL; in bpf_prog_test_run_raw_tp()
278 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) in bpf_prog_test_run_raw_tp()
279 return -EINVAL; in bpf_prog_test_run_raw_tp()
284 return -ENOMEM; in bpf_prog_test_run_raw_tp()
286 err = -EFAULT; in bpf_prog_test_run_raw_tp()
287 goto out; in bpf_prog_test_run_raw_tp()
296 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || in bpf_prog_test_run_raw_tp()
302 * space, let's do an extra quick check to filter out in bpf_prog_test_run_raw_tp()
305 err = -ENXIO; in bpf_prog_test_run_raw_tp()
313 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) in bpf_prog_test_run_raw_tp()
314 err = -EFAULT; in bpf_prog_test_run_raw_tp()
316 out: in bpf_prog_test_run_raw_tp()
323 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_ctx_init()
324 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_init()
325 u32 size = kattr->test.ctx_size_in; in bpf_ctx_init()
326 void *data; in bpf_ctx_init() local
332 data = kzalloc(max_size, GFP_USER); in bpf_ctx_init()
333 if (!data) in bpf_ctx_init()
334 return ERR_PTR(-ENOMEM); in bpf_ctx_init()
339 kfree(data); in bpf_ctx_init()
344 if (copy_from_user(data, data_in, size)) { in bpf_ctx_init()
345 kfree(data); in bpf_ctx_init()
346 return ERR_PTR(-EFAULT); in bpf_ctx_init()
349 return data; in bpf_ctx_init()
353 union bpf_attr __user *uattr, const void *data, in bpf_ctx_finish() argument
356 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_finish()
357 int err = -EFAULT; in bpf_ctx_finish()
360 if (!data || !data_out) in bpf_ctx_finish()
363 if (copy_size > kattr->test.ctx_size_out) { in bpf_ctx_finish()
364 copy_size = kattr->test.ctx_size_out; in bpf_ctx_finish()
365 err = -ENOSPC; in bpf_ctx_finish()
368 if (copy_to_user(data_out, data, copy_size)) in bpf_ctx_finish()
369 goto out; in bpf_ctx_finish()
370 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) in bpf_ctx_finish()
371 goto out; in bpf_ctx_finish()
372 if (err != -ENOSPC) in bpf_ctx_finish()
374 out: in bpf_ctx_finish()
379 * range_is_zero - test whether buffer is initialized
384 * This function returns true if the there is a non-zero byte
389 return !memchr_inv((u8 *)buf + from, 0, to - from); in range_is_zero()
394 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert___skb_to_skb()
401 return -EINVAL; in convert___skb_to_skb()
407 return -EINVAL; in convert___skb_to_skb()
413 return -EINVAL; in convert___skb_to_skb()
419 return -EINVAL; in convert___skb_to_skb()
425 return -EINVAL; in convert___skb_to_skb()
433 return -EINVAL; in convert___skb_to_skb()
439 return -EINVAL; in convert___skb_to_skb()
441 skb->mark = __skb->mark; in convert___skb_to_skb()
442 skb->priority = __skb->priority; in convert___skb_to_skb()
443 skb->tstamp = __skb->tstamp; in convert___skb_to_skb()
444 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); in convert___skb_to_skb()
446 if (__skb->wire_len == 0) { in convert___skb_to_skb()
447 cb->pkt_len = skb->len; in convert___skb_to_skb()
449 if (__skb->wire_len < skb->len || in convert___skb_to_skb()
450 __skb->wire_len > GSO_MAX_SIZE) in convert___skb_to_skb()
451 return -EINVAL; in convert___skb_to_skb()
452 cb->pkt_len = __skb->wire_len; in convert___skb_to_skb()
455 if (__skb->gso_segs > GSO_MAX_SEGS) in convert___skb_to_skb()
456 return -EINVAL; in convert___skb_to_skb()
457 skb_shinfo(skb)->gso_segs = __skb->gso_segs; in convert___skb_to_skb()
458 skb_shinfo(skb)->gso_size = __skb->gso_size; in convert___skb_to_skb()
465 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert_skb_to___skb()
470 __skb->mark = skb->mark; in convert_skb_to___skb()
471 __skb->priority = skb->priority; in convert_skb_to___skb()
472 __skb->ifindex = skb->dev->ifindex; in convert_skb_to___skb()
473 __skb->tstamp = skb->tstamp; in convert_skb_to___skb()
474 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); in convert_skb_to___skb()
475 __skb->wire_len = cb->pkt_len; in convert_skb_to___skb()
476 __skb->gso_segs = skb_shinfo(skb)->gso_segs; in convert_skb_to___skb()
483 struct net *net = current->nsproxy->net_ns; in bpf_prog_test_run_skb()
484 struct net_device *dev = net->loopback_dev; in bpf_prog_test_run_skb()
485 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_skb()
486 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_skb()
492 void *data; in bpf_prog_test_run_skb() local
495 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_skb()
496 return -EINVAL; in bpf_prog_test_run_skb()
498 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, in bpf_prog_test_run_skb()
500 if (IS_ERR(data)) in bpf_prog_test_run_skb()
501 return PTR_ERR(data); in bpf_prog_test_run_skb()
505 kfree(data); in bpf_prog_test_run_skb()
509 switch (prog->type) { in bpf_prog_test_run_skb()
525 kfree(data); in bpf_prog_test_run_skb()
527 return -ENOMEM; in bpf_prog_test_run_skb()
532 skb = build_skb(data, 0); in bpf_prog_test_run_skb()
534 kfree(data); in bpf_prog_test_run_skb()
537 return -ENOMEM; in bpf_prog_test_run_skb()
539 skb->sk = sk; in bpf_prog_test_run_skb()
543 if (ctx && ctx->ifindex > 1) { in bpf_prog_test_run_skb()
544 dev = dev_get_by_index(net, ctx->ifindex); in bpf_prog_test_run_skb()
546 ret = -ENODEV; in bpf_prog_test_run_skb()
547 goto out; in bpf_prog_test_run_skb()
550 skb->protocol = eth_type_trans(skb, dev); in bpf_prog_test_run_skb()
553 switch (skb->protocol) { in bpf_prog_test_run_skb()
555 sk->sk_family = AF_INET; in bpf_prog_test_run_skb()
557 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; in bpf_prog_test_run_skb()
558 sk->sk_daddr = ip_hdr(skb)->daddr; in bpf_prog_test_run_skb()
563 sk->sk_family = AF_INET6; in bpf_prog_test_run_skb()
565 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; in bpf_prog_test_run_skb()
566 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; in bpf_prog_test_run_skb()
580 goto out; in bpf_prog_test_run_skb()
583 goto out; in bpf_prog_test_run_skb()
586 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); in bpf_prog_test_run_skb()
589 ret = -ENOMEM; in bpf_prog_test_run_skb()
590 goto out; in bpf_prog_test_run_skb()
597 size = skb->len; in bpf_prog_test_run_skb()
598 /* bpf program can never convert linear skb to non-linear */ in bpf_prog_test_run_skb()
601 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); in bpf_prog_test_run_skb()
605 out: in bpf_prog_test_run_skb()
606 if (dev && dev != net->loopback_dev) in bpf_prog_test_run_skb()
620 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_xdp()
621 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_xdp()
626 void *data; in bpf_prog_test_run_xdp() local
629 if (kattr->test.ctx_in || kattr->test.ctx_out) in bpf_prog_test_run_xdp()
630 return -EINVAL; in bpf_prog_test_run_xdp()
633 max_data_sz = 4096 - headroom - tailroom; in bpf_prog_test_run_xdp()
635 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); in bpf_prog_test_run_xdp()
636 if (IS_ERR(data)) in bpf_prog_test_run_xdp()
637 return PTR_ERR(data); in bpf_prog_test_run_xdp()
639 xdp.data_hard_start = data; in bpf_prog_test_run_xdp()
640 xdp.data = data + headroom; in bpf_prog_test_run_xdp()
641 xdp.data_meta = xdp.data; in bpf_prog_test_run_xdp()
642 xdp.data_end = xdp.data + size; in bpf_prog_test_run_xdp()
645 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); in bpf_prog_test_run_xdp()
646 xdp.rxq = &rxqueue->xdp_rxq; in bpf_prog_test_run_xdp()
650 goto out; in bpf_prog_test_run_xdp()
651 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size) in bpf_prog_test_run_xdp()
652 size = xdp.data_end - xdp.data; in bpf_prog_test_run_xdp()
653 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); in bpf_prog_test_run_xdp()
654 out: in bpf_prog_test_run_xdp()
656 kfree(data); in bpf_prog_test_run_xdp()
664 return -EINVAL; in verify_user_bpf_flow_keys()
670 return -EINVAL; in verify_user_bpf_flow_keys()
679 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_flow_dissector()
681 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_flow_dissector()
688 void *data; in bpf_prog_test_run_flow_dissector() local
692 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) in bpf_prog_test_run_flow_dissector()
693 return -EINVAL; in bpf_prog_test_run_flow_dissector()
695 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_flow_dissector()
696 return -EINVAL; in bpf_prog_test_run_flow_dissector()
699 return -EINVAL; in bpf_prog_test_run_flow_dissector()
701 data = bpf_test_init(kattr, size, 0, 0); in bpf_prog_test_run_flow_dissector()
702 if (IS_ERR(data)) in bpf_prog_test_run_flow_dissector()
703 return PTR_ERR(data); in bpf_prog_test_run_flow_dissector()
705 eth = (struct ethhdr *)data; in bpf_prog_test_run_flow_dissector()
712 kfree(data); in bpf_prog_test_run_flow_dissector()
718 goto out; in bpf_prog_test_run_flow_dissector()
719 flags = user_ctx->flags; in bpf_prog_test_run_flow_dissector()
723 ctx.data = data; in bpf_prog_test_run_flow_dissector()
724 ctx.data_end = (__u8 *)data + size; in bpf_prog_test_run_flow_dissector()
730 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, in bpf_prog_test_run_flow_dissector()
737 ret = -EINTR; in bpf_prog_test_run_flow_dissector()
738 goto out; in bpf_prog_test_run_flow_dissector()
742 time_spent += ktime_get_ns() - time_start; in bpf_prog_test_run_flow_dissector()
753 time_spent += ktime_get_ns() - time_start; in bpf_prog_test_run_flow_dissector()
766 out: in bpf_prog_test_run_flow_dissector()
768 kfree(data); in bpf_prog_test_run_flow_dissector()