Lines Matching +full:unlock +full:- +full:keys
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com
17 #include <linux/error-injection.h>
59 for (i = 0; i < btm->module->num_bpf_raw_events; ++i) { in bpf_get_raw_tracepoint_module()
60 btp = &btm->module->bpf_raw_events[i]; in bpf_get_raw_tracepoint_module()
61 if (!strcmp(btp->tp->name, name)) { in bpf_get_raw_tracepoint_module()
62 if (try_module_get(btm->module)) in bpf_get_raw_tracepoint_module()
89 * trace_call_bpf - invoke BPF program
98 * 0 - return from kprobe (event is filtered out)
99 * 1 - store kprobe event into ring buffer
112 * and don't send kprobe event into ring-buffer, in trace_call_bpf()
122 * whether call->prog_array is empty or not, which is in trace_call_bpf()
126 * non-NULL, we go into trace_call_bpf() and do the actual in trace_call_bpf()
135 ret = bpf_prog_run_array(rcu_dereference(call->prog_array), in trace_call_bpf()
198 * strncpy_from_user() does long-sized strides in the fast path. If the in bpf_probe_read_user_str_common()
201 * and keys a hash map with it, then semantically identical strings can in bpf_probe_read_user_str_common()
333 * access_ok() should prevent writing to non-user memory, but in in BPF_CALL_3()
343 current->flags & (PF_KTHREAD | PF_EXITING))) in BPF_CALL_3()
344 return -EPERM; in BPF_CALL_3()
346 return -EPERM; in BPF_CALL_3()
366 current->comm, task_pid_nr(current)); in bpf_get_probe_write_proto()
439 return -EINVAL; in BPF_CALL_4()
481 return -EINVAL; in BPF_CALL_5()
492 return seq_has_overflowed(m) ? -EOVERFLOW : 0; in BPF_CALL_5()
511 return seq_write(m, data, len) ? -EOVERFLOW : 0; in BPF_CALL_3()
535 return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags); in BPF_CALL_4()
559 return -EINVAL; in get_map_perf_counter()
562 if (unlikely(index >= array->map.max_entries)) in get_map_perf_counter()
563 return -E2BIG; in get_map_perf_counter()
565 ee = READ_ONCE(array->ptrs[index]); in get_map_perf_counter()
567 return -ENOENT; in get_map_perf_counter()
569 return perf_event_read_local(ee->event, value, enabled, running); in get_map_perf_counter()
579 * this api is ugly since we miss [-22..-2] range of valid in BPF_CALL_2()
598 int err = -EINVAL; in BPF_CALL_4()
602 err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled, in BPF_CALL_4()
603 &buf->running); in BPF_CALL_4()
634 if (unlikely(index >= array->map.max_entries)) in __bpf_perf_event_output()
635 return -E2BIG; in __bpf_perf_event_output()
637 ee = READ_ONCE(array->ptrs[index]); in __bpf_perf_event_output()
639 return -ENOENT; in __bpf_perf_event_output()
641 event = ee->event; in __bpf_perf_event_output()
642 if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || in __bpf_perf_event_output()
643 event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) in __bpf_perf_event_output()
644 return -EINVAL; in __bpf_perf_event_output()
646 if (unlikely(event->oncpu != cpu)) in __bpf_perf_event_output()
647 return -EOPNOTSUPP; in __bpf_perf_event_output()
676 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) { in BPF_CALL_5()
677 err = -EBUSY; in BPF_CALL_5()
681 sd = &sds->sds[nest_level - 1]; in BPF_CALL_5()
684 err = -EINVAL; in BPF_CALL_5()
689 sd->raw = &raw; in BPF_CALL_5()
690 sd->sample_flags |= PERF_SAMPLE_RAW; in BPF_CALL_5()
740 ret = -EBUSY; in bpf_event_output()
743 sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]); in bpf_event_output()
744 regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]); in bpf_event_output()
748 sd->raw = &raw; in bpf_event_output()
749 sd->sample_flags |= PERF_SAMPLE_RAW; in bpf_event_output()
802 if (unlikely(idx >= array->map.max_entries)) in BPF_CALL_2()
803 return -E2BIG; in BPF_CALL_2()
805 cgrp = READ_ONCE(array->ptrs[idx]); in BPF_CALL_2()
807 return -EAGAIN; in BPF_CALL_2()
834 group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type); in do_bpf_send_signal()
846 if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING))) in bpf_send_signal_common()
847 return -EPERM; in bpf_send_signal_common()
849 return -EPERM; in bpf_send_signal_common()
856 return -EINVAL; in bpf_send_signal_common()
859 if (irq_work_is_busy(&work->irq_work)) in bpf_send_signal_common()
860 return -EBUSY; in bpf_send_signal_common()
866 work->task = current; in bpf_send_signal_common()
867 work->sig = sig; in bpf_send_signal_common()
868 work->type = type; in bpf_send_signal_common()
869 irq_work_queue(&work->irq_work); in bpf_send_signal_common()
912 len = buf + sz - p; in BPF_CALL_3()
937 if (prog->type == BPF_PROG_TYPE_TRACING && in BTF_ID()
938 prog->expected_attach_type == BPF_TRACE_ITER) in BTF_ID()
941 if (prog->type == BPF_PROG_TYPE_LSM) in BTF_ID()
942 return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id); in BTF_ID()
945 prog->aux->attach_btf_id); in BTF_ID()
971 return -EINVAL; in bpf_btf_printf_prepare()
974 return -EINVAL; in bpf_btf_printf_prepare()
979 return IS_ERR(*btf) ? PTR_ERR(*btf) : -EINVAL; in bpf_btf_printf_prepare()
981 if (ptr->type_id > 0) in bpf_btf_printf_prepare()
982 *btf_id = ptr->type_id; in bpf_btf_printf_prepare()
984 return -EINVAL; in bpf_btf_printf_prepare()
989 return -ENOENT; in bpf_btf_printf_prepare()
1005 return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size, in BPF_CALL_5()
1023 return ((u64 *)ctx)[-2]; in BPF_CALL_1()
1038 /* Being extra safe in here in case entry ip is on the page-edge. */ in get_entry_ip()
1039 if (get_kernel_nofault(instr, (u32 *) fentry_ip - 1)) in get_entry_ip()
1042 fentry_ip -= ENDBR_INSN_SIZE; in get_entry_ip()
1053 if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY)) in BPF_CALL_1()
1056 return get_entry_ip((uintptr_t)kp->addr); in BPF_CALL_1()
1068 return bpf_kprobe_multi_entry_ip(current->bpf_ctx); in BPF_CALL_1()
1080 return bpf_kprobe_multi_cookie(current->bpf_ctx); in BPF_CALL_1()
1094 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); in BPF_CALL_1()
1095 return run_ctx->bpf_cookie; in BPF_CALL_1()
1107 return ctx->event->bpf_cookie; in BPF_CALL_1()
1121 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); in BPF_CALL_1()
1122 return run_ctx->bpf_cookie; in BPF_CALL_1()
1135 return -ENOENT; in BPF_CALL_3()
1143 return -EINVAL; in BPF_CALL_3()
1146 return -ENOENT; in BPF_CALL_3()
1163 u64 nr_args = ((u64 *)ctx)[-1]; in BPF_CALL_3()
1166 return -EINVAL; in BPF_CALL_3()
1182 u64 nr_args = ((u64 *)ctx)[-1]; in BPF_CALL_2()
1198 return ((u64 *)ctx)[-1]; in BPF_CALL_1()
1209 __diag_ignore_all("-Wmissing-prototypes",
1213 * bpf_lookup_user_key - lookup a key by its serial
1215 * @flags: lookup-specific flags
1226 * one of the available key-specific kfuncs.
1231 * for the key construction, and to retrieve uninstantiated keys (keys
1259 bkey->key = key_ref_to_ptr(key_ref); in bpf_lookup_user_key()
1260 bkey->has_ref = true; in bpf_lookup_user_key()
1266 * bpf_lookup_system_key - lookup a key by a system-defined ID
1276 * 0 for the primary keyring (immutable keyring of system keys);
1278 * (where keys can be added only if they are vouched for by existing keys
1284 * pre-determined ID on success, a NULL pointer otherwise
1297 bkey->key = (struct key *)(unsigned long)id; in bpf_lookup_system_key()
1298 bkey->has_ref = false; in bpf_lookup_system_key()
1304 * bpf_key_put - decrement key reference count if key is valid and free bpf_key
1312 if (bkey->has_ref) in bpf_key_put()
1313 key_put(bkey->key); in bpf_key_put()
1320 * bpf_verify_pkcs7_signature - verify a PKCS#7 signature
1323 * @trusted_keyring: keyring with keys trusted for signature verification
1326 * with keys in a keyring referenced by *trusted_keyring*.
1336 if (trusted_keyring->has_ref) { in bpf_verify_pkcs7_signature()
1345 ret = key_validate(trusted_keyring->key); in bpf_verify_pkcs7_signature()
1350 return verify_pkcs7_signature(data_ptr->data, in bpf_verify_pkcs7_signature()
1352 sig_ptr->data, in bpf_verify_pkcs7_signature()
1354 trusted_keyring->key, in bpf_verify_pkcs7_signature()
1483 return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL; in bpf_tracing_func_proto()
1485 return prog->aux->sleepable ? &bpf_copy_from_user_task_proto : NULL; in bpf_tracing_func_proto()
1528 return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ? in kprobe_prog_func_proto()
1532 return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ? in kprobe_prog_func_proto()
1678 int err = -EINVAL; in BPF_CALL_3()
1682 err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled, in BPF_CALL_3()
1683 &buf->running); in BPF_CALL_3()
1705 struct perf_branch_stack *br_stack = ctx->data->br_stack; in BPF_CALL_4()
1709 return -EINVAL; in BPF_CALL_4()
1711 if (unlikely(!(ctx->data->sample_flags & PERF_SAMPLE_BRANCH_STACK))) in BPF_CALL_4()
1712 return -ENOENT; in BPF_CALL_4()
1715 return -ENOENT; in BPF_CALL_4()
1718 return br_stack->nr * br_entry_size; in BPF_CALL_4()
1721 return -EINVAL; in BPF_CALL_4()
1723 to_copy = min_t(u32, br_stack->nr * br_entry_size, size); in BPF_CALL_4()
1724 memcpy(buf, br_stack->entries, to_copy); in BPF_CALL_4()
1778 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) { in get_bpf_raw_tp_regs()
1780 return ERR_PTR(-EBUSY); in get_bpf_raw_tp_regs()
1783 return &tp_regs->regs[nest_level - 1]; in get_bpf_raw_tp_regs()
1926 return prog->expected_attach_type == BPF_TRACE_ITER ? in tracing_prog_func_proto()
1930 return prog->expected_attach_type == BPF_TRACE_ITER ? in tracing_prog_func_proto()
1934 return prog->expected_attach_type == BPF_TRACE_ITER ? in tracing_prog_func_proto()
1949 if (!fn && prog->expected_attach_type == BPF_TRACE_ITER) in tracing_prog_func_proto()
1975 return -ENOTSUPP; in bpf_prog_test_run_tracing()
2006 info->reg_type = PTR_TO_TP_BUFFER; in raw_tp_writable_prog_is_valid_access()
2064 switch (si->off) { in pe_prog_convert_ctx_access()
2067 data), si->dst_reg, si->src_reg, in pe_prog_convert_ctx_access()
2069 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, in pe_prog_convert_ctx_access()
2075 data), si->dst_reg, si->src_reg, in pe_prog_convert_ctx_access()
2077 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, in pe_prog_convert_ctx_access()
2083 regs), si->dst_reg, si->src_reg, in pe_prog_convert_ctx_access()
2085 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg, in pe_prog_convert_ctx_access()
2086 si->off); in pe_prog_convert_ctx_access()
2090 return insn - insn_buf; in pe_prog_convert_ctx_access()
2112 int ret = -EEXIST; in perf_event_attach_bpf_prog()
2116 * and only if they are on the opt-in list. in perf_event_attach_bpf_prog()
2118 if (prog->kprobe_override && in perf_event_attach_bpf_prog()
2119 (!trace_kprobe_on_func_entry(event->tp_event) || in perf_event_attach_bpf_prog()
2120 !trace_kprobe_error_injectable(event->tp_event))) in perf_event_attach_bpf_prog()
2121 return -EINVAL; in perf_event_attach_bpf_prog()
2125 if (event->prog) in perf_event_attach_bpf_prog()
2126 goto unlock; in perf_event_attach_bpf_prog()
2128 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); in perf_event_attach_bpf_prog()
2131 ret = -E2BIG; in perf_event_attach_bpf_prog()
2132 goto unlock; in perf_event_attach_bpf_prog()
2137 goto unlock; in perf_event_attach_bpf_prog()
2139 /* set the new array to event->tp_event and set event->prog */ in perf_event_attach_bpf_prog()
2140 event->prog = prog; in perf_event_attach_bpf_prog()
2141 event->bpf_cookie = bpf_cookie; in perf_event_attach_bpf_prog()
2142 rcu_assign_pointer(event->tp_event->prog_array, new_array); in perf_event_attach_bpf_prog()
2145 unlock: in perf_event_attach_bpf_prog()
2158 if (!event->prog) in perf_event_detach_bpf_prog()
2159 goto unlock; in perf_event_detach_bpf_prog()
2161 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); in perf_event_detach_bpf_prog()
2162 ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array); in perf_event_detach_bpf_prog()
2163 if (ret == -ENOENT) in perf_event_detach_bpf_prog()
2164 goto unlock; in perf_event_detach_bpf_prog()
2166 bpf_prog_array_delete_safe(old_array, event->prog); in perf_event_detach_bpf_prog()
2168 rcu_assign_pointer(event->tp_event->prog_array, new_array); in perf_event_detach_bpf_prog()
2172 bpf_prog_put(event->prog); in perf_event_detach_bpf_prog()
2173 event->prog = NULL; in perf_event_detach_bpf_prog()
2175 unlock: in perf_event_detach_bpf_prog()
2188 return -EPERM; in perf_event_query_prog_array()
2189 if (event->attr.type != PERF_TYPE_TRACEPOINT) in perf_event_query_prog_array()
2190 return -EINVAL; in perf_event_query_prog_array()
2192 return -EFAULT; in perf_event_query_prog_array()
2196 return -E2BIG; in perf_event_query_prog_array()
2199 return -ENOMEM; in perf_event_query_prog_array()
2202 * is required when user only wants to check for uquery->prog_cnt. in perf_event_query_prog_array()
2208 progs = bpf_event_rcu_dereference(event->tp_event->prog_array); in perf_event_query_prog_array()
2212 if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) || in perf_event_query_prog_array()
2213 copy_to_user(uquery->ids, ids, ids_len * sizeof(u32))) in perf_event_query_prog_array()
2214 ret = -EFAULT; in perf_event_query_prog_array()
2228 if (!strcmp(btp->tp->name, name)) in bpf_get_raw_tracepoint()
2249 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) { in __bpf_trace_run()
2257 this_cpu_dec(*(prog->active)); in __bpf_trace_run()
2307 struct tracepoint *tp = btp->tp; in __bpf_probe_register()
2313 if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64)) in __bpf_probe_register()
2314 return -EINVAL; in __bpf_probe_register()
2316 if (prog->aux->max_tp_access > btp->writable_size) in __bpf_probe_register()
2317 return -EINVAL; in __bpf_probe_register()
2319 return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func, in __bpf_probe_register()
2330 return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog); in bpf_probe_unregister()
2341 prog = event->prog; in bpf_get_perf_event_info()
2343 return -ENOENT; in bpf_get_perf_event_info()
2346 if (prog->type == BPF_PROG_TYPE_PERF_EVENT) in bpf_get_perf_event_info()
2347 return -EOPNOTSUPP; in bpf_get_perf_event_info()
2349 *prog_id = prog->aux->id; in bpf_get_perf_event_info()
2350 flags = event->tp_event->flags; in bpf_get_perf_event_info()
2352 is_syscall_tp = is_syscall_trace_event(event->tp_event); in bpf_get_perf_event_info()
2355 *buf = is_tracepoint ? event->tp_event->tp->name in bpf_get_perf_event_info()
2356 : event->tp_event->name; in bpf_get_perf_event_info()
2362 err = -EOPNOTSUPP; in bpf_get_perf_event_info()
2367 event->attr.type == PERF_TYPE_TRACEPOINT); in bpf_get_perf_event_info()
2373 event->attr.type == PERF_TYPE_TRACEPOINT); in bpf_get_perf_event_info()
2387 init_irq_work(&work->irq_work, do_bpf_send_signal); in send_signal_irq_work_init()
2402 if (mod->num_bpf_raw_events == 0 || in bpf_event_notify()
2412 btm->module = module; in bpf_event_notify()
2413 list_add(&btm->list, &bpf_trace_modules); in bpf_event_notify()
2415 ret = -ENOMEM; in bpf_event_notify()
2420 if (btm->module == module) { in bpf_event_notify()
2421 list_del(&btm->list); in bpf_event_notify()
2473 int err = -ENOMEM; in copy_user_syms()
2486 err = -EFAULT; in copy_user_syms()
2491 err = -E2BIG; in copy_user_syms()
2498 us->syms = syms; in copy_user_syms()
2499 us->buf = buf; in copy_user_syms()
2512 kvfree(us->syms); in free_user_syms()
2513 kvfree(us->buf); in free_user_syms()
2521 unregister_fprobe(&kmulti_link->fp); in bpf_kprobe_multi_link_release()
2529 kvfree(kmulti_link->addrs); in bpf_kprobe_multi_link_dealloc()
2530 kvfree(kmulti_link->cookies); in bpf_kprobe_multi_link_dealloc()
2545 cookie_a = link->cookies + (addr_a - link->addrs); in bpf_kprobe_multi_cookie_swap()
2546 cookie_b = link->cookies + (addr_b - link->addrs); in bpf_kprobe_multi_cookie_swap()
2559 return *addr_a < *addr_b ? -1 : 1; in __bpf_kprobe_multi_cookie_cmp()
2576 run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, run_ctx); in bpf_kprobe_multi_cookie()
2577 link = run_ctx->link; in bpf_kprobe_multi_cookie()
2578 if (!link->cookies) in bpf_kprobe_multi_cookie()
2580 entry_ip = run_ctx->entry_ip; in bpf_kprobe_multi_cookie()
2581 addr = bsearch(&entry_ip, link->addrs, link->cnt, sizeof(entry_ip), in bpf_kprobe_multi_cookie()
2585 cookie = link->cookies + (addr - link->addrs); in bpf_kprobe_multi_cookie()
2593 run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, run_ctx); in bpf_kprobe_multi_entry_ip()
2594 return run_ctx->entry_ip; in bpf_kprobe_multi_entry_ip()
2616 err = bpf_prog_run(link->link.prog, regs); in kprobe_multi_link_prog_run()
2657 if (data->cookies) { in symbols_swap_r()
2660 cookie_a = data->cookies + (name_a - data->funcs); in symbols_swap_r()
2661 cookie_b = data->cookies + (name_b - data->funcs); in symbols_swap_r()
2680 return -EOPNOTSUPP; in bpf_kprobe_multi_link_attach()
2682 if (prog->expected_attach_type != BPF_TRACE_KPROBE_MULTI) in bpf_kprobe_multi_link_attach()
2683 return -EINVAL; in bpf_kprobe_multi_link_attach()
2685 flags = attr->link_create.kprobe_multi.flags; in bpf_kprobe_multi_link_attach()
2687 return -EINVAL; in bpf_kprobe_multi_link_attach()
2689 uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs); in bpf_kprobe_multi_link_attach()
2690 usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms); in bpf_kprobe_multi_link_attach()
2692 return -EINVAL; in bpf_kprobe_multi_link_attach()
2694 cnt = attr->link_create.kprobe_multi.cnt; in bpf_kprobe_multi_link_attach()
2696 return -EINVAL; in bpf_kprobe_multi_link_attach()
2701 return -ENOMEM; in bpf_kprobe_multi_link_attach()
2703 ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies); in bpf_kprobe_multi_link_attach()
2707 err = -ENOMEM; in bpf_kprobe_multi_link_attach()
2711 err = -EFAULT; in bpf_kprobe_multi_link_attach()
2718 err = -EFAULT; in bpf_kprobe_multi_link_attach()
2745 err = -ENOMEM; in bpf_kprobe_multi_link_attach()
2749 bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI, in bpf_kprobe_multi_link_attach()
2752 err = bpf_link_prime(&link->link, &link_primer); in bpf_kprobe_multi_link_attach()
2757 link->fp.exit_handler = kprobe_multi_link_handler; in bpf_kprobe_multi_link_attach()
2759 link->fp.entry_handler = kprobe_multi_link_handler; in bpf_kprobe_multi_link_attach()
2761 link->addrs = addrs; in bpf_kprobe_multi_link_attach()
2762 link->cookies = cookies; in bpf_kprobe_multi_link_attach()
2763 link->cnt = cnt; in bpf_kprobe_multi_link_attach()
2778 err = register_fprobe_ips(&link->fp, addrs, cnt); in bpf_kprobe_multi_link_attach()
2795 return -EOPNOTSUPP; in bpf_kprobe_multi_link_attach()