Lines Matching refs:env
194 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
195 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
196 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
197 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
198 static int ref_set_non_owning(struct bpf_verifier_env *env,
200 static void specialize_kfunc(struct bpf_verifier_env *env,
341 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) in find_linfo() argument
347 prog = env->prog; in find_linfo()
363 struct bpf_verifier_env *env = private_data; in verbose() local
366 if (!bpf_verifier_log_needed(&env->log)) in verbose()
370 bpf_verifier_vlog(&env->log, fmt, args); in verbose()
382 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, in verbose_linfo() argument
388 if (!bpf_verifier_log_needed(&env->log)) in verbose_linfo()
391 linfo = find_linfo(env, insn_off); in verbose_linfo()
392 if (!linfo || linfo == env->prev_linfo) in verbose_linfo()
399 bpf_verifier_vlog(&env->log, prefix_fmt, args); in verbose_linfo()
403 verbose(env, "%s\n", in verbose_linfo()
404 ltrim(btf_name_by_offset(env->prog->aux->btf, in verbose_linfo()
407 env->prev_linfo = linfo; in verbose_linfo()
410 static void verbose_invalid_scalar(struct bpf_verifier_env *env, in verbose_invalid_scalar() argument
417 verbose(env, "At %s the register %s ", ctx, reg_name); in verbose_invalid_scalar()
420 verbose(env, "has value %s", tn_buf); in verbose_invalid_scalar()
422 verbose(env, "has unknown scalar value"); in verbose_invalid_scalar()
425 verbose(env, " should have been in %s\n", tn_buf); in verbose_invalid_scalar()
491 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) in subprog_is_global() argument
493 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; in subprog_is_global()
596 static const char *reg_type_str(struct bpf_verifier_env *env, in reg_type_str() argument
641 snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s", in reg_type_str()
643 return env->tmp_str_buf; in reg_type_str()
655 static void print_liveness(struct bpf_verifier_env *env, in print_liveness() argument
659 verbose(env, "_"); in print_liveness()
661 verbose(env, "r"); in print_liveness()
663 verbose(env, "w"); in print_liveness()
665 verbose(env, "D"); in print_liveness()
673 static struct bpf_func_state *func(struct bpf_verifier_env *env, in func() argument
676 struct bpf_verifier_state *cur = env->cur_state; in func()
695 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in stack_slot_obj_get_spi() argument
701 verbose(env, "%s has to be at a constant offset\n", obj_kind); in stack_slot_obj_get_spi()
707 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); in stack_slot_obj_get_spi()
713 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); in stack_slot_obj_get_spi()
717 if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots)) in stack_slot_obj_get_spi()
722 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_get_spi() argument
724 return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS); in dynptr_get_spi()
727 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) in iter_get_spi() argument
729 return stack_slot_obj_get_spi(env, reg, "iter", nr_slots); in iter_get_spi()
780 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) in mark_reg_scratched() argument
782 env->scratched_regs |= 1U << regno; in mark_reg_scratched()
785 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) in mark_stack_slot_scratched() argument
787 env->scratched_stack_slots |= 1ULL << spi; in mark_stack_slot_scratched()
790 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) in reg_scratched() argument
792 return (env->scratched_regs >> regno) & 1; in reg_scratched()
795 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) in stack_slot_scratched() argument
797 return (env->scratched_stack_slots >> regno) & 1; in stack_slot_scratched()
800 static bool verifier_state_scratched(const struct bpf_verifier_env *env) in verifier_state_scratched() argument
802 return env->scratched_regs || env->scratched_stack_slots; in verifier_state_scratched()
805 static void mark_verifier_state_clean(struct bpf_verifier_env *env) in mark_verifier_state_clean() argument
807 env->scratched_regs = 0U; in mark_verifier_state_clean()
808 env->scratched_stack_slots = 0ULL; in mark_verifier_state_clean()
812 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) in mark_verifier_state_scratched() argument
814 env->scratched_regs = ~0U; in mark_verifier_state_scratched()
815 env->scratched_stack_slots = ~0ULL; in mark_verifier_state_scratched()
859 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
862 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, in mark_dynptr_stack_regs() argument
867 int id = ++env->id_gen; in mark_dynptr_stack_regs()
873 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, in mark_dynptr_cb_reg() argument
877 __mark_dynptr_reg(reg, type, true, ++env->id_gen); in mark_dynptr_cb_reg()
880 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
883 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_stack_slots_dynptr() argument
886 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_dynptr()
890 spi = dynptr_get_spi(env, reg); in mark_stack_slots_dynptr()
903 err = destroy_if_dynptr_stack_slot(env, state, spi); in mark_stack_slots_dynptr()
906 err = destroy_if_dynptr_stack_slot(env, state, spi - 1); in mark_stack_slots_dynptr()
919 mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, in mark_stack_slots_dynptr()
929 id = acquire_reference_state(env, insn_idx); in mark_stack_slots_dynptr()
944 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) in invalidate_dynptr() argument
953 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); in invalidate_dynptr()
954 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); in invalidate_dynptr()
981 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in unmark_stack_slots_dynptr() argument
983 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_dynptr()
986 spi = dynptr_get_spi(env, reg); in unmark_stack_slots_dynptr()
991 invalidate_dynptr(env, state, spi); in unmark_stack_slots_dynptr()
1005 WARN_ON_ONCE(release_reference(env, ref_obj_id)); in unmark_stack_slots_dynptr()
1017 verbose(env, "verifier internal error: misconfigured ref_obj_id\n"); in unmark_stack_slots_dynptr()
1021 invalidate_dynptr(env, state, i); in unmark_stack_slots_dynptr()
1027 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
1030 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_reg_invalid() argument
1032 if (!env->allow_ptr_leaks) in mark_reg_invalid()
1033 __mark_reg_not_init(env, reg); in mark_reg_invalid()
1035 __mark_reg_unknown(env, reg); in mark_reg_invalid()
1038 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, in destroy_if_dynptr_stack_slot() argument
1058 verbose(env, "cannot overwrite referenced dynptr\n"); in destroy_if_dynptr_stack_slot()
1062 mark_stack_slot_scratched(env, spi); in destroy_if_dynptr_stack_slot()
1063 mark_stack_slot_scratched(env, spi - 1); in destroy_if_dynptr_stack_slot()
1073 bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({ in destroy_if_dynptr_stack_slot()
1078 mark_reg_invalid(env, dreg); in destroy_if_dynptr_stack_slot()
1084 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); in destroy_if_dynptr_stack_slot()
1085 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); in destroy_if_dynptr_stack_slot()
1094 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_uninit() argument
1101 spi = dynptr_get_spi(env, reg); in is_dynptr_reg_valid_uninit()
1122 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_init() argument
1124 struct bpf_func_state *state = func(env, reg); in is_dynptr_reg_valid_init()
1136 spi = dynptr_get_spi(env, reg); in is_dynptr_reg_valid_init()
1151 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in is_dynptr_type_expected() argument
1154 struct bpf_func_state *state = func(env, reg); in is_dynptr_type_expected()
1166 spi = dynptr_get_spi(env, reg); in is_dynptr_type_expected()
1175 static int mark_stack_slots_iter(struct bpf_verifier_env *env, in mark_stack_slots_iter() argument
1179 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_iter()
1182 spi = iter_get_spi(env, reg, nr_slots); in mark_stack_slots_iter()
1186 id = acquire_reference_state(env, insn_idx); in mark_stack_slots_iter()
1206 mark_stack_slot_scratched(env, spi - i); in mark_stack_slots_iter()
1212 static int unmark_stack_slots_iter(struct bpf_verifier_env *env, in unmark_stack_slots_iter() argument
1215 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_iter()
1218 spi = iter_get_spi(env, reg, nr_slots); in unmark_stack_slots_iter()
1227 WARN_ON_ONCE(release_reference(env, st->ref_obj_id)); in unmark_stack_slots_iter()
1229 __mark_reg_not_init(env, st); in unmark_stack_slots_iter()
1237 mark_stack_slot_scratched(env, spi - i); in unmark_stack_slots_iter()
1243 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, in is_iter_reg_valid_uninit() argument
1246 struct bpf_func_state *state = func(env, reg); in is_iter_reg_valid_uninit()
1253 spi = iter_get_spi(env, reg, nr_slots); in is_iter_reg_valid_uninit()
1270 static bool is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in is_iter_reg_valid_init() argument
1273 struct bpf_func_state *state = func(env, reg); in is_iter_reg_valid_init()
1276 spi = iter_get_spi(env, reg, nr_slots); in is_iter_reg_valid_init()
1344 static void print_verifier_state(struct bpf_verifier_env *env, in print_verifier_state() argument
1353 verbose(env, " frame%d:", state->frameno); in print_verifier_state()
1359 if (!print_all && !reg_scratched(env, i)) in print_verifier_state()
1361 verbose(env, " R%d", i); in print_verifier_state()
1362 print_liveness(env, reg->live); in print_verifier_state()
1363 verbose(env, "="); in print_verifier_state()
1365 verbose(env, "P"); in print_verifier_state()
1369 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); in print_verifier_state()
1370 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
1374 verbose(env, "%s", reg_type_str(env, t)); in print_verifier_state()
1376 verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id)); in print_verifier_state()
1377 verbose(env, "("); in print_verifier_state()
1382 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) in print_verifier_state()
1438 verbose(env, ")"); in print_verifier_state()
1454 if (!print_all && !stack_slot_scratched(env, i)) in print_verifier_state()
1461 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
1462 print_liveness(env, reg->live); in print_verifier_state()
1463 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); in print_verifier_state()
1465 verbose(env, "P"); in print_verifier_state()
1467 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
1473 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
1474 print_liveness(env, reg->live); in print_verifier_state()
1475 verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type)); in print_verifier_state()
1477 verbose(env, "(ref_id=%d)", reg->ref_obj_id); in print_verifier_state()
1485 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
1486 print_liveness(env, reg->live); in print_verifier_state()
1487 verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)", in print_verifier_state()
1501 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
1502 print_liveness(env, reg->live); in print_verifier_state()
1503 verbose(env, "=%s", types_buf); in print_verifier_state()
1508 verbose(env, " refs=%d", state->refs[0].id); in print_verifier_state()
1511 verbose(env, ",%d", state->refs[i].id); in print_verifier_state()
1514 verbose(env, " cb"); in print_verifier_state()
1516 verbose(env, " async_cb"); in print_verifier_state()
1517 verbose(env, "\n"); in print_verifier_state()
1518 mark_verifier_state_clean(env); in print_verifier_state()
1527 static void print_insn_state(struct bpf_verifier_env *env, in print_insn_state() argument
1530 if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { in print_insn_state()
1532 bpf_vlog_reset(&env->log, env->prev_log_pos - 1); in print_insn_state()
1533 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' '); in print_insn_state()
1535 verbose(env, "%d:", env->insn_idx); in print_insn_state()
1537 print_verifier_state(env, state, false); in print_insn_state()
1654 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) in acquire_reference_state() argument
1656 struct bpf_func_state *state = cur_func(env); in acquire_reference_state()
1663 id = ++env->id_gen; in acquire_reference_state()
1780 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in update_branch_counts() argument
1797 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, in pop_stack() argument
1800 struct bpf_verifier_state *cur = env->cur_state; in pop_stack()
1801 struct bpf_verifier_stack_elem *elem, *head = env->head; in pop_stack()
1804 if (env->head == NULL) in pop_stack()
1813 bpf_vlog_reset(&env->log, head->log_pos); in pop_stack()
1821 env->head = elem; in pop_stack()
1822 env->stack_size--; in pop_stack()
1826 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, in push_stack() argument
1830 struct bpf_verifier_state *cur = env->cur_state; in push_stack()
1840 elem->next = env->head; in push_stack()
1841 elem->log_pos = env->log.end_pos; in push_stack()
1842 env->head = elem; in push_stack()
1843 env->stack_size++; in push_stack()
1848 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_stack()
1849 verbose(env, "The sequence of %d jumps is too complex.\n", in push_stack()
1850 env->stack_size); in push_stack()
1867 free_verifier_state(env->cur_state, true); in push_stack()
1868 env->cur_state = NULL; in push_stack()
1870 while (!pop_stack(env, NULL, NULL, false)); in push_stack()
1930 static void mark_reg_known_zero(struct bpf_verifier_env *env, in mark_reg_known_zero() argument
1934 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); in mark_reg_known_zero()
1937 __mark_reg_not_init(env, regs + regno); in mark_reg_known_zero()
2263 static void __mark_reg_unknown(const struct bpf_verifier_env *env, in __mark_reg_unknown() argument
2276 reg->precise = !env->bpf_capable; in __mark_reg_unknown()
2280 static void mark_reg_unknown(struct bpf_verifier_env *env, in mark_reg_unknown() argument
2284 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); in mark_reg_unknown()
2287 __mark_reg_not_init(env, regs + regno); in mark_reg_unknown()
2290 __mark_reg_unknown(env, regs + regno); in mark_reg_unknown()
2293 static void __mark_reg_not_init(const struct bpf_verifier_env *env, in __mark_reg_not_init() argument
2296 __mark_reg_unknown(env, reg); in __mark_reg_not_init()
2300 static void mark_reg_not_init(struct bpf_verifier_env *env, in mark_reg_not_init() argument
2304 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); in mark_reg_not_init()
2307 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
2310 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
2313 static void mark_btf_ld_reg(struct bpf_verifier_env *env, in mark_btf_ld_reg() argument
2320 mark_reg_unknown(env, regs, regno); in mark_btf_ld_reg()
2323 mark_reg_known_zero(env, regs, regno); in mark_btf_ld_reg()
2330 static void init_reg_state(struct bpf_verifier_env *env, in init_reg_state() argument
2337 mark_reg_not_init(env, regs, i); in init_reg_state()
2345 mark_reg_known_zero(env, regs, BPF_REG_FP); in init_reg_state()
2350 static void init_func_state(struct bpf_verifier_env *env, in init_func_state() argument
2358 init_reg_state(env, state); in init_func_state()
2359 mark_verifier_state_scratched(env); in init_func_state()
2363 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, in push_async_cb() argument
2376 elem->next = env->head; in push_async_cb()
2377 elem->log_pos = env->log.end_pos; in push_async_cb()
2378 env->head = elem; in push_async_cb()
2379 env->stack_size++; in push_async_cb()
2380 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_async_cb()
2381 verbose(env, in push_async_cb()
2383 env->stack_size); in push_async_cb()
2395 init_func_state(env, frame, in push_async_cb()
2402 free_verifier_state(env->cur_state, true); in push_async_cb()
2403 env->cur_state = NULL; in push_async_cb()
2405 while (!pop_stack(env, NULL, NULL, false)); in push_async_cb()
2422 static int find_subprog(struct bpf_verifier_env *env, int off) in find_subprog() argument
2426 p = bsearch(&off, env->subprog_info, env->subprog_cnt, in find_subprog()
2427 sizeof(env->subprog_info[0]), cmp_subprogs); in find_subprog()
2430 return p - env->subprog_info; in find_subprog()
2434 static int add_subprog(struct bpf_verifier_env *env, int off) in add_subprog() argument
2436 int insn_cnt = env->prog->len; in add_subprog()
2440 verbose(env, "call to invalid destination\n"); in add_subprog()
2443 ret = find_subprog(env, off); in add_subprog()
2446 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { in add_subprog()
2447 verbose(env, "too many subprograms\n"); in add_subprog()
2451 env->subprog_info[env->subprog_cnt++].start = off; in add_subprog()
2452 sort(env->subprog_info, env->subprog_cnt, in add_subprog()
2453 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); in add_subprog()
2454 return env->subprog_cnt - 1; in add_subprog()
2533 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, in __find_kfunc_desc_btf() argument
2543 tab = env->prog->aux->kfunc_btf_tab; in __find_kfunc_desc_btf()
2548 verbose(env, "too many different module BTFs\n"); in __find_kfunc_desc_btf()
2552 if (bpfptr_is_null(env->fd_array)) { in __find_kfunc_desc_btf()
2553 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); in __find_kfunc_desc_btf()
2557 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, in __find_kfunc_desc_btf()
2564 verbose(env, "invalid module BTF fd specified\n"); in __find_kfunc_desc_btf()
2569 verbose(env, "BTF fd for kfunc is not a module BTF\n"); in __find_kfunc_desc_btf()
2603 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) in find_kfunc_desc_btf() argument
2610 verbose(env, "negative offset disallowed for kernel module function call\n"); in find_kfunc_desc_btf()
2614 return __find_kfunc_desc_btf(env, offset); in find_kfunc_desc_btf()
2619 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) in add_kfunc_call() argument
2632 prog_aux = env->prog->aux; in add_kfunc_call()
2637 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); in add_kfunc_call()
2641 if (!env->prog->jit_requested) { in add_kfunc_call()
2642 verbose(env, "JIT is required for calling kernel function\n"); in add_kfunc_call()
2647 verbose(env, "JIT does not support calling kernel function\n"); in add_kfunc_call()
2651 if (!env->prog->gpl_compatible) { in add_kfunc_call()
2652 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); in add_kfunc_call()
2678 desc_btf = find_kfunc_desc_btf(env, offset); in add_kfunc_call()
2680 verbose(env, "failed to find BTF for kernel function\n"); in add_kfunc_call()
2684 if (find_kfunc_desc(env->prog, func_id, offset)) in add_kfunc_call()
2688 verbose(env, "too many different kernel function calls\n"); in add_kfunc_call()
2694 verbose(env, "kernel btf_id %u is not a function\n", in add_kfunc_call()
2700 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", in add_kfunc_call()
2708 verbose(env, "cannot find address for kernel function %s\n", in add_kfunc_call()
2712 specialize_kfunc(env, func_id, offset, &addr); in add_kfunc_call()
2720 verbose(env, "address of kernel function %s is out of range\n", in add_kfunc_call()
2727 err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); in add_kfunc_call()
2737 err = btf_distill_func_proto(&env->log, desc_btf, in add_kfunc_call()
2793 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) in add_subprog_and_kfunc() argument
2795 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc()
2796 struct bpf_insn *insn = env->prog->insnsi; in add_subprog_and_kfunc()
2797 int i, ret, insn_cnt = env->prog->len; in add_subprog_and_kfunc()
2800 ret = add_subprog(env, 0); in add_subprog_and_kfunc()
2809 if (!env->bpf_capable) { in add_subprog_and_kfunc()
2810 …verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_AD… in add_subprog_and_kfunc()
2815 ret = add_subprog(env, i + insn->imm + 1); in add_subprog_and_kfunc()
2817 ret = add_kfunc_call(env, insn->imm, insn->off); in add_subprog_and_kfunc()
2826 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
2828 if (env->log.level & BPF_LOG_LEVEL2) in add_subprog_and_kfunc()
2829 for (i = 0; i < env->subprog_cnt; i++) in add_subprog_and_kfunc()
2830 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
2835 static int check_subprogs(struct bpf_verifier_env *env) in check_subprogs() argument
2838 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs()
2839 struct bpf_insn *insn = env->prog->insnsi; in check_subprogs()
2840 int insn_cnt = env->prog->len; in check_subprogs()
2864 verbose(env, "jump out of range from insn %d to %d\n", i, off); in check_subprogs()
2876 verbose(env, "last insn is not an exit or jmp\n"); in check_subprogs()
2881 if (cur_subprog < env->subprog_cnt) in check_subprogs()
2891 static int mark_reg_read(struct bpf_verifier_env *env, in mark_reg_read() argument
2903 verbose(env, "verifier BUG type %s var_off %lld off %d\n", in mark_reg_read()
2904 reg_type_str(env, parent->type), in mark_reg_read()
2934 if (env->longest_mark_read_walk < cnt) in mark_reg_read()
2935 env->longest_mark_read_walk = cnt; in mark_reg_read()
2939 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_dynptr_read() argument
2941 struct bpf_func_state *state = func(env, reg); in mark_dynptr_read()
2950 spi = dynptr_get_spi(env, reg); in mark_dynptr_read()
2957 ret = mark_reg_read(env, &state->stack[spi].spilled_ptr, in mark_dynptr_read()
2961 return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr, in mark_dynptr_read()
2965 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_iter_read() argument
2968 struct bpf_func_state *state = func(env, reg); in mark_iter_read()
2974 err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64); in mark_iter_read()
2978 mark_stack_slot_scratched(env, spi - i); in mark_iter_read()
2988 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, in is_reg64() argument
3098 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) in insn_has_def32() argument
3105 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
3108 static void mark_insn_zext(struct bpf_verifier_env *env, in mark_insn_zext() argument
3116 env->insn_aux_data[def_idx - 1].zext_dst = true; in mark_insn_zext()
3121 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, in check_reg_arg() argument
3124 struct bpf_verifier_state *vstate = env->cur_state; in check_reg_arg()
3126 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; in check_reg_arg()
3131 verbose(env, "R%d is invalid\n", regno); in check_reg_arg()
3135 mark_reg_scratched(env, regno); in check_reg_arg()
3138 rw64 = is_reg64(env, insn, regno, reg, t); in check_reg_arg()
3142 verbose(env, "R%d !read_ok\n", regno); in check_reg_arg()
3150 mark_insn_zext(env, reg); in check_reg_arg()
3152 return mark_reg_read(env, reg, reg->parent, in check_reg_arg()
3157 verbose(env, "frame pointer is read only\n"); in check_reg_arg()
3161 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; in check_reg_arg()
3163 mark_reg_unknown(env, regs, regno); in check_reg_arg()
3168 static void mark_jmp_point(struct bpf_verifier_env *env, int idx) in mark_jmp_point() argument
3170 env->insn_aux_data[idx].jmp_point = true; in mark_jmp_point()
3173 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) in is_jmp_point() argument
3175 return env->insn_aux_data[insn_idx].jmp_point; in is_jmp_point()
3179 static int push_jmp_history(struct bpf_verifier_env *env, in push_jmp_history() argument
3186 if (!is_jmp_point(env, env->insn_idx)) in push_jmp_history()
3194 p[cnt - 1].idx = env->insn_idx; in push_jmp_history()
3195 p[cnt - 1].prev_idx = env->prev_insn_idx; in push_jmp_history()
3241 struct bpf_verifier_env *env = bt->env; in bt_reset() local
3244 bt->env = env; in bt_reset()
3261 verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame); in bt_subprog_enter()
3272 verbose(bt->env, "BUG subprog exit from frame 0\n"); in bt_subprog_exit()
3398 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, in backtrack_insn() argument
3404 .private_data = env, in backtrack_insn()
3406 struct bpf_insn *insn = env->prog->insnsi + idx; in backtrack_insn()
3416 if (env->log.level & BPF_LOG_LEVEL2) { in backtrack_insn()
3417 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); in backtrack_insn()
3418 verbose(env, "mark_precise: frame%d: regs=%s ", in backtrack_insn()
3419 bt->frame, env->tmp_str_buf); in backtrack_insn()
3420 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); in backtrack_insn()
3421 verbose(env, "stack=%s before ", env->tmp_str_buf); in backtrack_insn()
3422 verbose(env, "%d: ", idx); in backtrack_insn()
3423 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in backtrack_insn()
3477 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
3494 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
3508 subprog = find_subprog(env, subprog_insn_idx); in backtrack_insn()
3512 if (subprog_is_global(env, subprog)) { in backtrack_insn()
3525 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
3540 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
3571 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
3596 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
3607 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
3621 bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && in backtrack_insn()
3718 static void mark_all_scalars_precise(struct bpf_verifier_env *env, in mark_all_scalars_precise() argument
3725 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
3726 verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", in mark_all_scalars_precise()
3744 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
3745 verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", in mark_all_scalars_precise()
3756 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
3757 verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", in mark_all_scalars_precise()
3765 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in mark_all_scalars_imprecise() argument
3817 static int mark_precise_scalar_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in mark_precise_scalar_ids() argument
3819 struct bpf_idset *precise_ids = &env->idset_scratch; in mark_precise_scalar_ids()
3820 struct backtrack_state *bt = &env->bt; in mark_precise_scalar_ids()
3967 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) in __mark_chain_precision() argument
3969 struct backtrack_state *bt = &env->bt; in __mark_chain_precision()
3970 struct bpf_verifier_state *st = env->cur_state; in __mark_chain_precision()
3972 int last_idx = env->insn_idx; in __mark_chain_precision()
3979 if (!env->bpf_capable) in __mark_chain_precision()
3983 bt_init(bt, env->cur_state->curframe); in __mark_chain_precision()
4006 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
4007 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", in __mark_chain_precision()
4033 if (mark_precise_scalar_ids(env, st)) in __mark_chain_precision()
4057 verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n", in __mark_chain_precision()
4068 err = backtrack_insn(env, i, subseq_idx, bt); in __mark_chain_precision()
4071 mark_all_scalars_precise(env, env->cur_state); in __mark_chain_precision()
4087 if (i >= env->prog->len) { in __mark_chain_precision()
4094 verbose(env, "BUG backtracking idx %d\n", i); in __mark_chain_precision()
4134 mark_all_scalars_precise(env, env->cur_state); in __mark_chain_precision()
4149 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
4150 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, in __mark_chain_precision()
4152 verbose(env, "mark_precise: frame%d: parent state regs=%s ", in __mark_chain_precision()
4153 fr, env->tmp_str_buf); in __mark_chain_precision()
4154 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, in __mark_chain_precision()
4156 verbose(env, "stack=%s: ", env->tmp_str_buf); in __mark_chain_precision()
4157 print_verifier_state(env, func, true); in __mark_chain_precision()
4174 mark_all_scalars_precise(env, env->cur_state); in __mark_chain_precision()
4181 int mark_chain_precision(struct bpf_verifier_env *env, int regno) in mark_chain_precision() argument
4183 return __mark_chain_precision(env, regno); in mark_chain_precision()
4189 static int mark_chain_precision_batch(struct bpf_verifier_env *env) in mark_chain_precision_batch() argument
4191 return __mark_chain_precision(env, -1); in mark_chain_precision_batch()
4291 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, in check_stack_write_fixed_off() argument
4299 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_stack_write_fixed_off()
4309 if (!env->allow_ptr_leaks && in check_stack_write_fixed_off()
4312 verbose(env, "attempt to corrupt spilled pointer on stack\n"); in check_stack_write_fixed_off()
4316 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_fixed_off()
4319 if (!env->bypass_spec_v4) { in check_stack_write_fixed_off()
4332 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; in check_stack_write_fixed_off()
4335 err = destroy_if_dynptr_stack_slot(env, state, spi); in check_stack_write_fixed_off()
4339 mark_stack_slot_scratched(env, spi); in check_stack_write_fixed_off()
4341 !register_is_null(reg) && env->bpf_capable) { in check_stack_write_fixed_off()
4349 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
4358 insn->imm != 0 && env->bpf_capable) { in check_stack_write_fixed_off()
4367 verbose_linfo(env, insn_idx, "; "); in check_stack_write_fixed_off()
4368 verbose(env, "invalid size of register spill\n"); in check_stack_write_fixed_off()
4372 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); in check_stack_write_fixed_off()
4401 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
4434 static int check_stack_write_var_off(struct bpf_verifier_env *env, in check_stack_write_var_off() argument
4444 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_stack_write_var_off()
4451 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_var_off()
4469 err = destroy_if_dynptr_stack_slot(env, state, spi); in check_stack_write_var_off()
4482 mark_stack_slot_scratched(env, spi); in check_stack_write_var_off()
4484 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { in check_stack_write_var_off()
4496 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", in check_stack_write_var_off()
4518 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { in check_stack_write_var_off()
4519 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", in check_stack_write_var_off()
4527 err = mark_chain_precision(env, value_regno); in check_stack_write_var_off()
4542 static void mark_reg_stack_read(struct bpf_verifier_env *env, in mark_reg_stack_read() argument
4547 struct bpf_verifier_state *vstate = env->cur_state; in mark_reg_stack_read()
4556 mark_stack_slot_scratched(env, spi); in mark_reg_stack_read()
4580 mark_reg_unknown(env, state->regs, dst_regno); in mark_reg_stack_read()
4594 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, in check_stack_read_fixed_off() argument
4599 struct bpf_verifier_state *vstate = env->cur_state; in check_stack_read_fixed_off()
4608 mark_stack_slot_scratched(env, spi); in check_stack_read_fixed_off()
4618 verbose_linfo(env, env->insn_idx, "; "); in check_stack_read_fixed_off()
4619 verbose(env, "invalid size of register fill\n"); in check_stack_read_fixed_off()
4623 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
4642 if (type == STACK_INVALID && env->allow_uninit_stack) in check_stack_read_fixed_off()
4644 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
4648 mark_reg_unknown(env, state->regs, dst_regno); in check_stack_read_fixed_off()
4662 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { in check_stack_read_fixed_off()
4669 verbose(env, "leaking pointer from stack off %d\n", in check_stack_read_fixed_off()
4673 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
4681 if (type == STACK_INVALID && env->allow_uninit_stack) in check_stack_read_fixed_off()
4683 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
4687 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
4689 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); in check_stack_read_fixed_off()
4699 static int check_stack_range_initialized(struct bpf_verifier_env *env,
4705 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) in reg_state() argument
4707 return cur_regs(env) + regno; in reg_state()
4723 static int check_stack_read_var_off(struct bpf_verifier_env *env, in check_stack_read_var_off() argument
4727 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read_var_off()
4728 struct bpf_func_state *ptr_state = func(env, reg); in check_stack_read_var_off()
4734 err = check_stack_range_initialized(env, ptr_regno, off, size, in check_stack_read_var_off()
4741 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); in check_stack_read_var_off()
4754 static int check_stack_read(struct bpf_verifier_env *env, in check_stack_read() argument
4758 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read()
4759 struct bpf_func_state *state = func(env, reg); in check_stack_read()
4772 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=… in check_stack_read()
4787 err = check_stack_read_fixed_off(env, state, off, size, in check_stack_read()
4794 err = check_stack_read_var_off(env, ptr_regno, off, size, in check_stack_read()
4811 static int check_stack_write(struct bpf_verifier_env *env, in check_stack_write() argument
4815 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_write()
4816 struct bpf_func_state *state = func(env, reg); in check_stack_write()
4821 err = check_stack_write_fixed_off(env, state, off, size, in check_stack_write()
4827 err = check_stack_write_var_off(env, state, in check_stack_write()
4834 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, in check_map_access_type() argument
4837 struct bpf_reg_state *regs = cur_regs(env); in check_map_access_type()
4842 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
4848 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
4857 static int __check_mem_access(struct bpf_verifier_env *env, int regno, in __check_mem_access() argument
4867 reg = &cur_regs(env)[regno]; in __check_mem_access()
4870 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", in __check_mem_access()
4874 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", in __check_mem_access()
4880 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", in __check_mem_access()
4885 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", in __check_mem_access()
4893 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, in check_mem_region_access() argument
4897 struct bpf_verifier_state *vstate = env->cur_state; in check_mem_region_access()
4916 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_mem_region_access()
4920 err = __check_mem_access(env, regno, reg->smin_value + off, size, in check_mem_region_access()
4923 verbose(env, "R%d min value is outside of the allowed memory range\n", in check_mem_region_access()
4933 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", in check_mem_region_access()
4937 err = __check_mem_access(env, regno, reg->umax_value + off, size, in check_mem_region_access()
4940 verbose(env, "R%d max value is outside of the allowed memory range\n", in check_mem_region_access()
4948 static int __check_ptr_off_reg(struct bpf_verifier_env *env, in __check_ptr_off_reg() argument
4957 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
4958 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
4963 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
4964 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
4972 verbose(env, "variable %s access var_off=%s disallowed\n", in __check_ptr_off_reg()
4973 reg_type_str(env, reg->type), tn_buf); in __check_ptr_off_reg()
4980 int check_ptr_off_reg(struct bpf_verifier_env *env, in check_ptr_off_reg() argument
4983 return __check_ptr_off_reg(env, reg, regno, false); in check_ptr_off_reg()
4986 static int map_kptr_match_type(struct bpf_verifier_env *env, in map_kptr_match_type() argument
5016 if (__check_ptr_off_reg(env, reg, regno, true)) in map_kptr_match_type()
5043 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in map_kptr_match_type()
5049 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, in map_kptr_match_type()
5050 reg_type_str(env, reg->type), reg_name); in map_kptr_match_type()
5051 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); in map_kptr_match_type()
5053 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), in map_kptr_match_type()
5056 verbose(env, "\n"); in map_kptr_match_type()
5063 static bool in_rcu_cs(struct bpf_verifier_env *env) in in_rcu_cs() argument
5065 return env->cur_state->active_rcu_lock || in in_rcu_cs()
5066 env->cur_state->active_lock.ptr || in in_rcu_cs()
5067 !env->prog->aux->sleepable; in in_rcu_cs()
5092 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, in check_map_kptr_access() argument
5096 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_map_kptr_access()
5108 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); in check_map_kptr_access()
5116 verbose(env, "store to referenced kptr disallowed\n"); in check_map_kptr_access()
5121 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
5125 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, in check_map_kptr_access()
5127 rcu_safe_kptr(kptr_field) && in_rcu_cs(env) ? in check_map_kptr_access()
5131 val_reg->id = ++env->id_gen; in check_map_kptr_access()
5133 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
5135 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) in check_map_kptr_access()
5139 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", in check_map_kptr_access()
5144 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); in check_map_kptr_access()
5151 static int check_map_access(struct bpf_verifier_env *env, u32 regno, in check_map_access() argument
5155 struct bpf_verifier_state *vstate = env->cur_state; in check_map_access()
5162 err = check_mem_region_access(env, regno, off, size, map->value_size, in check_map_access()
5184 verbose(env, "kptr cannot be accessed indirectly by helper\n"); in check_map_access()
5188 verbose(env, "kptr access cannot have variable offset\n"); in check_map_access()
5192 verbose(env, "kptr access misaligned expected=%u off=%llu\n", in check_map_access()
5197 verbose(env, "kptr access size must be BPF_DW\n"); in check_map_access()
5202 verbose(env, "%s cannot be accessed directly by load/store\n", in check_map_access()
5213 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, in may_access_direct_pkt_data() argument
5217 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in may_access_direct_pkt_data()
5241 env->seen_direct_write = true; in may_access_direct_pkt_data()
5246 env->seen_direct_write = true; in may_access_direct_pkt_data()
5255 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, in check_packet_access() argument
5258 struct bpf_reg_state *regs = cur_regs(env); in check_packet_access()
5271 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_packet_access()
5277 __check_mem_access(env, regno, off, size, reg->range, in check_packet_access()
5280 verbose(env, "R%d offset is outside of the packet\n", regno); in check_packet_access()
5290 env->prog->aux->max_pkt_offset = in check_packet_access()
5291 max_t(u32, env->prog->aux->max_pkt_offset, in check_packet_access()
5298 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, in check_ctx_access() argument
5304 .log = &env->log, in check_ctx_access()
5307 if (env->ops->is_valid_access && in check_ctx_access()
5308 env->ops->is_valid_access(off, size, t, env->prog, &info)) { in check_ctx_access()
5322 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; in check_ctx_access()
5325 if (env->prog->aux->max_ctx_offset < off + size) in check_ctx_access()
5326 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
5330 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); in check_ctx_access()
5334 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, in check_flow_keys_access() argument
5339 verbose(env, "invalid access to flow keys off=%d size=%d\n", in check_flow_keys_access()
5346 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, in check_sock_access() argument
5350 struct bpf_reg_state *regs = cur_regs(env); in check_sock_access()
5356 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_sock_access()
5380 env->insn_aux_data[insn_idx].ctx_field_size = in check_sock_access()
5385 verbose(env, "R%d invalid %s access off=%d size=%d\n", in check_sock_access()
5386 regno, reg_type_str(env, reg->type), off, size); in check_sock_access()
5391 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) in is_pointer_value() argument
5393 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); in is_pointer_value()
5396 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) in is_ctx_reg() argument
5398 const struct bpf_reg_state *reg = reg_state(env, regno); in is_ctx_reg()
5403 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) in is_sk_reg() argument
5405 const struct bpf_reg_state *reg = reg_state(env, regno); in is_sk_reg()
5410 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) in is_pkt_reg() argument
5412 const struct bpf_reg_state *reg = reg_state(env, regno); in is_pkt_reg()
5417 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) in is_flow_key_reg() argument
5419 const struct bpf_reg_state *reg = reg_state(env, regno); in is_flow_key_reg()
5467 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, in check_pkt_ptr_alignment() argument
5493 verbose(env, in check_pkt_ptr_alignment()
5502 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, in check_generic_ptr_alignment() argument
5518 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", in check_generic_ptr_alignment()
5526 static int check_ptr_alignment(struct bpf_verifier_env *env, in check_ptr_alignment() argument
5530 bool strict = env->strict_alignment || strict_alignment_once; in check_ptr_alignment()
5539 return check_pkt_ptr_alignment(env, reg, off, size, strict); in check_ptr_alignment()
5575 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, in check_ptr_alignment()
5579 static int update_stack_depth(struct bpf_verifier_env *env, in update_stack_depth() argument
5583 u16 stack = env->subprog_info[func->subprogno].stack_depth; in update_stack_depth()
5589 env->subprog_info[func->subprogno].stack_depth = -off; in update_stack_depth()
5599 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx) in check_max_stack_depth_subprog() argument
5601 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth_subprog()
5602 struct bpf_insn *insn = env->prog->insnsi; in check_max_stack_depth_subprog()
5631 verbose(env, in check_max_stack_depth_subprog()
5641 verbose(env, "combined stack size of %d calls is %d. Too large\n", in check_max_stack_depth_subprog()
5658 sidx = find_subprog(env, next_insn); in check_max_stack_depth_subprog()
5666 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); in check_max_stack_depth_subprog()
5681 verbose(env, "the call stack of %d frames is too deep !\n", in check_max_stack_depth_subprog()
5696 env->prog->aux->tail_call_reachable = true; in check_max_stack_depth_subprog()
5710 static int check_max_stack_depth(struct bpf_verifier_env *env) in check_max_stack_depth() argument
5712 struct bpf_subprog_info *si = env->subprog_info; in check_max_stack_depth()
5715 for (int i = 0; i < env->subprog_cnt; i++) { in check_max_stack_depth()
5717 ret = check_max_stack_depth_subprog(env, i); in check_max_stack_depth()
5727 static int get_callee_stack_depth(struct bpf_verifier_env *env, in get_callee_stack_depth() argument
5732 subprog = find_subprog(env, start); in get_callee_stack_depth()
5738 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
5742 static int __check_buffer_access(struct bpf_verifier_env *env, in __check_buffer_access() argument
5748 verbose(env, in __check_buffer_access()
5757 verbose(env, in __check_buffer_access()
5766 static int check_tp_buffer_access(struct bpf_verifier_env *env, in check_tp_buffer_access() argument
5772 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); in check_tp_buffer_access()
5776 if (off + size > env->prog->aux->max_tp_access) in check_tp_buffer_access()
5777 env->prog->aux->max_tp_access = off + size; in check_tp_buffer_access()
5782 static int check_buffer_access(struct bpf_verifier_env *env, in check_buffer_access() argument
5791 err = __check_buffer_access(env, buf_info, reg, regno, off, size); in check_buffer_access()
6101 static bool type_is_rcu(struct bpf_verifier_env *env, in type_is_rcu() argument
6109 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu"); in type_is_rcu()
6112 static bool type_is_rcu_or_null(struct bpf_verifier_env *env, in type_is_rcu_or_null() argument
6120 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null"); in type_is_rcu_or_null()
6123 static bool type_is_trusted(struct bpf_verifier_env *env, in type_is_trusted() argument
6134 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); in type_is_trusted()
6137 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, in check_ptr_to_btf_access() argument
6151 if (!env->allow_ptr_leaks) { in check_ptr_to_btf_access()
6152 verbose(env, in check_ptr_to_btf_access()
6157 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { in check_ptr_to_btf_access()
6158 verbose(env, in check_ptr_to_btf_access()
6164 verbose(env, in check_ptr_to_btf_access()
6173 verbose(env, in check_ptr_to_btf_access()
6180 verbose(env, in check_ptr_to_btf_access()
6187 verbose(env, in check_ptr_to_btf_access()
6193 if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { in check_ptr_to_btf_access()
6195 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); in check_ptr_to_btf_access()
6198 ret = env->ops->btf_struct_access(&env->log, reg, off, size); in check_ptr_to_btf_access()
6205 verbose(env, "only read is supported\n"); in check_ptr_to_btf_access()
6211 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); in check_ptr_to_btf_access()
6215 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name); in check_ptr_to_btf_access()
6243 if (type_is_trusted(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
6245 } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) { in check_ptr_to_btf_access()
6246 if (type_is_rcu(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
6250 type_is_rcu_or_null(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
6255 if (type_is_rcu_or_null(env, reg, field_name, btf_id) && in check_ptr_to_btf_access()
6280 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); in check_ptr_to_btf_access()
6285 static int check_ptr_to_map_access(struct bpf_verifier_env *env, in check_ptr_to_map_access() argument
6301 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); in check_ptr_to_map_access()
6306 verbose(env, "map_ptr access not supported for map type %d\n", in check_ptr_to_map_access()
6314 if (!env->allow_ptr_leaks) { in check_ptr_to_map_access()
6315 verbose(env, in check_ptr_to_map_access()
6322 verbose(env, "R%d is %s invalid negative access: off=%d\n", in check_ptr_to_map_access()
6328 verbose(env, "only read from %s is supported\n", tname); in check_ptr_to_map_access()
6334 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); in check_ptr_to_map_access()
6335 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL); in check_ptr_to_map_access()
6340 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); in check_ptr_to_map_access()
6373 struct bpf_verifier_env *env, in check_stack_access_within_bounds() argument
6377 struct bpf_reg_state *regs = cur_regs(env); in check_stack_access_within_bounds()
6379 struct bpf_func_state *state = func(env, reg); in check_stack_access_within_bounds()
6401 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", in check_stack_access_within_bounds()
6418 verbose(env, "invalid%s stack R%d off=%d size=%d\n", in check_stack_access_within_bounds()
6424 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", in check_stack_access_within_bounds()
6437 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, in check_mem_access() argument
6441 struct bpf_reg_state *regs = cur_regs(env); in check_mem_access()
6451 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); in check_mem_access()
6460 verbose(env, "write to change key R%d not allowed\n", regno); in check_mem_access()
6464 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
6469 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6474 is_pointer_value(env, value_regno)) { in check_mem_access()
6475 verbose(env, "R%d leaks addr into map\n", value_regno); in check_mem_access()
6478 err = check_map_access_type(env, regno, off, size, t); in check_mem_access()
6481 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); in check_mem_access()
6488 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); in check_mem_access()
6507 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6514 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
6515 reg_type_str(env, reg->type)); in check_mem_access()
6520 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
6521 regno, reg_type_str(env, reg->type)); in check_mem_access()
6526 is_pointer_value(env, value_regno)) { in check_mem_access()
6527 verbose(env, "R%d leaks addr into mem\n", value_regno); in check_mem_access()
6531 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
6534 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6541 is_pointer_value(env, value_regno)) { in check_mem_access()
6542 verbose(env, "R%d leaks addr into ctx\n", value_regno); in check_mem_access()
6546 err = check_ptr_off_reg(env, reg, regno); in check_mem_access()
6550 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, in check_mem_access()
6553 verbose_linfo(env, insn_idx, "; "); in check_mem_access()
6560 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6562 mark_reg_known_zero(env, regs, in check_mem_access()
6565 regs[value_regno].id = ++env->id_gen; in check_mem_access()
6582 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); in check_mem_access()
6586 state = func(env, reg); in check_mem_access()
6587 err = update_stack_depth(env, state, off); in check_mem_access()
6592 err = check_stack_read(env, regno, off, size, in check_mem_access()
6595 err = check_stack_write(env, regno, off, size, in check_mem_access()
6598 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { in check_mem_access()
6599 verbose(env, "cannot write into packet\n"); in check_mem_access()
6603 is_pointer_value(env, value_regno)) { in check_mem_access()
6604 verbose(env, "R%d leaks addr into packet\n", in check_mem_access()
6608 err = check_packet_access(env, regno, off, size, false); in check_mem_access()
6610 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6613 is_pointer_value(env, value_regno)) { in check_mem_access()
6614 verbose(env, "R%d leaks addr into flow keys\n", in check_mem_access()
6619 err = check_flow_keys_access(env, off, size); in check_mem_access()
6621 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6624 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
6625 regno, reg_type_str(env, reg->type)); in check_mem_access()
6628 err = check_sock_access(env, insn_idx, regno, off, size, t); in check_mem_access()
6630 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6632 err = check_tp_buffer_access(env, reg, regno, off, size); in check_mem_access()
6634 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6637 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, in check_mem_access()
6640 err = check_ptr_to_map_access(env, regs, regno, off, size, t, in check_mem_access()
6648 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
6649 regno, reg_type_str(env, reg->type)); in check_mem_access()
6652 max_access = &env->prog->aux->max_rdonly_access; in check_mem_access()
6654 max_access = &env->prog->aux->max_rdwr_access; in check_mem_access()
6657 err = check_buffer_access(env, reg, regno, off, size, false, in check_mem_access()
6661 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
6663 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
6664 reg_type_str(env, reg->type)); in check_mem_access()
6679 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) in check_atomic() argument
6697 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); in check_atomic()
6702 verbose(env, "invalid atomic operand size\n"); in check_atomic()
6707 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_atomic()
6712 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
6720 err = check_reg_arg(env, aux_reg, SRC_OP); in check_atomic()
6724 if (is_pointer_value(env, aux_reg)) { in check_atomic()
6725 verbose(env, "R%d leaks addr into mem\n", aux_reg); in check_atomic()
6730 if (is_pointer_value(env, insn->src_reg)) { in check_atomic()
6731 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); in check_atomic()
6735 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
6736 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
6737 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
6738 is_sk_reg(env, insn->dst_reg)) { in check_atomic()
6739 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", in check_atomic()
6741 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
6752 err = check_reg_arg(env, load_reg, DST_OP); in check_atomic()
6765 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
6768 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
6775 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
6793 struct bpf_verifier_env *env, int regno, int off, in check_stack_range_initialized() argument
6797 struct bpf_reg_state *reg = reg_state(env, regno); in check_stack_range_initialized()
6798 struct bpf_func_state *state = func(env, reg); in check_stack_range_initialized()
6808 verbose(env, "invalid zero-sized read\n"); in check_stack_range_initialized()
6822 err = check_stack_access_within_bounds(env, regno, off, access_size, in check_stack_range_initialized()
6836 if (!env->bypass_spec_v1) { in check_stack_range_initialized()
6840 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", in check_stack_range_initialized()
6879 verbose(env, "potential write to dynptr at off=%d disallowed\n", i); in check_stack_range_initialized()
6899 (*stype == STACK_INVALID && env->allow_uninit_stack)) { in check_stack_range_initialized()
6909 env->allow_ptr_leaks)) { in check_stack_range_initialized()
6911 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); in check_stack_range_initialized()
6920 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", in check_stack_range_initialized()
6926 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", in check_stack_range_initialized()
6934 mark_reg_read(env, &state->stack[spi].spilled_ptr, in check_stack_range_initialized()
6943 return update_stack_depth(env, state, min_off); in check_stack_range_initialized()
6946 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, in check_helper_mem_access() argument
6950 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_helper_mem_access()
6956 return check_packet_access(env, regno, reg->off, access_size, in check_helper_mem_access()
6960 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
6961 reg_type_str(env, reg->type)); in check_helper_mem_access()
6964 return check_mem_region_access(env, regno, reg->off, access_size, in check_helper_mem_access()
6967 if (check_map_access_type(env, regno, reg->off, access_size, in check_helper_mem_access()
6971 return check_map_access(env, regno, reg->off, access_size, in check_helper_mem_access()
6976 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
6977 reg_type_str(env, reg->type)); in check_helper_mem_access()
6981 return check_mem_region_access(env, regno, reg->off, in check_helper_mem_access()
6987 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
6988 reg_type_str(env, reg->type)); in check_helper_mem_access()
6992 max_access = &env->prog->aux->max_rdonly_access; in check_helper_mem_access()
6994 max_access = &env->prog->aux->max_rdwr_access; in check_helper_mem_access()
6996 return check_buffer_access(env, reg, regno, reg->off, in check_helper_mem_access()
7001 env, in check_helper_mem_access()
7005 return check_ptr_to_btf_access(env, regs, regno, reg->off, in check_helper_mem_access()
7013 if (!env->ops->convert_ctx_access) { in check_helper_mem_access()
7021 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, in check_helper_mem_access()
7032 verbose(env, "R%d type=%s ", regno, in check_helper_mem_access()
7033 reg_type_str(env, reg->type)); in check_helper_mem_access()
7034 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); in check_helper_mem_access()
7039 static int check_mem_size_reg(struct bpf_verifier_env *env, in check_mem_size_reg() argument
7068 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", in check_mem_size_reg()
7074 err = check_helper_mem_access(env, regno - 1, 0, in check_mem_size_reg()
7082 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", in check_mem_size_reg()
7086 err = check_helper_mem_access(env, regno - 1, in check_mem_size_reg()
7090 err = mark_chain_precision(env, regno); in check_mem_size_reg()
7094 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_mem_reg() argument
7115 err = check_helper_mem_access(env, regno, mem_size, true, &meta); in check_mem_reg()
7118 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); in check_mem_reg()
7126 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_kfunc_mem_size_reg() argument
7129 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; in check_kfunc_mem_size_reg()
7144 err = check_mem_size_reg(env, reg, regno, true, &meta); in check_kfunc_mem_size_reg()
7147 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); in check_kfunc_mem_size_reg()
7176 static int process_spin_lock(struct bpf_verifier_env *env, int regno, in process_spin_lock() argument
7179 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_spin_lock()
7180 struct bpf_verifier_state *cur = env->cur_state; in process_spin_lock()
7188 verbose(env, in process_spin_lock()
7196 verbose(env, in process_spin_lock()
7207 verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local", in process_spin_lock()
7212 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n", in process_spin_lock()
7218 verbose(env, in process_spin_lock()
7236 verbose(env, "bpf_spin_unlock without taking a lock\n"); in process_spin_lock()
7241 verbose(env, "bpf_spin_unlock of different lock\n"); in process_spin_lock()
7245 invalidate_non_owning_refs(env); in process_spin_lock()
7253 static int process_timer_func(struct bpf_verifier_env *env, int regno, in process_timer_func() argument
7256 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_timer_func()
7262 verbose(env, in process_timer_func()
7268 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", in process_timer_func()
7273 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); in process_timer_func()
7277 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", in process_timer_func()
7282 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); in process_timer_func()
7290 static int process_kptr_func(struct bpf_verifier_env *env, int regno, in process_kptr_func() argument
7293 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_kptr_func()
7299 verbose(env, in process_kptr_func()
7305 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", in process_kptr_func()
7310 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); in process_kptr_func()
7318 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); in process_kptr_func()
7322 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); in process_kptr_func()
7354 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, in process_dynptr_func() argument
7357 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_dynptr_func()
7364 verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n"); in process_dynptr_func()
7386 if (!is_dynptr_reg_valid_uninit(env, reg)) { in process_dynptr_func()
7387 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); in process_dynptr_func()
7393 err = check_mem_access(env, insn_idx, regno, in process_dynptr_func()
7399 err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id); in process_dynptr_func()
7403 verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n"); in process_dynptr_func()
7407 if (!is_dynptr_reg_valid_init(env, reg)) { in process_dynptr_func()
7408 verbose(env, in process_dynptr_func()
7415 if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) { in process_dynptr_func()
7416 verbose(env, in process_dynptr_func()
7422 err = mark_dynptr_read(env, reg); in process_dynptr_func()
7427 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) in iter_ref_obj_id() argument
7429 struct bpf_func_state *state = func(env, reg); in iter_ref_obj_id()
7462 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx, in process_iter_arg() argument
7465 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_iter_arg()
7479 if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { in process_iter_arg()
7480 verbose(env, "expected uninitialized iter_%s as arg #%d\n", in process_iter_arg()
7486 err = check_mem_access(env, insn_idx, regno, in process_iter_arg()
7492 err = mark_stack_slots_iter(env, reg, insn_idx, meta->btf, btf_id, nr_slots); in process_iter_arg()
7497 if (!is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots)) { in process_iter_arg()
7498 verbose(env, "expected an initialized iter_%s as arg #%d\n", in process_iter_arg()
7503 spi = iter_get_spi(env, reg, nr_slots); in process_iter_arg()
7507 err = mark_iter_read(env, reg, spi, nr_slots); in process_iter_arg()
7514 meta->ref_obj_id = iter_ref_obj_id(env, reg, spi); in process_iter_arg()
7517 err = unmark_stack_slots_iter(env, reg, nr_slots); in process_iter_arg()
7582 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, in process_iter_next_call() argument
7585 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st; in process_iter_next_call()
7593 cur_iter = &env->cur_state->frame[iter_frameno]->stack[iter_spi].spilled_ptr; in process_iter_next_call()
7597 verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n", in process_iter_next_call()
7604 queued_st = push_stack(env, insn_idx + 1, insn_idx, false); in process_iter_next_call()
7650 static int resolve_map_arg_type(struct bpf_verifier_env *env, in resolve_map_arg_type() argument
7656 verbose(env, "invalid map_ptr to access map->type\n"); in resolve_map_arg_type()
7666 verbose(env, "invalid arg_type for sockmap/sockhash\n"); in resolve_map_arg_type()
7797 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, in check_reg_type() argument
7802 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_reg_type()
7809 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); in check_reg_type()
7844 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); in check_reg_type()
7846 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); in check_reg_type()
7847 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); in check_reg_type()
7856 verbose(env, in check_reg_type()
7859 regno, reg_type_str(env, reg->type)); in check_reg_type()
7881 verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno); in check_reg_type()
7887 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); in check_reg_type()
7894 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) in check_reg_type()
7898 verbose(env, "verifier internal error:"); in check_reg_type()
7899 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", in check_reg_type()
7904 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in check_reg_type()
7907 verbose(env, "R%d is of type %s but %s is expected\n", in check_reg_type()
7918 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); in check_reg_type()
7922 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) in check_reg_type()
7931 verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n"); in check_reg_type()
7954 int check_func_arg_reg_off(struct bpf_verifier_env *env, in check_func_arg_reg_off() argument
7981 verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n", in check_func_arg_reg_off()
7985 return __check_ptr_off_reg(env, reg, regno, false); in check_func_arg_reg_off()
8018 return __check_ptr_off_reg(env, reg, regno, true); in check_func_arg_reg_off()
8020 return __check_ptr_off_reg(env, reg, regno, false); in check_func_arg_reg_off()
8024 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env, in get_dynptr_arg_reg() argument
8034 verbose(env, "verifier internal error: multiple dynptr args\n"); in get_dynptr_arg_reg()
8041 verbose(env, "verifier internal error: no dynptr arg found\n"); in get_dynptr_arg_reg()
8046 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_id() argument
8048 struct bpf_func_state *state = func(env, reg); in dynptr_id()
8053 spi = dynptr_get_spi(env, reg); in dynptr_id()
8059 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_ref_obj_id() argument
8061 struct bpf_func_state *state = func(env, reg); in dynptr_ref_obj_id()
8066 spi = dynptr_get_spi(env, reg); in dynptr_ref_obj_id()
8072 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, in dynptr_get_type() argument
8075 struct bpf_func_state *state = func(env, reg); in dynptr_get_type()
8083 verbose(env, "verifier internal error: invalid spi when querying dynptr type\n"); in dynptr_get_type()
8090 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, in check_func_arg() argument
8096 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_func_arg()
8105 err = check_reg_arg(env, regno, SRC_OP); in check_func_arg()
8110 if (is_pointer_value(env, regno)) { in check_func_arg()
8111 verbose(env, "R%d leaks addr into helper function\n", in check_func_arg()
8119 !may_access_direct_pkt_data(env, meta, BPF_READ)) { in check_func_arg()
8120 verbose(env, "helper access to the packet is not allowed\n"); in check_func_arg()
8125 err = resolve_map_arg_type(env, meta, &arg_type); in check_func_arg()
8141 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); in check_func_arg()
8145 err = check_func_arg_reg_off(env, reg, regno, arg_type); in check_func_arg()
8152 struct bpf_func_state *state = func(env, reg); in check_func_arg()
8161 spi = dynptr_get_spi(env, reg); in check_func_arg()
8163 verbose(env, "arg %d is an unacquired reference\n", regno); in check_func_arg()
8167 verbose(env, "cannot release unowned const bpf_dynptr\n"); in check_func_arg()
8171 verbose(env, "R%d must be referenced when passed to release function\n", in check_func_arg()
8176 verbose(env, "verifier internal error: more than one release argument\n"); in check_func_arg()
8184 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_func_arg()
8210 verbose(env, in check_func_arg()
8230 verbose(env, "invalid map_ptr to access map->key\n"); in check_func_arg()
8233 err = check_helper_mem_access(env, regno, in check_func_arg()
8246 verbose(env, "invalid map_ptr to access map->value\n"); in check_func_arg()
8250 err = check_helper_mem_access(env, regno, in check_func_arg()
8256 verbose(env, "Helper has invalid btf_id in R%d\n", regno); in check_func_arg()
8263 if (in_rbtree_lock_required_cb(env)) { in check_func_arg()
8264 verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); in check_func_arg()
8268 err = process_spin_lock(env, regno, true); in check_func_arg()
8272 err = process_spin_lock(env, regno, false); in check_func_arg()
8276 verbose(env, "verifier internal error\n"); in check_func_arg()
8281 err = process_timer_func(env, regno, meta); in check_func_arg()
8294 err = check_helper_mem_access(env, regno, in check_func_arg()
8300 err = check_mem_size_reg(env, reg, regno, false, meta); in check_func_arg()
8303 err = check_mem_size_reg(env, reg, regno, true, meta); in check_func_arg()
8306 err = process_dynptr_func(env, regno, insn_idx, arg_type, 0); in check_func_arg()
8312 verbose(env, "R%d is not a known constant'\n", in check_func_arg()
8317 err = mark_chain_precision(env, regno); in check_func_arg()
8326 err = check_helper_mem_access(env, regno, size, false, meta); in check_func_arg()
8329 err = check_ptr_alignment(env, reg, 0, size, true); in check_func_arg()
8340 verbose(env, "R%d does not point to a readonly map'\n", regno); in check_func_arg()
8345 verbose(env, "R%d is not a constant address'\n", regno); in check_func_arg()
8350 verbose(env, "no direct value access support for this map type\n"); in check_func_arg()
8354 err = check_map_access(env, regno, reg->off, in check_func_arg()
8363 verbose(env, "direct value access on string failed\n"); in check_func_arg()
8369 verbose(env, "string is not zero-terminated\n"); in check_func_arg()
8375 err = process_kptr_func(env, regno, meta); in check_func_arg()
8384 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) in may_update_sockmap() argument
8386 enum bpf_attach_type eatype = env->prog->expected_attach_type; in may_update_sockmap()
8387 enum bpf_prog_type type = resolve_prog_type(env->prog); in may_update_sockmap()
8412 verbose(env, "cannot update sockmap in this context\n"); in may_update_sockmap()
8416 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) in allow_tail_call_in_subprogs() argument
8418 return env->prog->jit_requested && in allow_tail_call_in_subprogs()
8422 static int check_map_func_compatibility(struct bpf_verifier_env *env, in check_map_func_compatibility() argument
8499 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
8509 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
8561 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { in check_map_func_compatibility()
8562 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in check_map_func_compatibility()
8670 verbose(env, "cannot pass map_type %d into func %s#%d\n", in check_map_func_compatibility()
8762 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) in clear_all_pkt_pointers() argument
8767 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in clear_all_pkt_pointers()
8769 mark_reg_invalid(env, reg); in clear_all_pkt_pointers()
8802 static int release_reference(struct bpf_verifier_env *env, in release_reference() argument
8809 err = release_reference_state(cur_func(env), ref_obj_id); in release_reference()
8813 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in release_reference()
8815 mark_reg_invalid(env, reg); in release_reference()
8821 static void invalidate_non_owning_refs(struct bpf_verifier_env *env) in invalidate_non_owning_refs() argument
8826 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ in invalidate_non_owning_refs()
8828 mark_reg_invalid(env, reg); in invalidate_non_owning_refs()
8832 static void clear_caller_saved_regs(struct bpf_verifier_env *env, in clear_caller_saved_regs() argument
8839 mark_reg_not_init(env, regs, caller_saved[i]); in clear_caller_saved_regs()
8840 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in clear_caller_saved_regs()
8844 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
8849 static int set_callee_state(struct bpf_verifier_env *env,
8853 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in __check_func_call() argument
8857 struct bpf_verifier_state *state = env->cur_state; in __check_func_call()
8862 verbose(env, "the call stack of %d frames is too deep\n", in __check_func_call()
8869 verbose(env, "verifier bug. Frame %d already allocated\n", in __check_func_call()
8874 err = btf_check_subprog_call(env, subprog, caller->regs); in __check_func_call()
8877 if (subprog_is_global(env, subprog)) { in __check_func_call()
8879 verbose(env, "Caller passes invalid args into func#%d\n", in __check_func_call()
8883 if (env->log.level & BPF_LOG_LEVEL) in __check_func_call()
8884 verbose(env, in __check_func_call()
8887 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
8890 mark_reg_unknown(env, caller->regs, BPF_REG_0); in __check_func_call()
8905 verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n", in __check_func_call()
8910 verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n", in __check_func_call()
8922 env->subprog_info[subprog].is_async_cb = true; in __check_func_call()
8923 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in __check_func_call()
8931 err = set_callee_state_cb(env, caller, callee, *insn_idx); in __check_func_call()
8935 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
8936 mark_reg_unknown(env, caller->regs, BPF_REG_0); in __check_func_call()
8951 init_func_state(env, callee, in __check_func_call()
8962 err = set_callee_state_cb(env, caller, callee, *insn_idx); in __check_func_call()
8966 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
8972 *insn_idx = env->subprog_info[subprog].start - 1; in __check_func_call()
8974 if (env->log.level & BPF_LOG_LEVEL) { in __check_func_call()
8975 verbose(env, "caller:\n"); in __check_func_call()
8976 print_verifier_state(env, caller, true); in __check_func_call()
8977 verbose(env, "callee:\n"); in __check_func_call()
8978 print_verifier_state(env, callee, true); in __check_func_call()
8988 int map_set_for_each_callback_args(struct bpf_verifier_env *env, in map_set_for_each_callback_args() argument
9011 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in map_set_for_each_callback_args()
9015 static int set_callee_state(struct bpf_verifier_env *env, in set_callee_state() argument
9029 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_func_call() argument
9035 subprog = find_subprog(env, target_insn); in check_func_call()
9037 verbose(env, "verifier bug. No program starts at insn %d\n", in check_func_call()
9042 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); in check_func_call()
9045 static int set_map_elem_callback_state(struct bpf_verifier_env *env, in set_map_elem_callback_state() argument
9050 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; in set_map_elem_callback_state()
9055 verbose(env, "tail_call abusing map_ptr\n"); in set_map_elem_callback_state()
9062 verbose(env, "callback function not allowed for map\n"); in set_map_elem_callback_state()
9066 err = map->ops->map_set_for_each_callback_args(env, caller, callee); in set_map_elem_callback_state()
9075 static int set_loop_callback_state(struct bpf_verifier_env *env, in set_loop_callback_state() argument
9088 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_loop_callback_state()
9089 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_loop_callback_state()
9090 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_loop_callback_state()
9097 static int set_timer_callback_state(struct bpf_verifier_env *env, in set_timer_callback_state() argument
9120 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_timer_callback_state()
9121 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_timer_callback_state()
9127 static int set_find_vma_callback_state(struct bpf_verifier_env *env, in set_find_vma_callback_state() argument
9148 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_find_vma_callback_state()
9149 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_find_vma_callback_state()
9155 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, in set_user_ringbuf_callback_state() argument
9164 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); in set_user_ringbuf_callback_state()
9165 mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); in set_user_ringbuf_callback_state()
9169 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_user_ringbuf_callback_state()
9170 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_user_ringbuf_callback_state()
9171 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_user_ringbuf_callback_state()
9178 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, in set_rbtree_add_callback_state() argument
9198 ref_set_non_owning(env, &callee->regs[BPF_REG_1]); in set_rbtree_add_callback_state()
9200 ref_set_non_owning(env, &callee->regs[BPF_REG_2]); in set_rbtree_add_callback_state()
9202 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_rbtree_add_callback_state()
9203 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_rbtree_add_callback_state()
9204 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_rbtree_add_callback_state()
9216 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) in in_rbtree_lock_required_cb() argument
9218 struct bpf_verifier_state *state = env->cur_state; in in_rbtree_lock_required_cb()
9219 struct bpf_insn *insn = env->prog->insnsi; in in_rbtree_lock_required_cb()
9235 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) in prepare_func_exit() argument
9237 struct bpf_verifier_state *state = env->cur_state; in prepare_func_exit()
9251 verbose(env, "cannot return stack pointer to the caller\n"); in prepare_func_exit()
9261 verbose(env, "R0 not a scalar value\n"); in prepare_func_exit()
9265 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); in prepare_func_exit()
9286 if (env->log.level & BPF_LOG_LEVEL) { in prepare_func_exit()
9287 verbose(env, "returning from callee:\n"); in prepare_func_exit()
9288 print_verifier_state(env, callee, true); in prepare_func_exit()
9289 verbose(env, "to caller at %d:\n", *insn_idx); in prepare_func_exit()
9290 print_verifier_state(env, caller, true); in prepare_func_exit()
9334 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_map() argument
9337 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map()
9353 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_map()
9366 verbose(env, "write into map forbidden\n"); in record_func_map()
9380 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_key() argument
9383 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_key()
9384 struct bpf_reg_state *regs = cur_regs(env), *reg; in record_func_key()
9392 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_key()
9405 err = mark_chain_precision(env, BPF_REG_3); in record_func_key()
9416 static int check_reference_leak(struct bpf_verifier_env *env) in check_reference_leak() argument
9418 struct bpf_func_state *state = cur_func(env); in check_reference_leak()
9428 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", in check_reference_leak()
9435 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, in check_bpf_snprintf_call() argument
9458 verbose(env, "verifier bug\n"); in check_bpf_snprintf_call()
9468 verbose(env, "Invalid format string\n"); in check_bpf_snprintf_call()
9473 static int check_get_func_ip(struct bpf_verifier_env *env) in check_get_func_ip() argument
9475 enum bpf_prog_type type = resolve_prog_type(env->prog); in check_get_func_ip()
9479 if (!bpf_prog_has_trampoline(env->prog)) { in check_get_func_ip()
9480 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", in check_get_func_ip()
9489 verbose(env, "func %s#%d not supported for program type %d\n", in check_get_func_ip()
9494 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) in cur_aux() argument
9496 return &env->insn_aux_data[env->insn_idx]; in cur_aux()
9499 static bool loop_flag_is_zero(struct bpf_verifier_env *env) in loop_flag_is_zero() argument
9501 struct bpf_reg_state *regs = cur_regs(env); in loop_flag_is_zero()
9506 mark_chain_precision(env, BPF_REG_4); in loop_flag_is_zero()
9511 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) in update_loop_inline_state() argument
9513 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; in update_loop_inline_state()
9517 state->fit_for_inline = loop_flag_is_zero(env); in update_loop_inline_state()
9525 state->fit_for_inline = (loop_flag_is_zero(env) && in update_loop_inline_state()
9529 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_helper_call() argument
9532 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_helper_call()
9545 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), in check_helper_call()
9550 if (env->ops->get_func_proto) in check_helper_call()
9551 fn = env->ops->get_func_proto(func_id, env->prog); in check_helper_call()
9553 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), in check_helper_call()
9559 if (!env->prog->gpl_compatible && fn->gpl_only) { in check_helper_call()
9560 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); in check_helper_call()
9564 if (fn->allowed && !fn->allowed(env->prog)) { in check_helper_call()
9565 verbose(env, "helper call is not allowed in probe\n"); in check_helper_call()
9569 if (!env->prog->aux->sleepable && fn->might_sleep) { in check_helper_call()
9570 verbose(env, "helper call might sleep in a non-sleepable prog\n"); in check_helper_call()
9577 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", in check_helper_call()
9587 verbose(env, "kernel subsystem misconfigured func %s#%d\n", in check_helper_call()
9592 if (env->cur_state->active_rcu_lock) { in check_helper_call()
9594 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", in check_helper_call()
9599 if (env->prog->aux->sleepable && is_storage_get_function(func_id)) in check_helper_call()
9600 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; in check_helper_call()
9606 err = check_func_arg(env, i, &meta, fn, insn_idx); in check_helper_call()
9611 err = record_func_map(env, &meta, func_id, insn_idx); in check_helper_call()
9615 err = record_func_key(env, &meta, func_id, insn_idx); in check_helper_call()
9623 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, in check_helper_call()
9629 regs = cur_regs(env); in check_helper_call()
9639 verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n"); in check_helper_call()
9642 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); in check_helper_call()
9644 err = release_reference(env, meta.ref_obj_id); in check_helper_call()
9652 verbose(env, "func %s#%d reference has not been acquired before\n", in check_helper_call()
9660 err = check_reference_leak(env); in check_helper_call()
9662 verbose(env, "tail_call would lead to reference leak\n"); in check_helper_call()
9671 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); in check_helper_call()
9676 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
9680 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
9684 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
9688 err = check_bpf_snprintf_call(env, regs); in check_helper_call()
9691 update_loop_inline_state(env, meta.subprogno); in check_helper_call()
9692 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
9697 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", in check_helper_call()
9698 reg_type_str(env, regs[BPF_REG_1].type)); in check_helper_call()
9704 env->prog->expected_attach_type == BPF_LSM_CGROUP) { in check_helper_call()
9705 if (!env->prog->aux->attach_func_proto->type) { in check_helper_call()
9709 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_helper_call()
9719 reg = get_dynptr_arg_reg(env, fn, regs); in check_helper_call()
9725 verbose(env, "verifier internal error: meta.dynptr_id already set\n"); in check_helper_call()
9729 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); in check_helper_call()
9733 id = dynptr_id(env, reg); in check_helper_call()
9735 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); in check_helper_call()
9739 ref_obj_id = dynptr_ref_obj_id(env, reg); in check_helper_call()
9741 verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n"); in check_helper_call()
9755 reg = get_dynptr_arg_reg(env, fn, regs); in check_helper_call()
9759 dynptr_type = dynptr_get_type(env, reg); in check_helper_call()
9772 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
9782 mark_reg_not_init(env, regs, caller_saved[i]); in check_helper_call()
9783 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_helper_call()
9796 mark_reg_unknown(env, regs, BPF_REG_0); in check_helper_call()
9803 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
9809 verbose(env, in check_helper_call()
9818 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
9822 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
9826 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
9830 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
9834 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
9842 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
9853 verbose(env, "unable to resolve the size of type '%s': %ld\n", in check_helper_call()
9878 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
9887 verbose(env, "verifier internal error:"); in check_helper_call()
9888 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", in check_helper_call()
9896 verbose(env, "invalid return type %u of func %s#%d\n", in check_helper_call()
9906 verbose(env, "unknown return type %u of func %s#%d\n", in check_helper_call()
9912 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
9915 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", in check_helper_call()
9927 int id = acquire_reference_state(env, insn_idx); in check_helper_call()
9939 err = check_map_func_compatibility(env, meta.map_ptr, func_id); in check_helper_call()
9945 !env->prog->has_callchain_buf) { in check_helper_call()
9956 verbose(env, err_str, func_id_name(func_id), func_id); in check_helper_call()
9960 env->prog->has_callchain_buf = true; in check_helper_call()
9964 env->prog->call_get_stack = true; in check_helper_call()
9967 if (check_get_func_ip(env)) in check_helper_call()
9969 env->prog->call_get_func_ip = true; in check_helper_call()
9973 clear_all_pkt_pointers(env); in check_helper_call()
9980 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, in mark_btf_func_reg_size() argument
9983 struct bpf_reg_state *reg = &cur_regs(env)[regno]; in mark_btf_func_reg_size()
9989 DEF_NOT_SUBREG : env->insn_idx + 1; in mark_btf_func_reg_size()
9993 mark_insn_zext(env, reg); in mark_btf_func_reg_size()
9994 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in mark_btf_func_reg_size()
9996 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); in mark_btf_func_reg_size()
10181 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, in is_kfunc_arg_callback() argument
10194 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, in __btf_type_is_scalar_struct() argument
10211 verbose(env, "max struct nesting depth exceeded\n"); in __btf_type_is_scalar_struct()
10214 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) in __btf_type_is_scalar_struct()
10333 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, in get_kfunc_ptr_arg_type() argument
10340 struct bpf_reg_state *regs = cur_regs(env); in get_kfunc_ptr_arg_type()
10352 if (btf_get_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) in get_kfunc_ptr_arg_type()
10381 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", in get_kfunc_ptr_arg_type()
10388 if (is_kfunc_arg_callback(env, meta->btf, &args[argno])) in get_kfunc_ptr_arg_type()
10402 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && in get_kfunc_ptr_arg_type()
10404 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", in get_kfunc_ptr_arg_type()
10411 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_btf_id() argument
10458 btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) in process_kf_arg_ptr_to_btf_id()
10465 …if (!btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type… in process_kf_arg_ptr_to_btf_id()
10466 …verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\… in process_kf_arg_ptr_to_btf_id()
10474 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in ref_set_non_owning() argument
10476 struct bpf_verifier_state *state = env->cur_state; in ref_set_non_owning()
10480 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); in ref_set_non_owning()
10485 verbose(env, "verifier internal error: NON_OWN_REF already set\n"); in ref_set_non_owning()
10496 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) in ref_convert_owning_non_owning() argument
10502 state = cur_func(env); in ref_convert_owning_non_owning()
10505 verbose(env, "verifier internal error: ref_obj_id is zero for " in ref_convert_owning_non_owning()
10517 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ in ref_convert_owning_non_owning()
10520 ref_set_non_owning(env, reg); in ref_convert_owning_non_owning()
10526 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); in ref_convert_owning_non_owning()
10574 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in check_reg_allocation_locked() argument
10587 verbose(env, "verifier internal error: unknown reg type for lock check\n"); in check_reg_allocation_locked()
10592 if (!env->cur_state->active_lock.ptr) in check_reg_allocation_locked()
10594 if (env->cur_state->active_lock.ptr != ptr || in check_reg_allocation_locked()
10595 env->cur_state->active_lock.id != id) { in check_reg_allocation_locked()
10596 verbose(env, "held lock and object are not in the same allocation\n"); in check_reg_allocation_locked()
10633 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, in check_kfunc_is_graph_root_api() argument
10647 verbose(env, "verifier internal error: unexpected graph root argument type %s\n", in check_kfunc_is_graph_root_api()
10653 verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", in check_kfunc_is_graph_root_api()
10658 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, in check_kfunc_is_graph_node_api() argument
10674 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", in check_kfunc_is_graph_node_api()
10680 verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", in check_kfunc_is_graph_node_api()
10686 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, in __process_kf_arg_ptr_to_graph_root() argument
10698 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); in __process_kf_arg_ptr_to_graph_root()
10702 if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) in __process_kf_arg_ptr_to_graph_root()
10707 verbose(env, in __process_kf_arg_ptr_to_graph_root()
10717 verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); in __process_kf_arg_ptr_to_graph_root()
10722 if (check_reg_allocation_locked(env, reg)) { in __process_kf_arg_ptr_to_graph_root()
10723 verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", in __process_kf_arg_ptr_to_graph_root()
10729 verbose(env, "verifier internal error: repeating %s arg\n", head_type_name); in __process_kf_arg_ptr_to_graph_root()
10736 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_list_head() argument
10740 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD, in process_kf_arg_ptr_to_list_head()
10744 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_rbtree_root() argument
10748 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT, in process_kf_arg_ptr_to_rbtree_root()
10753 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, in __process_kf_arg_ptr_to_graph_node() argument
10766 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); in __process_kf_arg_ptr_to_graph_node()
10770 if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) in __process_kf_arg_ptr_to_graph_node()
10775 verbose(env, in __process_kf_arg_ptr_to_graph_node()
10784 verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); in __process_kf_arg_ptr_to_graph_node()
10792 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, in __process_kf_arg_ptr_to_graph_node()
10794 verbose(env, "operation on %s expects arg#1 %s at offset=%d " in __process_kf_arg_ptr_to_graph_node()
10807 verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", in __process_kf_arg_ptr_to_graph_node()
10817 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_list_node() argument
10821 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, in process_kf_arg_ptr_to_list_node()
10826 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_rbtree_node() argument
10830 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, in process_kf_arg_ptr_to_rbtree_node()
10835 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, in check_kfunc_args() argument
10848 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, in check_kfunc_args()
10857 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; in check_kfunc_args()
10871 verbose(env, "R%d is not a scalar\n", regno); in check_kfunc_args()
10877 verbose(env, "verifier internal error: only one constant argument permitted\n"); in check_kfunc_args()
10881 verbose(env, "R%d must be a known constant\n", regno); in check_kfunc_args()
10884 ret = mark_chain_precision(env, regno); in check_kfunc_args()
10898 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); in check_kfunc_args()
10903 verbose(env, "R%d is not a const\n", regno); in check_kfunc_args()
10908 ret = mark_chain_precision(env, regno); in check_kfunc_args()
10916 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); in check_kfunc_args()
10922 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); in check_kfunc_args()
10928 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_kfunc_args()
10941 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); in check_kfunc_args()
10953 verbose(env, "R%d must be referenced or trusted\n", regno); in check_kfunc_args()
10957 verbose(env, "R%d must be a rcu pointer\n", regno); in check_kfunc_args()
10986 ret = check_func_arg_reg_off(env, reg, regno, arg_type); in check_kfunc_args()
10993 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t)); in check_kfunc_args()
10998 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); in check_kfunc_args()
11006 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
11010 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
11026 verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i); in check_kfunc_args()
11045 verbose(env, "verifier internal error: no dynptr type for parent of clone\n"); in check_kfunc_args()
11052 verbose(env, "verifier internal error: missing ref obj id for parent of clone\n"); in check_kfunc_args()
11057 ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id); in check_kfunc_args()
11062 int id = dynptr_id(env, reg); in check_kfunc_args()
11065 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); in check_kfunc_args()
11069 meta->initialized_dynptr.type = dynptr_get_type(env, reg); in check_kfunc_args()
11070 meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg); in check_kfunc_args()
11076 ret = process_iter_arg(env, regno, insn_idx, meta); in check_kfunc_args()
11083 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); in check_kfunc_args()
11087 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
11090 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); in check_kfunc_args()
11097 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); in check_kfunc_args()
11101 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
11104 ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta); in check_kfunc_args()
11110 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
11114 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
11117 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); in check_kfunc_args()
11124 verbose(env, "rbtree_remove node input must be non-owning ref\n"); in check_kfunc_args()
11127 if (in_rbtree_lock_required_cb(env)) { in check_kfunc_args()
11128 verbose(env, "rbtree_remove not allowed in rbtree cb\n"); in check_kfunc_args()
11133 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
11137 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
11142 ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta); in check_kfunc_args()
11151 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); in check_kfunc_args()
11152 verbose(env, "expected %s or socket\n", in check_kfunc_args()
11153 reg_type_str(env, base_type(reg->type) | in check_kfunc_args()
11157 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); in check_kfunc_args()
11164 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", in check_kfunc_args()
11168 ret = check_mem_reg(env, reg, regno, type_size); in check_kfunc_args()
11180 ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1); in check_kfunc_args()
11182 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); in check_kfunc_args()
11189 verbose(env, "verifier internal error: only one constant argument permitted\n"); in check_kfunc_args()
11193 verbose(env, "R%d must be a known constant\n", regno + 1); in check_kfunc_args()
11209 verbose(env, "arg#%d is neither owning or non-owning ref\n", i); in check_kfunc_args()
11217 verbose(env, "verifier internal error: Couldn't find btf_record\n"); in check_kfunc_args()
11222 verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i); in check_kfunc_args()
11233 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", in check_kfunc_args()
11241 static int fetch_kfunc_meta(struct bpf_verifier_env *env, in fetch_kfunc_meta() argument
11257 desc_btf = find_kfunc_desc_btf(env, insn->off); in fetch_kfunc_meta()
11268 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog); in fetch_kfunc_meta()
11283 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_kfunc_call() argument
11288 struct bpf_reg_state *regs = cur_regs(env); in check_kfunc_call()
11302 err = fetch_kfunc_meta(env, insn, &meta, &func_name); in check_kfunc_call()
11304 verbose(env, "calling kernel function %s is not allowed\n", func_name); in check_kfunc_call()
11308 insn_aux = &env->insn_aux_data[insn_idx]; in check_kfunc_call()
11313 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); in check_kfunc_call()
11318 if (sleepable && !env->prog->aux->sleepable) { in check_kfunc_call()
11319 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); in check_kfunc_call()
11326 if (env->cur_state->active_rcu_lock) { in check_kfunc_call()
11330 if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { in check_kfunc_call()
11331 verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); in check_kfunc_call()
11336 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); in check_kfunc_call()
11339 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in check_kfunc_call()
11345 env->cur_state->active_rcu_lock = false; in check_kfunc_call()
11347 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); in check_kfunc_call()
11351 env->cur_state->active_rcu_lock = true; in check_kfunc_call()
11353 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); in check_kfunc_call()
11358 err = check_kfunc_args(env, &meta, insn_idx); in check_kfunc_call()
11365 err = release_reference(env, regs[meta.release_regno].ref_obj_id); in check_kfunc_call()
11367 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
11379 err = ref_convert_owning_non_owning(env, release_ref_obj_id); in check_kfunc_call()
11381 verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", in check_kfunc_call()
11386 err = release_reference(env, release_ref_obj_id); in check_kfunc_call()
11388 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
11395 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_kfunc_call()
11398 verbose(env, "kfunc %s#%d failed callback verification\n", in check_kfunc_call()
11405 mark_reg_not_init(env, regs, caller_saved[i]); in check_kfunc_call()
11415 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); in check_kfunc_call()
11421 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
11422 mark_btf_func_reg_size(env, BPF_REG_0, t->size); in check_kfunc_call()
11435 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); in check_kfunc_call()
11439 ret_btf = env->prog->aux->btf; in check_kfunc_call()
11444 verbose(env, "bpf_obj_new requires prog BTF\n"); in check_kfunc_call()
11450 verbose(env, "bpf_obj_new type ID argument must be of a struct\n"); in check_kfunc_call()
11454 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
11463 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
11482 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
11489 verbose(env, in check_kfunc_call()
11494 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
11502 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
11505 verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n"); in check_kfunc_call()
11518 if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { in check_kfunc_call()
11519 verbose(env, "the prog does not allow writes to packet data\n"); in check_kfunc_call()
11525 verbose(env, "verifier internal error: no dynptr id\n"); in check_kfunc_call()
11535 verbose(env, "kernel function %s unhandled dynamic return type\n", in check_kfunc_call()
11551 verbose(env, in check_kfunc_call()
11559 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
11570 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
11579 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
11581 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); in check_kfunc_call()
11583 int id = acquire_reference_state(env, insn_idx); in check_kfunc_call()
11591 ref_set_non_owning(env, ®s[BPF_REG_0]); in check_kfunc_call()
11595 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
11613 mark_btf_func_reg_size(env, regno, sizeof(void *)); in check_kfunc_call()
11616 mark_btf_func_reg_size(env, regno, t->size); in check_kfunc_call()
11620 err = process_iter_next_call(env, insn_idx, &meta); in check_kfunc_call()
11668 static bool check_reg_sane_offset(struct bpf_verifier_env *env, in check_reg_sane_offset() argument
11677 verbose(env, "math between %s pointer and %lld is not allowed\n", in check_reg_sane_offset()
11678 reg_type_str(env, type), val); in check_reg_sane_offset()
11683 verbose(env, "%s pointer offset %d is not allowed\n", in check_reg_sane_offset()
11684 reg_type_str(env, type), reg->off); in check_reg_sane_offset()
11689 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", in check_reg_sane_offset()
11690 reg_type_str(env, type)); in check_reg_sane_offset()
11695 verbose(env, "value %lld makes %s pointer be out of bounds\n", in check_reg_sane_offset()
11696 smin, reg_type_str(env, type)); in check_reg_sane_offset()
11742 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, in can_skip_alu_sanitation() argument
11745 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; in can_skip_alu_sanitation()
11765 static int sanitize_val_alu(struct bpf_verifier_env *env, in sanitize_val_alu() argument
11768 struct bpf_insn_aux_data *aux = cur_aux(env); in sanitize_val_alu()
11770 if (can_skip_alu_sanitation(env, insn)) in sanitize_val_alu()
11787 sanitize_speculative_path(struct bpf_verifier_env *env, in sanitize_speculative_path() argument
11794 branch = push_stack(env, next_idx, curr_idx, true); in sanitize_speculative_path()
11798 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
11800 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
11801 mark_reg_unknown(env, regs, insn->src_reg); in sanitize_speculative_path()
11807 static int sanitize_ptr_alu(struct bpf_verifier_env *env, in sanitize_ptr_alu() argument
11815 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; in sanitize_ptr_alu()
11816 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_ptr_alu()
11826 if (can_skip_alu_sanitation(env, insn)) in sanitize_ptr_alu()
11865 env->explore_alu_limits = true; in sanitize_ptr_alu()
11896 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, in sanitize_ptr_alu()
11897 env->insn_idx); in sanitize_ptr_alu()
11903 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) in sanitize_mark_insn_seen() argument
11905 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_mark_insn_seen()
11913 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; in sanitize_mark_insn_seen()
11916 static int sanitize_err(struct bpf_verifier_env *env, in sanitize_err() argument
11927 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", in sanitize_err()
11931 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", in sanitize_err()
11935 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", in sanitize_err()
11939 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", in sanitize_err()
11943 verbose(env, "R%d could not be pushed for speculative verification, %s\n", in sanitize_err()
11947 verbose(env, "verifier internal error: unknown reason (%d)\n", in sanitize_err()
11966 struct bpf_verifier_env *env, in check_stack_access_for_ptr_arithmetic() argument
11975 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", in check_stack_access_for_ptr_arithmetic()
11981 verbose(env, "R%d stack pointer arithmetic goes out of range, " in check_stack_access_for_ptr_arithmetic()
11989 static int sanitize_check_bounds(struct bpf_verifier_env *env, in sanitize_check_bounds() argument
11998 if (env->bypass_spec_v1) in sanitize_check_bounds()
12003 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
12008 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
12009 verbose(env, "R%d pointer arithmetic of map value goes out of range, " in sanitize_check_bounds()
12026 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, in adjust_ptr_min_max_vals() argument
12031 struct bpf_verifier_state *vstate = env->cur_state; in adjust_ptr_min_max_vals()
12051 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
12057 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_ptr_min_max_vals()
12058 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
12062 verbose(env, in adjust_ptr_min_max_vals()
12069 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", in adjust_ptr_min_max_vals()
12070 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
12085 verbose(env, "R%d pointer arithmetic on %s prohibited\n", in adjust_ptr_min_max_vals()
12086 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
12098 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || in adjust_ptr_min_max_vals()
12099 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
12106 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
12109 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
12158 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
12166 verbose(env, "R%d tried to subtract pointer from scalar\n", in adjust_ptr_min_max_vals()
12175 verbose(env, "R%d subtraction from stack pointer prohibited\n", in adjust_ptr_min_max_vals()
12217 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
12227 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", in adjust_ptr_min_max_vals()
12232 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", in adjust_ptr_min_max_vals()
12237 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
12240 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
12243 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
12246 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
12823 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, in adjust_scalar_min_max_vals() argument
12828 struct bpf_reg_state *regs = cur_regs(env); in adjust_scalar_min_max_vals()
12857 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
12868 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
12875 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
12880 ret = sanitize_val_alu(env, insn); in adjust_scalar_min_max_vals()
12882 return sanitize_err(env, insn, ret, NULL, NULL); in adjust_scalar_min_max_vals()
12935 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12948 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12961 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12970 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12984 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, in adjust_reg_min_max_vals() argument
12987 struct bpf_verifier_state *vstate = env->cur_state; in adjust_reg_min_max_vals()
13011 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_reg_min_max_vals()
13012 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
13015 verbose(env, "R%d pointer %s pointer prohibited\n", in adjust_reg_min_max_vals()
13024 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
13027 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
13032 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
13035 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
13039 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
13051 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
13057 print_verifier_state(env, state, true); in adjust_reg_min_max_vals()
13058 verbose(env, "verifier internal error: unexpected ptr_reg\n"); in adjust_reg_min_max_vals()
13062 print_verifier_state(env, state, true); in adjust_reg_min_max_vals()
13063 verbose(env, "verifier internal error: no src_reg\n"); in adjust_reg_min_max_vals()
13066 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
13070 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_alu_op() argument
13072 struct bpf_reg_state *regs = cur_regs(env); in check_alu_op()
13081 verbose(env, "BPF_NEG uses reserved fields\n"); in check_alu_op()
13089 verbose(env, "BPF_END uses reserved fields\n"); in check_alu_op()
13095 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
13099 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
13100 verbose(env, "R%d pointer arithmetic prohibited\n", in check_alu_op()
13106 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
13114 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
13120 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
13126 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
13132 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
13137 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
13143 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
13163 src_reg->id = ++env->id_gen; in check_alu_op()
13169 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
13170 verbose(env, in check_alu_op()
13179 src_reg->id = ++env->id_gen; in check_alu_op()
13187 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
13192 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
13193 verbose(env, in check_alu_op()
13202 src_reg->id = ++env->id_gen; in check_alu_op()
13211 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
13217 src_reg->id = ++env->id_gen; in check_alu_op()
13222 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
13226 mark_reg_unknown(env, regs, in check_alu_op()
13237 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
13249 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); in check_alu_op()
13257 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
13261 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
13267 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
13273 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
13279 verbose(env, "div by zero\n"); in check_alu_op()
13288 verbose(env, "invalid shift %d\n", insn->imm); in check_alu_op()
13294 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
13298 return adjust_reg_min_max_vals(env, insn); in check_alu_op()
14032 static int check_cond_jmp_op(struct bpf_verifier_env *env, in check_cond_jmp_op() argument
14035 struct bpf_verifier_state *this_branch = env->cur_state; in check_cond_jmp_op()
14047 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); in check_cond_jmp_op()
14052 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
14059 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
14064 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_cond_jmp_op()
14070 is_pointer_value(env, insn->src_reg)) { in check_cond_jmp_op()
14071 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
14077 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
14121 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
14124 err = mark_chain_precision(env, insn->src_reg); in check_cond_jmp_op()
14134 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
14135 !sanitize_speculative_path(env, insn, *insn_idx + 1, in check_cond_jmp_op()
14145 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
14146 !sanitize_speculative_path(env, insn, in check_cond_jmp_op()
14153 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, in check_cond_jmp_op()
14265 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
14266 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
14270 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
14271 print_insn_state(env, this_branch->frame[this_branch->curframe]); in check_cond_jmp_op()
14276 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_imm() argument
14278 struct bpf_insn_aux_data *aux = cur_aux(env); in check_ld_imm()
14279 struct bpf_reg_state *regs = cur_regs(env); in check_ld_imm()
14285 verbose(env, "invalid BPF_LD_IMM insn\n"); in check_ld_imm()
14289 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); in check_ld_imm()
14293 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
14310 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
14323 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
14330 struct bpf_prog_aux *aux = env->prog->aux; in check_ld_imm()
14331 u32 subprogno = find_subprog(env, in check_ld_imm()
14332 env->insn_idx + insn->imm + 1); in check_ld_imm()
14335 verbose(env, "missing btf func_info\n"); in check_ld_imm()
14339 verbose(env, "callback function not static\n"); in check_ld_imm()
14348 map = env->used_maps[aux->map_index]; in check_ld_imm()
14361 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
14395 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_abs() argument
14397 struct bpf_reg_state *regs = cur_regs(env); in check_ld_abs()
14402 if (!may_access_skb(resolve_prog_type(env->prog))) { in check_ld_abs()
14403 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); in check_ld_abs()
14407 if (!env->ops->gen_ld_abs) { in check_ld_abs()
14408 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_abs()
14415 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); in check_ld_abs()
14420 err = check_reg_arg(env, ctx_reg, SRC_OP); in check_ld_abs()
14428 err = check_reference_leak(env); in check_ld_abs()
14430 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); in check_ld_abs()
14434 if (env->cur_state->active_lock.ptr) { in check_ld_abs()
14435 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); in check_ld_abs()
14439 if (env->cur_state->active_rcu_lock) { in check_ld_abs()
14440 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n"); in check_ld_abs()
14445 verbose(env, in check_ld_abs()
14452 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_ld_abs()
14457 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); in check_ld_abs()
14463 mark_reg_not_init(env, regs, caller_saved[i]); in check_ld_abs()
14464 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_ld_abs()
14471 mark_reg_unknown(env, regs, BPF_REG_0); in check_ld_abs()
14473 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; in check_ld_abs()
14477 static int check_return_code(struct bpf_verifier_env *env) in check_return_code() argument
14480 const struct bpf_prog *prog = env->prog; in check_return_code()
14483 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_return_code()
14485 struct bpf_func_state *frame = env->cur_state->frame[0]; in check_return_code()
14511 err = check_reg_arg(env, BPF_REG_0, SRC_OP); in check_return_code()
14515 if (is_pointer_value(env, BPF_REG_0)) { in check_return_code()
14516 verbose(env, "R0 leaks addr as return value\n"); in check_return_code()
14520 reg = cur_regs(env) + BPF_REG_0; in check_return_code()
14525 verbose(env, "In async callback the register R0 is not a known value (%s)\n", in check_return_code()
14526 reg_type_str(env, reg->type)); in check_return_code()
14531 verbose_invalid_scalar(env, reg, &const_0, "async callback", "R0"); in check_return_code()
14539 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", in check_return_code()
14540 reg_type_str(env, reg->type)); in check_return_code()
14548 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || in check_return_code()
14549 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || in check_return_code()
14550 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || in check_return_code()
14551 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || in check_return_code()
14552 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || in check_return_code()
14553 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) in check_return_code()
14555 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || in check_return_code()
14556 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) in check_return_code()
14560 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { in check_return_code()
14572 if (!env->prog->aux->attach_btf_id) in check_return_code()
14577 switch (env->prog->expected_attach_type) { in check_return_code()
14596 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { in check_return_code()
14602 if (!env->prog->aux->attach_func_proto->type) { in check_return_code()
14622 verbose(env, "At program exit the register R0 is not a known value (%s)\n", in check_return_code()
14623 reg_type_str(env, reg->type)); in check_return_code()
14628 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); in check_return_code()
14632 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_return_code()
14638 env->prog->enforce_expected_attach_type = 1; in check_return_code()
14682 static u32 state_htab_size(struct bpf_verifier_env *env) in state_htab_size() argument
14684 return env->prog->len; in state_htab_size()
14688 struct bpf_verifier_env *env, in explored_state() argument
14691 struct bpf_verifier_state *cur = env->cur_state; in explored_state()
14694 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; in explored_state()
14697 static void mark_prune_point(struct bpf_verifier_env *env, int idx) in mark_prune_point() argument
14699 env->insn_aux_data[idx].prune_point = true; in mark_prune_point()
14702 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx) in is_prune_point() argument
14704 return env->insn_aux_data[insn_idx].prune_point; in is_prune_point()
14707 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx) in mark_force_checkpoint() argument
14709 env->insn_aux_data[idx].force_checkpoint = true; in mark_force_checkpoint()
14712 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx) in is_force_checkpoint() argument
14714 return env->insn_aux_data[insn_idx].force_checkpoint; in is_force_checkpoint()
14728 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, in push_insn() argument
14731 int *insn_stack = env->cfg.insn_stack; in push_insn()
14732 int *insn_state = env->cfg.insn_state; in push_insn()
14740 if (w < 0 || w >= env->prog->len) { in push_insn()
14741 verbose_linfo(env, t, "%d: ", t); in push_insn()
14742 verbose(env, "jump out of range from insn %d to %d\n", t, w); in push_insn()
14748 mark_prune_point(env, w); in push_insn()
14749 mark_jmp_point(env, w); in push_insn()
14756 if (env->cfg.cur_stack >= env->prog->len) in push_insn()
14758 insn_stack[env->cfg.cur_stack++] = w; in push_insn()
14761 if (loop_ok && env->bpf_capable) in push_insn()
14763 verbose_linfo(env, t, "%d: ", t); in push_insn()
14764 verbose_linfo(env, w, "%d: ", w); in push_insn()
14765 verbose(env, "back-edge from insn %d to %d\n", t, w); in push_insn()
14771 verbose(env, "insn state internal bug\n"); in push_insn()
14778 struct bpf_verifier_env *env, in visit_func_call_insn() argument
14783 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); in visit_func_call_insn()
14787 mark_prune_point(env, t + 1); in visit_func_call_insn()
14789 mark_jmp_point(env, t + 1); in visit_func_call_insn()
14792 mark_prune_point(env, t); in visit_func_call_insn()
14793 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, in visit_func_call_insn()
14808 static int visit_insn(int t, struct bpf_verifier_env *env) in visit_insn() argument
14810 struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; in visit_insn()
14814 return visit_func_call_insn(t, insns, env, true); in visit_insn()
14819 return push_insn(t, t + 1, FALLTHROUGH, env, false); in visit_insn()
14832 mark_prune_point(env, t); in visit_insn()
14836 ret = fetch_kfunc_meta(env, insn, &meta, NULL); in visit_insn()
14838 mark_prune_point(env, t); in visit_insn()
14850 mark_force_checkpoint(env, t); in visit_insn()
14853 return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); in visit_insn()
14865 ret = push_insn(t, t + off + 1, FALLTHROUGH, env, in visit_insn()
14870 mark_prune_point(env, t + off + 1); in visit_insn()
14871 mark_jmp_point(env, t + off + 1); in visit_insn()
14877 mark_prune_point(env, t); in visit_insn()
14879 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); in visit_insn()
14883 return push_insn(t, t + insn->off + 1, BRANCH, env, true); in visit_insn()
14890 static int check_cfg(struct bpf_verifier_env *env) in check_cfg() argument
14892 int insn_cnt = env->prog->len; in check_cfg()
14897 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
14901 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
14909 env->cfg.cur_stack = 1; in check_cfg()
14911 while (env->cfg.cur_stack > 0) { in check_cfg()
14912 int t = insn_stack[env->cfg.cur_stack - 1]; in check_cfg()
14914 ret = visit_insn(t, env); in check_cfg()
14918 env->cfg.cur_stack--; in check_cfg()
14924 verbose(env, "visit_insn internal bug\n"); in check_cfg()
14931 if (env->cfg.cur_stack < 0) { in check_cfg()
14932 verbose(env, "pop stack internal bug\n"); in check_cfg()
14939 verbose(env, "unreachable insn %d\n", i); in check_cfg()
14949 env->cfg.insn_state = env->cfg.insn_stack = NULL; in check_cfg()
14953 static int check_abnormal_return(struct bpf_verifier_env *env) in check_abnormal_return() argument
14957 for (i = 1; i < env->subprog_cnt; i++) { in check_abnormal_return()
14958 if (env->subprog_info[i].has_ld_abs) { in check_abnormal_return()
14959 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); in check_abnormal_return()
14962 if (env->subprog_info[i].has_tail_call) { in check_abnormal_return()
14963 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); in check_abnormal_return()
14974 static int check_btf_func(struct bpf_verifier_env *env, in check_btf_func() argument
14992 if (check_abnormal_return(env)) in check_btf_func()
14997 if (nfuncs != env->subprog_cnt) { in check_btf_func()
14998 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); in check_btf_func()
15006 verbose(env, "invalid func info rec size %u\n", urec_size); in check_btf_func()
15010 prog = env->prog; in check_btf_func()
15027 verbose(env, "nonzero tailing record in func info"); in check_btf_func()
15048 verbose(env, in check_btf_func()
15054 verbose(env, in check_btf_func()
15060 if (env->subprog_info[i].start != krecord[i].insn_off) { in check_btf_func()
15061 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); in check_btf_func()
15068 verbose(env, "invalid type id %d in func info", in check_btf_func()
15081 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { in check_btf_func()
15082 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); in check_btf_func()
15085 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { in check_btf_func()
15086 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); in check_btf_func()
15105 static void adjust_btf_func(struct bpf_verifier_env *env) in adjust_btf_func() argument
15107 struct bpf_prog_aux *aux = env->prog->aux; in adjust_btf_func()
15113 for (i = 0; i < env->subprog_cnt; i++) in adjust_btf_func()
15114 aux->func_info[i].insn_off = env->subprog_info[i].start; in adjust_btf_func()
15120 static int check_btf_line(struct bpf_verifier_env *env, in check_btf_line() argument
15152 prog = env->prog; in check_btf_line()
15156 sub = env->subprog_info; in check_btf_line()
15164 verbose(env, "nonzero tailing record in line_info"); in check_btf_line()
15191 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", in check_btf_line()
15199 verbose(env, in check_btf_line()
15208 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); in check_btf_line()
15213 if (s != env->subprog_cnt) { in check_btf_line()
15218 verbose(env, "missing bpf_line_info for func#%u\n", s); in check_btf_line()
15228 if (s != env->subprog_cnt) { in check_btf_line()
15229 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", in check_btf_line()
15230 env->subprog_cnt - s, s); in check_btf_line()
15248 static int check_core_relo(struct bpf_verifier_env *env, in check_core_relo() argument
15254 struct bpf_prog *prog = env->prog; in check_core_relo()
15257 .log = &env->log, in check_core_relo()
15287 verbose(env, "nonzero tailing record in core_relo"); in check_core_relo()
15302 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", in check_core_relo()
15317 static int check_btf_info(struct bpf_verifier_env *env, in check_btf_info() argument
15325 if (check_abnormal_return(env)) in check_btf_info()
15337 env->prog->aux->btf = btf; in check_btf_info()
15339 err = check_btf_func(env, attr, uattr); in check_btf_info()
15343 err = check_btf_line(env, attr, uattr); in check_btf_info()
15347 err = check_core_relo(env, attr, uattr); in check_btf_info()
15419 static void clean_func_state(struct bpf_verifier_env *env, in clean_func_state() argument
15433 __mark_reg_not_init(env, &st->regs[i]); in clean_func_state()
15441 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); in clean_func_state()
15448 static void clean_verifier_state(struct bpf_verifier_env *env, in clean_verifier_state() argument
15458 clean_func_state(env, st->frame[i]); in clean_verifier_state()
15493 static void clean_live_states(struct bpf_verifier_env *env, int insn, in clean_live_states() argument
15499 sl = *explored_state(env, insn); in clean_live_states()
15509 clean_verifier_state(env, &sl->state); in clean_live_states()
15525 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, in regsafe() argument
15563 if (env->explore_alu_limits) { in regsafe()
15643 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, in stacksafe() argument
15666 if (env->allow_uninit_stack && in stacksafe()
15706 if (!regsafe(env, &old->stack[spi].spilled_ptr, in stacksafe()
15788 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, in func_states_equal() argument
15794 if (!regsafe(env, &old->regs[i], &cur->regs[i], in func_states_equal()
15795 &env->idmap_scratch)) in func_states_equal()
15798 if (!stacksafe(env, old, cur, &env->idmap_scratch)) in func_states_equal()
15801 if (!refsafe(old, cur, &env->idmap_scratch)) in func_states_equal()
15807 static bool states_equal(struct bpf_verifier_env *env, in states_equal() argument
15816 env->idmap_scratch.tmp_id_gen = env->id_gen; in states_equal()
15817 memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map)); in states_equal()
15835 !check_ids(old->active_lock.id, cur->active_lock.id, &env->idmap_scratch)) in states_equal()
15847 if (!func_states_equal(env, old->frame[i], cur->frame[i])) in states_equal()
15856 static int propagate_liveness_reg(struct bpf_verifier_env *env, in propagate_liveness_reg() argument
15875 err = mark_reg_read(env, reg, parent_reg, flag); in propagate_liveness_reg()
15889 static int propagate_liveness(struct bpf_verifier_env *env, in propagate_liveness() argument
15911 err = propagate_liveness_reg(env, &state_reg[i], in propagate_liveness()
15916 mark_insn_zext(env, &parent_reg[i]); in propagate_liveness()
15924 err = propagate_liveness_reg(env, state_reg, in propagate_liveness()
15936 static int propagate_precision(struct bpf_verifier_env *env, in propagate_precision() argument
15953 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
15955 verbose(env, "frame %d: propagating r%d", fr, i); in propagate_precision()
15957 verbose(env, ",r%d", i); in propagate_precision()
15959 bt_set_frame_reg(&env->bt, fr, i); in propagate_precision()
15971 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
15973 verbose(env, "frame %d: propagating fp%d", in propagate_precision()
15976 verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE); in propagate_precision()
15978 bt_set_frame_slot(&env->bt, fr, i); in propagate_precision()
15982 verbose(env, "\n"); in propagate_precision()
15985 err = mark_chain_precision_batch(env); in propagate_precision()
16010 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx) in is_iter_next_insn() argument
16012 return env->insn_aux_data[insn_idx].is_iter_next; in is_iter_next_insn()
16097 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) in is_state_visited() argument
16101 struct bpf_verifier_state *cur = env->cur_state, *new; in is_state_visited()
16103 bool force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx); in is_state_visited()
16114 if (env->jmps_processed - env->prev_jmps_processed >= 2 && in is_state_visited()
16115 env->insn_processed - env->prev_insn_processed >= 8) in is_state_visited()
16118 pprev = explored_state(env, insn_idx); in is_state_visited()
16121 clean_live_states(env, insn_idx, cur); in is_state_visited()
16158 if (is_iter_next_insn(env, insn_idx)) { in is_state_visited()
16159 if (states_equal(env, &sl->state, cur)) { in is_state_visited()
16174 iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr; in is_state_visited()
16182 states_equal(env, &sl->state, cur) && in is_state_visited()
16184 verbose_linfo(env, insn_idx, "; "); in is_state_visited()
16185 verbose(env, "infinite loop detected at insn %d\n", insn_idx); in is_state_visited()
16202 env->jmps_processed - env->prev_jmps_processed < 20 && in is_state_visited()
16203 env->insn_processed - env->prev_insn_processed < 100) in is_state_visited()
16207 if (states_equal(env, &sl->state, cur)) { in is_state_visited()
16220 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
16227 err = err ? : push_jmp_history(env, cur); in is_state_visited()
16228 err = err ? : propagate_precision(env, &sl->state); in is_state_visited()
16260 env->peak_states--; in is_state_visited()
16266 sl->next = env->free_list; in is_state_visited()
16267 env->free_list = sl; in is_state_visited()
16277 if (env->max_states_per_insn < states_cnt) in is_state_visited()
16278 env->max_states_per_insn = states_cnt; in is_state_visited()
16280 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) in is_state_visited()
16298 env->total_states++; in is_state_visited()
16299 env->peak_states++; in is_state_visited()
16300 env->prev_jmps_processed = env->jmps_processed; in is_state_visited()
16301 env->prev_insn_processed = env->insn_processed; in is_state_visited()
16304 if (env->bpf_capable) in is_state_visited()
16305 mark_all_scalars_imprecise(env, cur); in is_state_visited()
16322 new_sl->next = *explored_state(env, insn_idx); in is_state_visited()
16323 *explored_state(env, insn_idx) = new_sl; in is_state_visited()
16392 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, in save_aux_ptr_type() argument
16395 enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; in save_aux_ptr_type()
16422 verbose(env, "same insn cannot be used with different pointers\n"); in save_aux_ptr_type()
16430 static int do_check(struct bpf_verifier_env *env) in do_check() argument
16432 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check()
16433 struct bpf_verifier_state *state = env->cur_state; in do_check()
16434 struct bpf_insn *insns = env->prog->insnsi; in do_check()
16436 int insn_cnt = env->prog->len; in do_check()
16445 env->prev_insn_idx = prev_insn_idx; in do_check()
16446 if (env->insn_idx >= insn_cnt) { in do_check()
16447 verbose(env, "invalid insn idx %d insn_cnt %d\n", in do_check()
16448 env->insn_idx, insn_cnt); in do_check()
16452 insn = &insns[env->insn_idx]; in do_check()
16455 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { in do_check()
16456 verbose(env, in do_check()
16458 env->insn_processed); in do_check()
16462 state->last_insn_idx = env->prev_insn_idx; in do_check()
16464 if (is_prune_point(env, env->insn_idx)) { in do_check()
16465 err = is_state_visited(env, env->insn_idx); in do_check()
16470 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
16472 verbose(env, "\nfrom %d to %d%s: safe\n", in do_check()
16473 env->prev_insn_idx, env->insn_idx, in do_check()
16474 env->cur_state->speculative ? in do_check()
16477 verbose(env, "%d: safe\n", env->insn_idx); in do_check()
16483 if (is_jmp_point(env, env->insn_idx)) { in do_check()
16484 err = push_jmp_history(env, state); in do_check()
16495 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { in do_check()
16496 verbose(env, "\nfrom %d to %d%s:", in do_check()
16497 env->prev_insn_idx, env->insn_idx, in do_check()
16498 env->cur_state->speculative ? in do_check()
16500 print_verifier_state(env, state->frame[state->curframe], true); in do_check()
16504 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
16508 .private_data = env, in do_check()
16511 if (verifier_state_scratched(env)) in do_check()
16512 print_insn_state(env, state->frame[state->curframe]); in do_check()
16514 verbose_linfo(env, env->insn_idx, "; "); in do_check()
16515 env->prev_log_pos = env->log.end_pos; in do_check()
16516 verbose(env, "%d: ", env->insn_idx); in do_check()
16517 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in do_check()
16518 env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; in do_check()
16519 env->prev_log_pos = env->log.end_pos; in do_check()
16522 if (bpf_prog_is_offloaded(env->prog->aux)) { in do_check()
16523 err = bpf_prog_offload_verify_insn(env, env->insn_idx, in do_check()
16524 env->prev_insn_idx); in do_check()
16529 regs = cur_regs(env); in do_check()
16530 sanitize_mark_insn_seen(env); in do_check()
16531 prev_insn_idx = env->insn_idx; in do_check()
16534 err = check_alu_op(env, insn); in do_check()
16544 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
16548 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
16557 err = check_mem_access(env, env->insn_idx, insn->src_reg, in do_check()
16564 err = save_aux_ptr_type(env, src_reg_type, true); in do_check()
16571 err = check_atomic(env, env->insn_idx, insn); in do_check()
16574 env->insn_idx++; in do_check()
16579 verbose(env, "BPF_STX uses reserved fields\n"); in do_check()
16584 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
16588 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
16595 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
16601 err = save_aux_ptr_type(env, dst_reg_type, false); in do_check()
16609 verbose(env, "BPF_ST uses reserved fields\n"); in do_check()
16613 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
16620 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
16626 err = save_aux_ptr_type(env, dst_reg_type, false); in do_check()
16632 env->jmps_processed++; in do_check()
16642 verbose(env, "BPF_CALL uses reserved fields\n"); in do_check()
16646 if (env->cur_state->active_lock.ptr) { in do_check()
16651 verbose(env, "function calls are not allowed while holding a lock\n"); in do_check()
16656 err = check_func_call(env, insn, &env->insn_idx); in do_check()
16658 err = check_kfunc_call(env, insn, &env->insn_idx); in do_check()
16660 err = check_helper_call(env, insn, &env->insn_idx); in do_check()
16664 mark_reg_scratched(env, BPF_REG_0); in do_check()
16671 verbose(env, "BPF_JA uses reserved fields\n"); in do_check()
16676 env->insn_idx += insn->off + 1; in do_check()
16678 env->insn_idx += insn->imm + 1; in do_check()
16687 verbose(env, "BPF_EXIT uses reserved fields\n"); in do_check()
16691 if (env->cur_state->active_lock.ptr && in do_check()
16692 !in_rbtree_lock_required_cb(env)) { in do_check()
16693 verbose(env, "bpf_spin_unlock is missing\n"); in do_check()
16697 if (env->cur_state->active_rcu_lock && in do_check()
16698 !in_rbtree_lock_required_cb(env)) { in do_check()
16699 verbose(env, "bpf_rcu_read_unlock is missing\n"); in do_check()
16709 err = check_reference_leak(env); in do_check()
16715 err = prepare_func_exit(env, &env->insn_idx); in do_check()
16722 err = check_return_code(env); in do_check()
16726 mark_verifier_state_scratched(env); in do_check()
16727 update_branch_counts(env, env->cur_state); in do_check()
16728 err = pop_stack(env, &prev_insn_idx, in do_check()
16729 &env->insn_idx, pop_log); in do_check()
16739 err = check_cond_jmp_op(env, insn, &env->insn_idx); in do_check()
16747 err = check_ld_abs(env, insn); in do_check()
16752 err = check_ld_imm(env, insn); in do_check()
16756 env->insn_idx++; in do_check()
16757 sanitize_mark_insn_seen(env); in do_check()
16759 verbose(env, "invalid BPF_LD mode\n"); in do_check()
16763 verbose(env, "unknown insn class %d\n", class); in do_check()
16767 env->insn_idx++; in do_check()
16804 static int check_pseudo_btf_id(struct bpf_verifier_env *env, in check_pseudo_btf_id() argument
16824 verbose(env, "invalid module BTF object FD specified.\n"); in check_pseudo_btf_id()
16829 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"… in check_pseudo_btf_id()
16838 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); in check_pseudo_btf_id()
16844 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); in check_pseudo_btf_id()
16852 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", in check_pseudo_btf_id()
16892 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", in check_pseudo_btf_id()
16906 for (i = 0; i < env->used_btf_cnt; i++) { in check_pseudo_btf_id()
16907 if (env->used_btfs[i].btf == btf) { in check_pseudo_btf_id()
16913 if (env->used_btf_cnt >= MAX_USED_BTFS) { in check_pseudo_btf_id()
16918 btf_mod = &env->used_btfs[env->used_btf_cnt]; in check_pseudo_btf_id()
16931 env->used_btf_cnt++; in check_pseudo_btf_id()
16953 static int check_map_prog_compatibility(struct bpf_verifier_env *env, in check_map_prog_compatibility() argument
16963 verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); in check_map_prog_compatibility()
16970 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
16975 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
16982 verbose(env, "tracing progs cannot use bpf_timer yet\n"); in check_map_prog_compatibility()
16989 verbose(env, "offload device mismatch between prog and map\n"); in check_map_prog_compatibility()
16994 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); in check_map_prog_compatibility()
17016 verbose(env, in check_map_prog_compatibility()
17037 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) in resolve_pseudo_ldimm64() argument
17039 struct bpf_insn *insn = env->prog->insnsi; in resolve_pseudo_ldimm64()
17040 int insn_cnt = env->prog->len; in resolve_pseudo_ldimm64()
17043 err = bpf_prog_calc_tag(env->prog); in resolve_pseudo_ldimm64()
17051 verbose(env, "BPF_LDX uses reserved fields\n"); in resolve_pseudo_ldimm64()
17065 verbose(env, "invalid bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
17074 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
17075 err = check_pseudo_btf_id(env, insn, aux); in resolve_pseudo_ldimm64()
17082 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
17100 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
17107 if (bpfptr_is_null(env->fd_array)) { in resolve_pseudo_ldimm64()
17108 verbose(env, "fd_idx without fd_array is invalid\n"); in resolve_pseudo_ldimm64()
17111 if (copy_from_bpfptr_offset(&fd, env->fd_array, in resolve_pseudo_ldimm64()
17124 verbose(env, "fd %d is not pointing to valid bpf_map\n", in resolve_pseudo_ldimm64()
17129 err = check_map_prog_compatibility(env, map, env->prog); in resolve_pseudo_ldimm64()
17135 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
17143 verbose(env, "direct value offset of %u is not allowed\n", off); in resolve_pseudo_ldimm64()
17149 verbose(env, "no direct value access support for this map type\n"); in resolve_pseudo_ldimm64()
17156 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", in resolve_pseudo_ldimm64()
17170 for (j = 0; j < env->used_map_cnt; j++) { in resolve_pseudo_ldimm64()
17171 if (env->used_maps[j] == map) { in resolve_pseudo_ldimm64()
17178 if (env->used_map_cnt >= MAX_USED_MAPS) { in resolve_pseudo_ldimm64()
17190 aux->map_index = env->used_map_cnt; in resolve_pseudo_ldimm64()
17191 env->used_maps[env->used_map_cnt++] = map; in resolve_pseudo_ldimm64()
17194 bpf_cgroup_storage_assign(env->prog->aux, map)) { in resolve_pseudo_ldimm64()
17195 verbose(env, "only one cgroup storage of each type is allowed\n"); in resolve_pseudo_ldimm64()
17209 verbose(env, "unknown opcode %02x\n", insn->code); in resolve_pseudo_ldimm64()
17222 static void release_maps(struct bpf_verifier_env *env) in release_maps() argument
17224 __bpf_free_used_maps(env->prog->aux, env->used_maps, in release_maps()
17225 env->used_map_cnt); in release_maps()
17229 static void release_btfs(struct bpf_verifier_env *env) in release_btfs() argument
17231 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, in release_btfs()
17232 env->used_btf_cnt); in release_btfs()
17236 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) in convert_pseudo_ld_imm64() argument
17238 struct bpf_insn *insn = env->prog->insnsi; in convert_pseudo_ld_imm64()
17239 int insn_cnt = env->prog->len; in convert_pseudo_ld_imm64()
17255 static void adjust_insn_aux_data(struct bpf_verifier_env *env, in adjust_insn_aux_data() argument
17259 struct bpf_insn_aux_data *old_data = env->insn_aux_data; in adjust_insn_aux_data()
17269 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); in adjust_insn_aux_data()
17281 new_data[i].zext_dst = insn_has_def32(env, insn + i); in adjust_insn_aux_data()
17283 env->insn_aux_data = new_data; in adjust_insn_aux_data()
17287 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) in adjust_subprog_starts() argument
17294 for (i = 0; i <= env->subprog_cnt; i++) { in adjust_subprog_starts()
17295 if (env->subprog_info[i].start <= off) in adjust_subprog_starts()
17297 env->subprog_info[i].start += len - 1; in adjust_subprog_starts()
17315 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, in bpf_patch_insn_data() argument
17322 new_data = vzalloc(array_size(env->prog->len + len - 1, in bpf_patch_insn_data()
17328 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); in bpf_patch_insn_data()
17331 verbose(env, in bpf_patch_insn_data()
17333 env->insn_aux_data[off].orig_idx); in bpf_patch_insn_data()
17337 adjust_insn_aux_data(env, new_data, new_prog, off, len); in bpf_patch_insn_data()
17338 adjust_subprog_starts(env, off, len); in bpf_patch_insn_data()
17343 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, in adjust_subprog_starts_after_remove() argument
17349 for (i = 0; i < env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
17350 if (env->subprog_info[i].start >= off) in adjust_subprog_starts_after_remove()
17353 for (j = i; j < env->subprog_cnt; j++) in adjust_subprog_starts_after_remove()
17354 if (env->subprog_info[j].start >= off + cnt) in adjust_subprog_starts_after_remove()
17359 if (env->subprog_info[j].start != off + cnt) in adjust_subprog_starts_after_remove()
17363 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove()
17367 move = env->subprog_cnt + 1 - j; in adjust_subprog_starts_after_remove()
17369 memmove(env->subprog_info + i, in adjust_subprog_starts_after_remove()
17370 env->subprog_info + j, in adjust_subprog_starts_after_remove()
17371 sizeof(*env->subprog_info) * move); in adjust_subprog_starts_after_remove()
17372 env->subprog_cnt -= j - i; in adjust_subprog_starts_after_remove()
17388 if (env->subprog_info[i].start == off) in adjust_subprog_starts_after_remove()
17393 for (; i <= env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
17394 env->subprog_info[i].start -= cnt; in adjust_subprog_starts_after_remove()
17399 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, in bpf_adj_linfo_after_remove() argument
17402 struct bpf_prog *prog = env->prog; in bpf_adj_linfo_after_remove()
17449 for (i = 0; i <= env->subprog_cnt; i++) in bpf_adj_linfo_after_remove()
17450 if (env->subprog_info[i].linfo_idx > l_off) { in bpf_adj_linfo_after_remove()
17454 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) in bpf_adj_linfo_after_remove()
17455 env->subprog_info[i].linfo_idx -= l_cnt; in bpf_adj_linfo_after_remove()
17457 env->subprog_info[i].linfo_idx = l_off; in bpf_adj_linfo_after_remove()
17463 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) in verifier_remove_insns() argument
17465 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in verifier_remove_insns()
17466 unsigned int orig_prog_len = env->prog->len; in verifier_remove_insns()
17469 if (bpf_prog_is_offloaded(env->prog->aux)) in verifier_remove_insns()
17470 bpf_prog_offload_remove_insns(env, off, cnt); in verifier_remove_insns()
17472 err = bpf_remove_insns(env->prog, off, cnt); in verifier_remove_insns()
17476 err = adjust_subprog_starts_after_remove(env, off, cnt); in verifier_remove_insns()
17480 err = bpf_adj_linfo_after_remove(env, off, cnt); in verifier_remove_insns()
17501 static void sanitize_dead_code(struct bpf_verifier_env *env) in sanitize_dead_code() argument
17503 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in sanitize_dead_code()
17505 struct bpf_insn *insn = env->prog->insnsi; in sanitize_dead_code()
17506 const int insn_cnt = env->prog->len; in sanitize_dead_code()
17531 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) in opt_hard_wire_dead_code_branches() argument
17533 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_hard_wire_dead_code_branches()
17535 struct bpf_insn *insn = env->prog->insnsi; in opt_hard_wire_dead_code_branches()
17536 const int insn_cnt = env->prog->len; in opt_hard_wire_dead_code_branches()
17550 if (bpf_prog_is_offloaded(env->prog->aux)) in opt_hard_wire_dead_code_branches()
17551 bpf_prog_offload_replace_insn(env, i, &ja); in opt_hard_wire_dead_code_branches()
17557 static int opt_remove_dead_code(struct bpf_verifier_env *env) in opt_remove_dead_code() argument
17559 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_remove_dead_code()
17560 int insn_cnt = env->prog->len; in opt_remove_dead_code()
17572 err = verifier_remove_insns(env, i, j); in opt_remove_dead_code()
17575 insn_cnt = env->prog->len; in opt_remove_dead_code()
17581 static int opt_remove_nops(struct bpf_verifier_env *env) in opt_remove_nops() argument
17584 struct bpf_insn *insn = env->prog->insnsi; in opt_remove_nops()
17585 int insn_cnt = env->prog->len; in opt_remove_nops()
17592 err = verifier_remove_insns(env, i, 1); in opt_remove_nops()
17602 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, in opt_subreg_zext_lo32_rnd_hi32() argument
17606 struct bpf_insn_aux_data *aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
17607 int i, patch_len, delta = 0, len = env->prog->len; in opt_subreg_zext_lo32_rnd_hi32()
17608 struct bpf_insn *insns = env->prog->insnsi; in opt_subreg_zext_lo32_rnd_hi32()
17640 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { in opt_subreg_zext_lo32_rnd_hi32()
17679 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); in opt_subreg_zext_lo32_rnd_hi32()
17689 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); in opt_subreg_zext_lo32_rnd_hi32()
17692 env->prog = new_prog; in opt_subreg_zext_lo32_rnd_hi32()
17694 aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
17706 static int convert_ctx_accesses(struct bpf_verifier_env *env) in convert_ctx_accesses() argument
17708 const struct bpf_verifier_ops *ops = env->ops; in convert_ctx_accesses()
17710 const int insn_cnt = env->prog->len; in convert_ctx_accesses()
17717 if (ops->gen_prologue || env->seen_direct_write) { in convert_ctx_accesses()
17719 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
17722 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in convert_ctx_accesses()
17723 env->prog); in convert_ctx_accesses()
17725 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
17728 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
17732 env->prog = new_prog; in convert_ctx_accesses()
17737 if (bpf_prog_is_offloaded(env->prog->aux)) in convert_ctx_accesses()
17740 insn = env->prog->insnsi + delta; in convert_ctx_accesses()
17768 env->insn_aux_data[i + delta].sanitize_stack_spill) { in convert_ctx_accesses()
17775 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); in convert_ctx_accesses()
17780 env->prog = new_prog; in convert_ctx_accesses()
17785 switch ((int)env->insn_aux_data[i + delta].ptr_type) { in convert_ctx_accesses()
17817 env->prog->aux->num_exentries++; in convert_ctx_accesses()
17824 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; in convert_ctx_accesses()
17840 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); in convert_ctx_accesses()
17855 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in convert_ctx_accesses()
17859 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
17867 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); in convert_ctx_accesses()
17891 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in convert_ctx_accesses()
17898 env->prog = new_prog; in convert_ctx_accesses()
17905 static int jit_subprogs(struct bpf_verifier_env *env) in jit_subprogs() argument
17907 struct bpf_prog *prog = env->prog, **func, *tmp; in jit_subprogs()
17914 if (env->subprog_cnt <= 1) in jit_subprogs()
17925 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
17938 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
17954 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); in jit_subprogs()
17958 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
17960 subprog_end = env->subprog_info[i + 1].start; in jit_subprogs()
17996 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
18004 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
18014 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
18027 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
18054 func[i]->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
18056 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
18060 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); in jit_subprogs()
18071 for (i = 1; i < env->subprog_cnt; i++) { in jit_subprogs()
18082 insn[0].imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
18089 insn->off = env->insn_aux_data[i].call_imm; in jit_subprogs()
18090 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
18100 prog->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
18116 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
18131 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
18137 static int fixup_call_args(struct bpf_verifier_env *env) in fixup_call_args() argument
18140 struct bpf_prog *prog = env->prog; in fixup_call_args()
18147 if (env->prog->jit_requested && in fixup_call_args()
18148 !bpf_prog_is_offloaded(env->prog->aux)) { in fixup_call_args()
18149 err = jit_subprogs(env); in fixup_call_args()
18157 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); in fixup_call_args()
18160 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in fixup_call_args()
18164 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in fixup_call_args()
18172 verbose(env, "callbacks are not allowed in non-JITed programs\n"); in fixup_call_args()
18178 depth = get_callee_stack_depth(env, insn, i); in fixup_call_args()
18189 static void specialize_kfunc(struct bpf_verifier_env *env, in specialize_kfunc() argument
18192 struct bpf_prog *prog = env->prog; in specialize_kfunc()
18210 seen_direct_write = env->seen_direct_write; in specialize_kfunc()
18211 is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); in specialize_kfunc()
18219 env->seen_direct_write = seen_direct_write; in specialize_kfunc()
18240 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in fixup_kfunc_call() argument
18246 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); in fixup_kfunc_call()
18256 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); in fixup_kfunc_call()
18258 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", in fixup_kfunc_call()
18268 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
18270 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; in fixup_kfunc_call()
18279 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
18284 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
18296 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
18307 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
18312 __fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg, in fixup_kfunc_call()
18325 static int do_misc_fixups(struct bpf_verifier_env *env) in do_misc_fixups() argument
18327 struct bpf_prog *prog = env->prog; in do_misc_fixups()
18372 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in do_misc_fixups()
18377 env->prog = prog = new_prog; in do_misc_fixups()
18386 cnt = env->ops->gen_ld_abs(insn, insn_buf); in do_misc_fixups()
18388 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
18392 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18397 env->prog = prog = new_prog; in do_misc_fixups()
18411 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
18445 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18450 env->prog = prog = new_prog; in do_misc_fixups()
18460 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); in do_misc_fixups()
18466 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18471 env->prog = prog = new_prog; in do_misc_fixups()
18489 if (!allow_tail_call_in_subprogs(env)) in do_misc_fixups()
18501 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
18502 if (env->bpf_capable && !prog->blinding_requested && in do_misc_fixups()
18516 verbose(env, "adding tail call poke descriptor failed\n"); in do_misc_fixups()
18534 verbose(env, "tail_call abusing map_ptr\n"); in do_misc_fixups()
18547 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18552 env->prog = prog = new_prog; in do_misc_fixups()
18580 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18585 env->prog = prog = new_prog; in do_misc_fixups()
18591 if (!env->prog->aux->sleepable || in do_misc_fixups()
18592 env->insn_aux_data[i + delta].storage_get_func_atomic) in do_misc_fixups()
18599 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18604 env->prog = prog = new_prog; in do_misc_fixups()
18623 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
18635 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
18639 new_prog = bpf_patch_insn_data(env, i + delta, in do_misc_fixups()
18645 env->prog = prog = new_prog; in do_misc_fixups()
18722 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, in do_misc_fixups()
18728 env->prog = prog = new_prog; in do_misc_fixups()
18748 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18753 env->prog = prog = new_prog; in do_misc_fixups()
18776 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
18781 env->prog = prog = new_prog; in do_misc_fixups()
18792 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
18796 env->prog = prog = new_prog; in do_misc_fixups()
18807 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
18811 env->prog = prog = new_prog; in do_misc_fixups()
18817 fn = env->ops->get_func_proto(insn->imm, env->prog); in do_misc_fixups()
18822 verbose(env, in do_misc_fixups()
18836 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
18842 verbose(env, "tracking tail call prog failed\n"); in do_misc_fixups()
18847 sort_kfunc_descs_by_imm_off(env->prog); in do_misc_fixups()
18852 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, in inline_bpf_loop() argument
18913 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); in inline_bpf_loop()
18918 callback_start = env->subprog_info[callback_subprogno].start; in inline_bpf_loop()
18943 static int optimize_bpf_loop(struct bpf_verifier_env *env) in optimize_bpf_loop() argument
18945 struct bpf_subprog_info *subprogs = env->subprog_info; in optimize_bpf_loop()
18947 struct bpf_insn *insn = env->prog->insnsi; in optimize_bpf_loop()
18948 int insn_cnt = env->prog->len; in optimize_bpf_loop()
18955 &env->insn_aux_data[i + delta].loop_inline_state; in optimize_bpf_loop()
18961 new_prog = inline_bpf_loop(env, in optimize_bpf_loop()
18970 env->prog = new_prog; in optimize_bpf_loop()
18983 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in optimize_bpf_loop()
18988 static void free_states(struct bpf_verifier_env *env) in free_states() argument
18993 sl = env->free_list; in free_states()
19000 env->free_list = NULL; in free_states()
19002 if (!env->explored_states) in free_states()
19005 for (i = 0; i < state_htab_size(env); i++) { in free_states()
19006 sl = env->explored_states[i]; in free_states()
19014 env->explored_states[i] = NULL; in free_states()
19018 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
19020 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check_common()
19025 env->prev_linfo = NULL; in do_check_common()
19026 env->pass_cnt++; in do_check_common()
19039 env->cur_state = state; in do_check_common()
19040 init_func_state(env, state->frame[0], in do_check_common()
19044 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
19048 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
19049 ret = btf_prepare_func_args(env, subprog, regs); in do_check_common()
19054 mark_reg_known_zero(env, regs, i); in do_check_common()
19056 mark_reg_unknown(env, regs, i); in do_check_common()
19060 mark_reg_known_zero(env, regs, i); in do_check_common()
19062 regs[i].id = ++env->id_gen; in do_check_common()
19068 mark_reg_known_zero(env, regs, BPF_REG_1); in do_check_common()
19069 ret = btf_check_subprog_arg_match(env, subprog, regs); in do_check_common()
19083 ret = do_check(env); in do_check_common()
19088 if (env->cur_state) { in do_check_common()
19089 free_verifier_state(env->cur_state, true); in do_check_common()
19090 env->cur_state = NULL; in do_check_common()
19092 while (!pop_stack(env, NULL, NULL, false)); in do_check_common()
19094 bpf_vlog_reset(&env->log, 0); in do_check_common()
19095 free_states(env); in do_check_common()
19116 static int do_check_subprogs(struct bpf_verifier_env *env) in do_check_subprogs() argument
19118 struct bpf_prog_aux *aux = env->prog->aux; in do_check_subprogs()
19124 for (i = 1; i < env->subprog_cnt; i++) { in do_check_subprogs()
19127 env->insn_idx = env->subprog_info[i].start; in do_check_subprogs()
19128 WARN_ON_ONCE(env->insn_idx == 0); in do_check_subprogs()
19129 ret = do_check_common(env, i); in do_check_subprogs()
19132 } else if (env->log.level & BPF_LOG_LEVEL) { in do_check_subprogs()
19133 verbose(env, in do_check_subprogs()
19141 static int do_check_main(struct bpf_verifier_env *env) in do_check_main() argument
19145 env->insn_idx = 0; in do_check_main()
19146 ret = do_check_common(env, 0); in do_check_main()
19148 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check_main()
19153 static void print_verification_stats(struct bpf_verifier_env *env) in print_verification_stats() argument
19157 if (env->log.level & BPF_LOG_STATS) { in print_verification_stats()
19158 verbose(env, "verification time %lld usec\n", in print_verification_stats()
19159 div_u64(env->verification_time, 1000)); in print_verification_stats()
19160 verbose(env, "stack depth "); in print_verification_stats()
19161 for (i = 0; i < env->subprog_cnt; i++) { in print_verification_stats()
19162 u32 depth = env->subprog_info[i].stack_depth; in print_verification_stats()
19164 verbose(env, "%d", depth); in print_verification_stats()
19165 if (i + 1 < env->subprog_cnt) in print_verification_stats()
19166 verbose(env, "+"); in print_verification_stats()
19168 verbose(env, "\n"); in print_verification_stats()
19170 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " in print_verification_stats()
19172 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, in print_verification_stats()
19173 env->max_states_per_insn, env->total_states, in print_verification_stats()
19174 env->peak_states, env->longest_mark_read_walk); in print_verification_stats()
19177 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) in check_struct_ops_btf_id() argument
19182 struct bpf_prog *prog = env->prog; in check_struct_ops_btf_id()
19187 verbose(env, "struct ops programs must have a GPL compatible license\n"); in check_struct_ops_btf_id()
19194 verbose(env, "attach_btf_id %u is not a supported struct\n", in check_struct_ops_btf_id()
19202 verbose(env, "attach to invalid member idx %u of struct %s\n", in check_struct_ops_btf_id()
19212 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", in check_struct_ops_btf_id()
19221 verbose(env, "attach to unsupported member %s of struct %s\n", in check_struct_ops_btf_id()
19229 env->ops = st_ops->verifier_ops; in check_struct_ops_btf_id()
19567 static int check_attach_btf_id(struct bpf_verifier_env *env) in check_attach_btf_id() argument
19569 struct bpf_prog *prog = env->prog; in check_attach_btf_id()
19581 verbose(env, "Syscall programs can only be sleepable\n"); in check_attach_btf_id()
19586 …verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepa… in check_attach_btf_id()
19591 return check_struct_ops_btf_id(env); in check_attach_btf_id()
19598 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); in check_attach_btf_id()
19607 env->ops = bpf_verifier_ops[tgt_prog->type]; in check_attach_btf_id()
19631 ret = bpf_lsm_verify_prog(&env->log, prog); in check_attach_btf_id()
19662 struct bpf_verifier_env *env; in bpf_check() local
19674 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); in bpf_check()
19675 if (!env) in bpf_check()
19678 env->bt.env = env; in bpf_check()
19681 env->insn_aux_data = in bpf_check()
19684 if (!env->insn_aux_data) in bpf_check()
19687 env->insn_aux_data[i].orig_idx = i; in bpf_check()
19688 env->prog = *prog; in bpf_check()
19689 env->ops = bpf_verifier_ops[env->prog->type]; in bpf_check()
19690 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); in bpf_check()
19702 ret = bpf_vlog_init(&env->log, attr->log_level, in bpf_check()
19708 mark_verifier_state_clean(env); in bpf_check()
19712 verbose(env, "in-kernel BTF is malformed\n"); in bpf_check()
19717 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); in bpf_check()
19719 env->strict_alignment = true; in bpf_check()
19721 env->strict_alignment = false; in bpf_check()
19723 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); in bpf_check()
19724 env->allow_uninit_stack = bpf_allow_uninit_stack(); in bpf_check()
19725 env->bypass_spec_v1 = bpf_bypass_spec_v1(); in bpf_check()
19726 env->bypass_spec_v4 = bpf_bypass_spec_v4(); in bpf_check()
19727 env->bpf_capable = bpf_capable(); in bpf_check()
19730 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; in bpf_check()
19732 env->explored_states = kvcalloc(state_htab_size(env), in bpf_check()
19736 if (!env->explored_states) in bpf_check()
19739 ret = add_subprog_and_kfunc(env); in bpf_check()
19743 ret = check_subprogs(env); in bpf_check()
19747 ret = check_btf_info(env, attr, uattr); in bpf_check()
19751 ret = check_attach_btf_id(env); in bpf_check()
19755 ret = resolve_pseudo_ldimm64(env); in bpf_check()
19759 if (bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
19760 ret = bpf_prog_offload_verifier_prep(env->prog); in bpf_check()
19765 ret = check_cfg(env); in bpf_check()
19769 ret = do_check_subprogs(env); in bpf_check()
19770 ret = ret ?: do_check_main(env); in bpf_check()
19772 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) in bpf_check()
19773 ret = bpf_prog_offload_finalize(env); in bpf_check()
19776 kvfree(env->explored_states); in bpf_check()
19779 ret = check_max_stack_depth(env); in bpf_check()
19783 ret = optimize_bpf_loop(env); in bpf_check()
19787 opt_hard_wire_dead_code_branches(env); in bpf_check()
19789 ret = opt_remove_dead_code(env); in bpf_check()
19791 ret = opt_remove_nops(env); in bpf_check()
19794 sanitize_dead_code(env); in bpf_check()
19799 ret = convert_ctx_accesses(env); in bpf_check()
19802 ret = do_misc_fixups(env); in bpf_check()
19807 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
19808 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); in bpf_check()
19809 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret in bpf_check()
19814 ret = fixup_call_args(env); in bpf_check()
19816 env->verification_time = ktime_get_ns() - start_time; in bpf_check()
19817 print_verification_stats(env); in bpf_check()
19818 env->prog->aux->verified_insns = env->insn_processed; in bpf_check()
19821 err = bpf_vlog_finalize(&env->log, &log_true_size); in bpf_check()
19835 if (env->used_map_cnt) { in bpf_check()
19837 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, in bpf_check()
19838 sizeof(env->used_maps[0]), in bpf_check()
19841 if (!env->prog->aux->used_maps) { in bpf_check()
19846 memcpy(env->prog->aux->used_maps, env->used_maps, in bpf_check()
19847 sizeof(env->used_maps[0]) * env->used_map_cnt); in bpf_check()
19848 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
19850 if (env->used_btf_cnt) { in bpf_check()
19852 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, in bpf_check()
19853 sizeof(env->used_btfs[0]), in bpf_check()
19855 if (!env->prog->aux->used_btfs) { in bpf_check()
19860 memcpy(env->prog->aux->used_btfs, env->used_btfs, in bpf_check()
19861 sizeof(env->used_btfs[0]) * env->used_btf_cnt); in bpf_check()
19862 env->prog->aux->used_btf_cnt = env->used_btf_cnt; in bpf_check()
19864 if (env->used_map_cnt || env->used_btf_cnt) { in bpf_check()
19868 convert_pseudo_ld_imm64(env); in bpf_check()
19871 adjust_btf_func(env); in bpf_check()
19874 if (!env->prog->aux->used_maps) in bpf_check()
19878 release_maps(env); in bpf_check()
19879 if (!env->prog->aux->used_btfs) in bpf_check()
19880 release_btfs(env); in bpf_check()
19885 if (env->prog->type == BPF_PROG_TYPE_EXT) in bpf_check()
19886 env->prog->expected_attach_type = 0; in bpf_check()
19888 *prog = env->prog; in bpf_check()
19892 vfree(env->insn_aux_data); in bpf_check()
19894 kfree(env); in bpf_check()