Lines Matching refs:env
191 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
192 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
274 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) in find_linfo() argument
280 prog = env->prog; in find_linfo()
335 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, in bpf_verifier_log_write() argument
340 if (!bpf_verifier_log_needed(&env->log)) in bpf_verifier_log_write()
344 bpf_verifier_vlog(&env->log, fmt, args); in bpf_verifier_log_write()
351 struct bpf_verifier_env *env = private_data; in verbose() local
354 if (!bpf_verifier_log_needed(&env->log)) in verbose()
358 bpf_verifier_vlog(&env->log, fmt, args); in verbose()
384 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, in verbose_linfo() argument
390 if (!bpf_verifier_log_needed(&env->log)) in verbose_linfo()
393 linfo = find_linfo(env, insn_off); in verbose_linfo()
394 if (!linfo || linfo == env->prev_linfo) in verbose_linfo()
401 bpf_verifier_vlog(&env->log, prefix_fmt, args); in verbose_linfo()
405 verbose(env, "%s\n", in verbose_linfo()
406 ltrim(btf_name_by_offset(env->prog->aux->btf, in verbose_linfo()
409 env->prev_linfo = linfo; in verbose_linfo()
412 static void verbose_invalid_scalar(struct bpf_verifier_env *env, in verbose_invalid_scalar() argument
419 verbose(env, "At %s the register %s ", ctx, reg_name); in verbose_invalid_scalar()
422 verbose(env, "has value %s", tn_buf); in verbose_invalid_scalar()
424 verbose(env, "has unknown scalar value"); in verbose_invalid_scalar()
427 verbose(env, " should have been in %s\n", tn_buf); in verbose_invalid_scalar()
541 static const char *reg_type_str(struct bpf_verifier_env *env, in reg_type_str() argument
587 snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", in reg_type_str()
589 return env->type_str_buf; in reg_type_str()
600 static void print_liveness(struct bpf_verifier_env *env, in print_liveness() argument
604 verbose(env, "_"); in print_liveness()
606 verbose(env, "r"); in print_liveness()
608 verbose(env, "w"); in print_liveness()
610 verbose(env, "D"); in print_liveness()
632 static struct bpf_func_state *func(struct bpf_verifier_env *env, in func() argument
635 struct bpf_verifier_state *cur = env->cur_state; in func()
645 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) in mark_reg_scratched() argument
647 env->scratched_regs |= 1U << regno; in mark_reg_scratched()
650 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) in mark_stack_slot_scratched() argument
652 env->scratched_stack_slots |= 1ULL << spi; in mark_stack_slot_scratched()
655 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) in reg_scratched() argument
657 return (env->scratched_regs >> regno) & 1; in reg_scratched()
660 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) in stack_slot_scratched() argument
662 return (env->scratched_stack_slots >> regno) & 1; in stack_slot_scratched()
665 static bool verifier_state_scratched(const struct bpf_verifier_env *env) in verifier_state_scratched() argument
667 return env->scratched_regs || env->scratched_stack_slots; in verifier_state_scratched()
670 static void mark_verifier_state_clean(struct bpf_verifier_env *env) in mark_verifier_state_clean() argument
672 env->scratched_regs = 0U; in mark_verifier_state_clean()
673 env->scratched_stack_slots = 0ULL; in mark_verifier_state_clean()
677 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) in mark_verifier_state_scratched() argument
679 env->scratched_regs = ~0U; in mark_verifier_state_scratched()
680 env->scratched_stack_slots = ~0ULL; in mark_verifier_state_scratched()
700 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_stack_slots_dynptr() argument
703 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_dynptr()
727 id = acquire_reference_state(env, insn_idx); in mark_stack_slots_dynptr()
738 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in unmark_stack_slots_dynptr() argument
740 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_dynptr()
755 release_reference(env, state->stack[spi].spilled_ptr.id); in unmark_stack_slots_dynptr()
767 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_uninit() argument
769 struct bpf_func_state *state = func(env, reg); in is_dynptr_reg_valid_uninit()
785 bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, in is_dynptr_reg_valid_init() argument
788 struct bpf_func_state *state = func(env, reg); in is_dynptr_reg_valid_init()
805 bool is_dynptr_type_expected(struct bpf_verifier_env *env, in is_dynptr_type_expected() argument
809 struct bpf_func_state *state = func(env, reg); in is_dynptr_type_expected()
836 static void print_verifier_state(struct bpf_verifier_env *env, in print_verifier_state() argument
845 verbose(env, " frame%d:", state->frameno); in print_verifier_state()
851 if (!print_all && !reg_scratched(env, i)) in print_verifier_state()
853 verbose(env, " R%d", i); in print_verifier_state()
854 print_liveness(env, reg->live); in print_verifier_state()
855 verbose(env, "="); in print_verifier_state()
857 verbose(env, "P"); in print_verifier_state()
861 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); in print_verifier_state()
862 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
866 verbose(env, "%s", reg_type_str(env, t)); in print_verifier_state()
868 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); in print_verifier_state()
869 verbose(env, "("); in print_verifier_state()
874 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) in print_verifier_state()
928 verbose(env, ")"); in print_verifier_state()
945 if (!print_all && !stack_slot_scratched(env, i)) in print_verifier_state()
947 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
948 print_liveness(env, state->stack[i].spilled_ptr.live); in print_verifier_state()
952 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); in print_verifier_state()
954 verbose(env, "P"); in print_verifier_state()
956 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
958 verbose(env, "=%s", types_buf); in print_verifier_state()
962 verbose(env, " refs=%d", state->refs[0].id); in print_verifier_state()
965 verbose(env, ",%d", state->refs[i].id); in print_verifier_state()
968 verbose(env, " cb"); in print_verifier_state()
970 verbose(env, " async_cb"); in print_verifier_state()
971 verbose(env, "\n"); in print_verifier_state()
972 mark_verifier_state_clean(env); in print_verifier_state()
981 static void print_insn_state(struct bpf_verifier_env *env, in print_insn_state() argument
984 if (env->prev_log_len && env->prev_log_len == env->log.len_used) { in print_insn_state()
986 bpf_vlog_reset(&env->log, env->prev_log_len - 1); in print_insn_state()
987 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' '); in print_insn_state()
989 verbose(env, "%d:", env->insn_idx); in print_insn_state()
991 print_verifier_state(env, state, false); in print_insn_state()
1104 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) in acquire_reference_state() argument
1106 struct bpf_func_state *state = cur_func(env); in acquire_reference_state()
1113 id = ++env->id_gen; in acquire_reference_state()
1228 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in update_branch_counts() argument
1245 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, in pop_stack() argument
1248 struct bpf_verifier_state *cur = env->cur_state; in pop_stack()
1249 struct bpf_verifier_stack_elem *elem, *head = env->head; in pop_stack()
1252 if (env->head == NULL) in pop_stack()
1261 bpf_vlog_reset(&env->log, head->log_pos); in pop_stack()
1269 env->head = elem; in pop_stack()
1270 env->stack_size--; in pop_stack()
1274 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, in push_stack() argument
1278 struct bpf_verifier_state *cur = env->cur_state; in push_stack()
1288 elem->next = env->head; in push_stack()
1289 elem->log_pos = env->log.len_used; in push_stack()
1290 env->head = elem; in push_stack()
1291 env->stack_size++; in push_stack()
1296 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_stack()
1297 verbose(env, "The sequence of %d jumps is too complex.\n", in push_stack()
1298 env->stack_size); in push_stack()
1315 free_verifier_state(env->cur_state, true); in push_stack()
1316 env->cur_state = NULL; in push_stack()
1318 while (!pop_stack(env, NULL, NULL, false)); in push_stack()
1327 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1379 static void mark_reg_known_zero(struct bpf_verifier_env *env, in mark_reg_known_zero() argument
1383 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); in mark_reg_known_zero()
1386 __mark_reg_not_init(env, regs + regno); in mark_reg_known_zero()
1681 static void __mark_reg_unknown(const struct bpf_verifier_env *env, in __mark_reg_unknown() argument
1692 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; in __mark_reg_unknown()
1696 static void mark_reg_unknown(struct bpf_verifier_env *env, in mark_reg_unknown() argument
1700 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); in mark_reg_unknown()
1703 __mark_reg_not_init(env, regs + regno); in mark_reg_unknown()
1706 __mark_reg_unknown(env, regs + regno); in mark_reg_unknown()
1709 static void __mark_reg_not_init(const struct bpf_verifier_env *env, in __mark_reg_not_init() argument
1712 __mark_reg_unknown(env, reg); in __mark_reg_not_init()
1716 static void mark_reg_not_init(struct bpf_verifier_env *env, in mark_reg_not_init() argument
1720 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); in mark_reg_not_init()
1723 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
1726 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
1729 static void mark_btf_ld_reg(struct bpf_verifier_env *env, in mark_btf_ld_reg() argument
1736 mark_reg_unknown(env, regs, regno); in mark_btf_ld_reg()
1739 mark_reg_known_zero(env, regs, regno); in mark_btf_ld_reg()
1746 static void init_reg_state(struct bpf_verifier_env *env, in init_reg_state() argument
1753 mark_reg_not_init(env, regs, i); in init_reg_state()
1761 mark_reg_known_zero(env, regs, BPF_REG_FP); in init_reg_state()
1766 static void init_func_state(struct bpf_verifier_env *env, in init_func_state() argument
1774 init_reg_state(env, state); in init_func_state()
1775 mark_verifier_state_scratched(env); in init_func_state()
1779 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, in push_async_cb() argument
1792 elem->next = env->head; in push_async_cb()
1793 elem->log_pos = env->log.len_used; in push_async_cb()
1794 env->head = elem; in push_async_cb()
1795 env->stack_size++; in push_async_cb()
1796 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_async_cb()
1797 verbose(env, in push_async_cb()
1799 env->stack_size); in push_async_cb()
1811 init_func_state(env, frame, in push_async_cb()
1818 free_verifier_state(env->cur_state, true); in push_async_cb()
1819 env->cur_state = NULL; in push_async_cb()
1821 while (!pop_stack(env, NULL, NULL, false)); in push_async_cb()
1838 static int find_subprog(struct bpf_verifier_env *env, int off) in find_subprog() argument
1842 p = bsearch(&off, env->subprog_info, env->subprog_cnt, in find_subprog()
1843 sizeof(env->subprog_info[0]), cmp_subprogs); in find_subprog()
1846 return p - env->subprog_info; in find_subprog()
1850 static int add_subprog(struct bpf_verifier_env *env, int off) in add_subprog() argument
1852 int insn_cnt = env->prog->len; in add_subprog()
1856 verbose(env, "call to invalid destination\n"); in add_subprog()
1859 ret = find_subprog(env, off); in add_subprog()
1862 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { in add_subprog()
1863 verbose(env, "too many subprograms\n"); in add_subprog()
1867 env->subprog_info[env->subprog_cnt++].start = off; in add_subprog()
1868 sort(env->subprog_info, env->subprog_cnt, in add_subprog()
1869 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); in add_subprog()
1870 return env->subprog_cnt - 1; in add_subprog()
1930 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, in __find_kfunc_desc_btf() argument
1940 tab = env->prog->aux->kfunc_btf_tab; in __find_kfunc_desc_btf()
1945 verbose(env, "too many different module BTFs\n"); in __find_kfunc_desc_btf()
1949 if (bpfptr_is_null(env->fd_array)) { in __find_kfunc_desc_btf()
1950 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); in __find_kfunc_desc_btf()
1954 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, in __find_kfunc_desc_btf()
1961 verbose(env, "invalid module BTF fd specified\n"); in __find_kfunc_desc_btf()
1966 verbose(env, "BTF fd for kfunc is not a module BTF\n"); in __find_kfunc_desc_btf()
2000 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) in find_kfunc_desc_btf() argument
2007 verbose(env, "negative offset disallowed for kernel module function call\n"); in find_kfunc_desc_btf()
2011 return __find_kfunc_desc_btf(env, offset); in find_kfunc_desc_btf()
2016 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) in add_kfunc_call() argument
2029 prog_aux = env->prog->aux; in add_kfunc_call()
2034 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); in add_kfunc_call()
2038 if (!env->prog->jit_requested) { in add_kfunc_call()
2039 verbose(env, "JIT is required for calling kernel function\n"); in add_kfunc_call()
2044 verbose(env, "JIT does not support calling kernel function\n"); in add_kfunc_call()
2048 if (!env->prog->gpl_compatible) { in add_kfunc_call()
2049 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); in add_kfunc_call()
2075 desc_btf = find_kfunc_desc_btf(env, offset); in add_kfunc_call()
2077 verbose(env, "failed to find BTF for kernel function\n"); in add_kfunc_call()
2081 if (find_kfunc_desc(env->prog, func_id, offset)) in add_kfunc_call()
2085 verbose(env, "too many different kernel function calls\n"); in add_kfunc_call()
2091 verbose(env, "kernel btf_id %u is not a function\n", in add_kfunc_call()
2097 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", in add_kfunc_call()
2105 verbose(env, "cannot find address for kernel function %s\n", in add_kfunc_call()
2113 verbose(env, "address of kernel function %s is out of range\n", in add_kfunc_call()
2122 err = btf_distill_func_proto(&env->log, desc_btf, in add_kfunc_call()
2177 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) in add_subprog_and_kfunc() argument
2179 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc()
2180 struct bpf_insn *insn = env->prog->insnsi; in add_subprog_and_kfunc()
2181 int i, ret, insn_cnt = env->prog->len; in add_subprog_and_kfunc()
2184 ret = add_subprog(env, 0); in add_subprog_and_kfunc()
2193 if (!env->bpf_capable) { in add_subprog_and_kfunc()
2194 …verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_AD… in add_subprog_and_kfunc()
2199 ret = add_subprog(env, i + insn->imm + 1); in add_subprog_and_kfunc()
2201 ret = add_kfunc_call(env, insn->imm, insn->off); in add_subprog_and_kfunc()
2210 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
2212 if (env->log.level & BPF_LOG_LEVEL2) in add_subprog_and_kfunc()
2213 for (i = 0; i < env->subprog_cnt; i++) in add_subprog_and_kfunc()
2214 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
2219 static int check_subprogs(struct bpf_verifier_env *env) in check_subprogs() argument
2222 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs()
2223 struct bpf_insn *insn = env->prog->insnsi; in check_subprogs()
2224 int insn_cnt = env->prog->len; in check_subprogs()
2245 verbose(env, "jump out of range from insn %d to %d\n", i, off); in check_subprogs()
2256 verbose(env, "last insn is not an exit or jmp\n"); in check_subprogs()
2261 if (cur_subprog < env->subprog_cnt) in check_subprogs()
2271 static int mark_reg_read(struct bpf_verifier_env *env, in mark_reg_read() argument
2283 verbose(env, "verifier BUG type %s var_off %lld off %d\n", in mark_reg_read()
2284 reg_type_str(env, parent->type), in mark_reg_read()
2314 if (env->longest_mark_read_walk < cnt) in mark_reg_read()
2315 env->longest_mark_read_walk = cnt; in mark_reg_read()
2323 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, in is_reg64() argument
2431 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) in insn_has_def32() argument
2438 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
2441 static void mark_insn_zext(struct bpf_verifier_env *env, in mark_insn_zext() argument
2449 env->insn_aux_data[def_idx - 1].zext_dst = true; in mark_insn_zext()
2454 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, in check_reg_arg() argument
2457 struct bpf_verifier_state *vstate = env->cur_state; in check_reg_arg()
2459 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; in check_reg_arg()
2464 verbose(env, "R%d is invalid\n", regno); in check_reg_arg()
2468 mark_reg_scratched(env, regno); in check_reg_arg()
2471 rw64 = is_reg64(env, insn, regno, reg, t); in check_reg_arg()
2475 verbose(env, "R%d !read_ok\n", regno); in check_reg_arg()
2483 mark_insn_zext(env, reg); in check_reg_arg()
2485 return mark_reg_read(env, reg, reg->parent, in check_reg_arg()
2490 verbose(env, "frame pointer is read only\n"); in check_reg_arg()
2494 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; in check_reg_arg()
2496 mark_reg_unknown(env, regs, regno); in check_reg_arg()
2502 static int push_jmp_history(struct bpf_verifier_env *env, in push_jmp_history() argument
2512 p[cnt - 1].idx = env->insn_idx; in push_jmp_history()
2513 p[cnt - 1].prev_idx = env->prev_insn_idx; in push_jmp_history()
2556 static int backtrack_insn(struct bpf_verifier_env *env, int idx, in backtrack_insn() argument
2562 .private_data = env, in backtrack_insn()
2564 struct bpf_insn *insn = env->prog->insnsi + idx; in backtrack_insn()
2574 if (env->log.level & BPF_LOG_LEVEL2) { in backtrack_insn()
2575 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); in backtrack_insn()
2576 verbose(env, "%d: ", idx); in backtrack_insn()
2577 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in backtrack_insn()
2631 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
2648 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
2667 verbose(env, "BUG regs %x\n", *reg_mask); in backtrack_insn()
2741 static void mark_all_scalars_precise(struct bpf_verifier_env *env, in mark_all_scalars_precise() argument
2771 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, in __mark_chain_precision() argument
2774 struct bpf_verifier_state *st = env->cur_state; in __mark_chain_precision()
2776 int last_idx = env->insn_idx; in __mark_chain_precision()
2785 if (!env->bpf_capable) in __mark_chain_precision()
2828 if (env->log.level & BPF_LOG_LEVEL2) in __mark_chain_precision()
2829 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); in __mark_chain_precision()
2835 err = backtrack_insn(env, i, ®_mask, &stack_mask); in __mark_chain_precision()
2838 mark_all_scalars_precise(env, st); in __mark_chain_precision()
2852 if (i >= env->prog->len) { in __mark_chain_precision()
2859 verbose(env, "BUG backtracking idx %d\n", i); in __mark_chain_precision()
2898 mark_all_scalars_precise(env, st); in __mark_chain_precision()
2915 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
2916 verbose(env, "parent %s regs=%x stack=%llx marks:", in __mark_chain_precision()
2919 print_verifier_state(env, func, true); in __mark_chain_precision()
2933 int mark_chain_precision(struct bpf_verifier_env *env, int regno) in mark_chain_precision() argument
2935 return __mark_chain_precision(env, regno, -1); in mark_chain_precision()
2938 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) in mark_chain_precision_stack() argument
2940 return __mark_chain_precision(env, -1, spi); in mark_chain_precision_stack()
3024 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, in check_stack_write_fixed_off() argument
3032 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; in check_stack_write_fixed_off()
3041 if (!env->allow_ptr_leaks && in check_stack_write_fixed_off()
3044 verbose(env, "attempt to corrupt spilled pointer on stack\n"); in check_stack_write_fixed_off()
3048 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_fixed_off()
3051 if (!env->bypass_spec_v4) { in check_stack_write_fixed_off()
3062 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; in check_stack_write_fixed_off()
3065 mark_stack_slot_scratched(env, spi); in check_stack_write_fixed_off()
3067 !register_is_null(reg) && env->bpf_capable) { in check_stack_write_fixed_off()
3075 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
3083 verbose_linfo(env, insn_idx, "; "); in check_stack_write_fixed_off()
3084 verbose(env, "invalid size of register spill\n"); in check_stack_write_fixed_off()
3088 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); in check_stack_write_fixed_off()
3116 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
3149 static int check_stack_write_var_off(struct bpf_verifier_env *env, in check_stack_write_var_off() argument
3165 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_var_off()
3187 mark_stack_slot_scratched(env, spi); in check_stack_write_var_off()
3189 if (!env->allow_ptr_leaks in check_stack_write_var_off()
3198 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", in check_stack_write_var_off()
3220 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { in check_stack_write_var_off()
3221 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", in check_stack_write_var_off()
3229 err = mark_chain_precision(env, value_regno); in check_stack_write_var_off()
3244 static void mark_reg_stack_read(struct bpf_verifier_env *env, in mark_reg_stack_read() argument
3249 struct bpf_verifier_state *vstate = env->cur_state; in mark_reg_stack_read()
3281 mark_reg_unknown(env, state->regs, dst_regno); in mark_reg_stack_read()
3295 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, in check_stack_read_fixed_off() argument
3300 struct bpf_verifier_state *vstate = env->cur_state; in check_stack_read_fixed_off()
3317 verbose_linfo(env, env->insn_idx, "; "); in check_stack_read_fixed_off()
3318 verbose(env, "invalid size of register fill\n"); in check_stack_read_fixed_off()
3322 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
3341 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
3345 mark_reg_unknown(env, state->regs, dst_regno); in check_stack_read_fixed_off()
3359 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { in check_stack_read_fixed_off()
3366 verbose(env, "leaking pointer from stack off %d\n", in check_stack_read_fixed_off()
3370 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
3378 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
3382 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
3384 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); in check_stack_read_fixed_off()
3394 static int check_stack_range_initialized(struct bpf_verifier_env *env,
3400 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) in reg_state() argument
3402 return cur_regs(env) + regno; in reg_state()
3418 static int check_stack_read_var_off(struct bpf_verifier_env *env, in check_stack_read_var_off() argument
3422 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read_var_off()
3423 struct bpf_func_state *ptr_state = func(env, reg); in check_stack_read_var_off()
3429 err = check_stack_range_initialized(env, ptr_regno, off, size, in check_stack_read_var_off()
3436 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); in check_stack_read_var_off()
3449 static int check_stack_read(struct bpf_verifier_env *env, in check_stack_read() argument
3453 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read()
3454 struct bpf_func_state *state = func(env, reg); in check_stack_read()
3467 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=… in check_stack_read()
3475 if (!env->bypass_spec_v1 && var_off) { in check_stack_read()
3479 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", in check_stack_read()
3486 err = check_stack_read_fixed_off(env, state, off, size, in check_stack_read()
3493 err = check_stack_read_var_off(env, ptr_regno, off, size, in check_stack_read()
3510 static int check_stack_write(struct bpf_verifier_env *env, in check_stack_write() argument
3514 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_write()
3515 struct bpf_func_state *state = func(env, reg); in check_stack_write()
3520 err = check_stack_write_fixed_off(env, state, off, size, in check_stack_write()
3526 err = check_stack_write_var_off(env, state, in check_stack_write()
3533 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, in check_map_access_type() argument
3536 struct bpf_reg_state *regs = cur_regs(env); in check_map_access_type()
3541 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
3547 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
3556 static int __check_mem_access(struct bpf_verifier_env *env, int regno, in __check_mem_access() argument
3566 reg = &cur_regs(env)[regno]; in __check_mem_access()
3569 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", in __check_mem_access()
3573 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", in __check_mem_access()
3579 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", in __check_mem_access()
3584 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", in __check_mem_access()
3592 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, in check_mem_region_access() argument
3596 struct bpf_verifier_state *vstate = env->cur_state; in check_mem_region_access()
3615 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_mem_region_access()
3619 err = __check_mem_access(env, regno, reg->smin_value + off, size, in check_mem_region_access()
3622 verbose(env, "R%d min value is outside of the allowed memory range\n", in check_mem_region_access()
3632 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", in check_mem_region_access()
3636 err = __check_mem_access(env, regno, reg->umax_value + off, size, in check_mem_region_access()
3639 verbose(env, "R%d max value is outside of the allowed memory range\n", in check_mem_region_access()
3647 static int __check_ptr_off_reg(struct bpf_verifier_env *env, in __check_ptr_off_reg() argument
3656 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
3657 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
3662 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
3663 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
3671 verbose(env, "variable %s access var_off=%s disallowed\n", in __check_ptr_off_reg()
3672 reg_type_str(env, reg->type), tn_buf); in __check_ptr_off_reg()
3679 int check_ptr_off_reg(struct bpf_verifier_env *env, in check_ptr_off_reg() argument
3682 return __check_ptr_off_reg(env, reg, regno, false); in check_ptr_off_reg()
3685 static int map_kptr_match_type(struct bpf_verifier_env *env, in map_kptr_match_type() argument
3701 verbose(env, "R%d must point to kernel BTF\n", regno); in map_kptr_match_type()
3713 if (__check_ptr_off_reg(env, reg, regno, true)) in map_kptr_match_type()
3740 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in map_kptr_match_type()
3746 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, in map_kptr_match_type()
3747 reg_type_str(env, reg->type), reg_name); in map_kptr_match_type()
3748 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); in map_kptr_match_type()
3750 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), in map_kptr_match_type()
3753 verbose(env, "\n"); in map_kptr_match_type()
3757 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, in check_map_kptr_access() argument
3761 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_map_kptr_access()
3773 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); in check_map_kptr_access()
3781 verbose(env, "store to referenced kptr disallowed\n"); in check_map_kptr_access()
3786 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
3790 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, off_desc->kptr.btf, in check_map_kptr_access()
3793 val_reg->id = ++env->id_gen; in check_map_kptr_access()
3795 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
3797 map_kptr_match_type(env, off_desc, val_reg, value_regno)) in check_map_kptr_access()
3801 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", in check_map_kptr_access()
3806 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); in check_map_kptr_access()
3813 static int check_map_access(struct bpf_verifier_env *env, u32 regno, in check_map_access() argument
3817 struct bpf_verifier_state *vstate = env->cur_state; in check_map_access()
3823 err = check_mem_region_access(env, regno, off, size, map->value_size, in check_map_access()
3838 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); in check_map_access()
3847 verbose(env, "bpf_timer cannot be accessed directly by load/store\n"); in check_map_access()
3861 verbose(env, "kptr cannot be accessed indirectly by helper\n"); in check_map_access()
3865 verbose(env, "kptr access cannot have variable offset\n"); in check_map_access()
3869 verbose(env, "kptr access misaligned expected=%u off=%llu\n", in check_map_access()
3874 verbose(env, "kptr access size must be BPF_DW\n"); in check_map_access()
3886 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, in may_access_direct_pkt_data() argument
3890 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in may_access_direct_pkt_data()
3914 env->seen_direct_write = true; in may_access_direct_pkt_data()
3919 env->seen_direct_write = true; in may_access_direct_pkt_data()
3928 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, in check_packet_access() argument
3931 struct bpf_reg_state *regs = cur_regs(env); in check_packet_access()
3944 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_packet_access()
3950 __check_mem_access(env, regno, off, size, reg->range, in check_packet_access()
3953 verbose(env, "R%d offset is outside of the packet\n", regno); in check_packet_access()
3963 env->prog->aux->max_pkt_offset = in check_packet_access()
3964 max_t(u32, env->prog->aux->max_pkt_offset, in check_packet_access()
3971 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, in check_ctx_access() argument
3977 .log = &env->log, in check_ctx_access()
3980 if (env->ops->is_valid_access && in check_ctx_access()
3981 env->ops->is_valid_access(off, size, t, env->prog, &info)) { in check_ctx_access()
3995 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; in check_ctx_access()
3998 if (env->prog->aux->max_ctx_offset < off + size) in check_ctx_access()
3999 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
4003 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); in check_ctx_access()
4007 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, in check_flow_keys_access() argument
4012 verbose(env, "invalid access to flow keys off=%d size=%d\n", in check_flow_keys_access()
4019 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, in check_sock_access() argument
4023 struct bpf_reg_state *regs = cur_regs(env); in check_sock_access()
4029 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_sock_access()
4053 env->insn_aux_data[insn_idx].ctx_field_size = in check_sock_access()
4058 verbose(env, "R%d invalid %s access off=%d size=%d\n", in check_sock_access()
4059 regno, reg_type_str(env, reg->type), off, size); in check_sock_access()
4064 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) in is_pointer_value() argument
4066 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); in is_pointer_value()
4069 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) in is_ctx_reg() argument
4071 const struct bpf_reg_state *reg = reg_state(env, regno); in is_ctx_reg()
4076 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) in is_sk_reg() argument
4078 const struct bpf_reg_state *reg = reg_state(env, regno); in is_sk_reg()
4083 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) in is_pkt_reg() argument
4085 const struct bpf_reg_state *reg = reg_state(env, regno); in is_pkt_reg()
4090 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) in is_flow_key_reg() argument
4092 const struct bpf_reg_state *reg = reg_state(env, regno); in is_flow_key_reg()
4098 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, in check_pkt_ptr_alignment() argument
4124 verbose(env, in check_pkt_ptr_alignment()
4133 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, in check_generic_ptr_alignment() argument
4149 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", in check_generic_ptr_alignment()
4157 static int check_ptr_alignment(struct bpf_verifier_env *env, in check_ptr_alignment() argument
4161 bool strict = env->strict_alignment || strict_alignment_once; in check_ptr_alignment()
4170 return check_pkt_ptr_alignment(env, reg, off, size, strict); in check_ptr_alignment()
4206 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, in check_ptr_alignment()
4210 static int update_stack_depth(struct bpf_verifier_env *env, in update_stack_depth() argument
4214 u16 stack = env->subprog_info[func->subprogno].stack_depth; in update_stack_depth()
4220 env->subprog_info[func->subprogno].stack_depth = -off; in update_stack_depth()
4230 static int check_max_stack_depth(struct bpf_verifier_env *env) in check_max_stack_depth() argument
4233 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth()
4234 struct bpf_insn *insn = env->prog->insnsi; in check_max_stack_depth()
4261 verbose(env, in check_max_stack_depth()
4271 verbose(env, "combined stack size of %d calls is %d. Too large\n", in check_max_stack_depth()
4288 idx = find_subprog(env, next_insn); in check_max_stack_depth()
4296 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); in check_max_stack_depth()
4309 verbose(env, "the call stack of %d frames is too deep !\n", in check_max_stack_depth()
4324 env->prog->aux->tail_call_reachable = true; in check_max_stack_depth()
4339 static int get_callee_stack_depth(struct bpf_verifier_env *env, in get_callee_stack_depth() argument
4344 subprog = find_subprog(env, start); in get_callee_stack_depth()
4350 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
4354 static int __check_buffer_access(struct bpf_verifier_env *env, in __check_buffer_access() argument
4360 verbose(env, in __check_buffer_access()
4369 verbose(env, in __check_buffer_access()
4378 static int check_tp_buffer_access(struct bpf_verifier_env *env, in check_tp_buffer_access() argument
4384 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); in check_tp_buffer_access()
4388 if (off + size > env->prog->aux->max_tp_access) in check_tp_buffer_access()
4389 env->prog->aux->max_tp_access = off + size; in check_tp_buffer_access()
4394 static int check_buffer_access(struct bpf_verifier_env *env, in check_buffer_access() argument
4403 err = __check_buffer_access(env, buf_info, reg, regno, off, size); in check_buffer_access()
4501 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, in check_ptr_to_btf_access() argument
4515 verbose(env, in check_ptr_to_btf_access()
4524 verbose(env, in check_ptr_to_btf_access()
4531 verbose(env, in check_ptr_to_btf_access()
4538 verbose(env, in check_ptr_to_btf_access()
4544 if (env->ops->btf_struct_access) { in check_ptr_to_btf_access()
4545 ret = env->ops->btf_struct_access(&env->log, reg->btf, t, in check_ptr_to_btf_access()
4549 verbose(env, "only read is supported\n"); in check_ptr_to_btf_access()
4553 ret = btf_struct_access(&env->log, reg->btf, t, off, size, in check_ptr_to_btf_access()
4567 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); in check_ptr_to_btf_access()
4572 static int check_ptr_to_map_access(struct bpf_verifier_env *env, in check_ptr_to_map_access() argument
4587 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); in check_ptr_to_map_access()
4592 verbose(env, "map_ptr access not supported for map type %d\n", in check_ptr_to_map_access()
4600 if (!env->allow_ptr_to_map_access) { in check_ptr_to_map_access()
4601 verbose(env, in check_ptr_to_map_access()
4608 verbose(env, "R%d is %s invalid negative access: off=%d\n", in check_ptr_to_map_access()
4614 verbose(env, "only read from %s is supported\n", tname); in check_ptr_to_map_access()
4618 ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id, &flag); in check_ptr_to_map_access()
4623 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); in check_ptr_to_map_access()
4656 struct bpf_verifier_env *env, in check_stack_access_within_bounds() argument
4660 struct bpf_reg_state *regs = cur_regs(env); in check_stack_access_within_bounds()
4662 struct bpf_func_state *state = func(env, reg); in check_stack_access_within_bounds()
4684 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", in check_stack_access_within_bounds()
4701 verbose(env, "invalid%s stack R%d off=%d size=%d\n", in check_stack_access_within_bounds()
4707 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", in check_stack_access_within_bounds()
4720 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, in check_mem_access() argument
4724 struct bpf_reg_state *regs = cur_regs(env); in check_mem_access()
4734 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); in check_mem_access()
4743 verbose(env, "write to change key R%d not allowed\n", regno); in check_mem_access()
4747 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
4752 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4757 is_pointer_value(env, value_regno)) { in check_mem_access()
4758 verbose(env, "R%d leaks addr into map\n", value_regno); in check_mem_access()
4761 err = check_map_access_type(env, regno, off, size, t); in check_mem_access()
4764 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); in check_mem_access()
4771 err = check_map_kptr_access(env, regno, value_regno, insn_idx, in check_mem_access()
4791 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4798 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
4799 reg_type_str(env, reg->type)); in check_mem_access()
4804 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
4805 regno, reg_type_str(env, reg->type)); in check_mem_access()
4810 is_pointer_value(env, value_regno)) { in check_mem_access()
4811 verbose(env, "R%d leaks addr into mem\n", value_regno); in check_mem_access()
4815 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
4818 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4825 is_pointer_value(env, value_regno)) { in check_mem_access()
4826 verbose(env, "R%d leaks addr into ctx\n", value_regno); in check_mem_access()
4830 err = check_ptr_off_reg(env, reg, regno); in check_mem_access()
4834 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, in check_mem_access()
4837 verbose_linfo(env, insn_idx, "; "); in check_mem_access()
4844 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4846 mark_reg_known_zero(env, regs, in check_mem_access()
4849 regs[value_regno].id = ++env->id_gen; in check_mem_access()
4866 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); in check_mem_access()
4870 state = func(env, reg); in check_mem_access()
4871 err = update_stack_depth(env, state, off); in check_mem_access()
4876 err = check_stack_read(env, regno, off, size, in check_mem_access()
4879 err = check_stack_write(env, regno, off, size, in check_mem_access()
4882 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { in check_mem_access()
4883 verbose(env, "cannot write into packet\n"); in check_mem_access()
4887 is_pointer_value(env, value_regno)) { in check_mem_access()
4888 verbose(env, "R%d leaks addr into packet\n", in check_mem_access()
4892 err = check_packet_access(env, regno, off, size, false); in check_mem_access()
4894 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4897 is_pointer_value(env, value_regno)) { in check_mem_access()
4898 verbose(env, "R%d leaks addr into flow keys\n", in check_mem_access()
4903 err = check_flow_keys_access(env, off, size); in check_mem_access()
4905 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4908 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
4909 regno, reg_type_str(env, reg->type)); in check_mem_access()
4912 err = check_sock_access(env, insn_idx, regno, off, size, t); in check_mem_access()
4914 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4916 err = check_tp_buffer_access(env, reg, regno, off, size); in check_mem_access()
4918 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4921 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, in check_mem_access()
4924 err = check_ptr_to_map_access(env, regs, regno, off, size, t, in check_mem_access()
4932 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
4933 regno, reg_type_str(env, reg->type)); in check_mem_access()
4936 max_access = &env->prog->aux->max_rdonly_access; in check_mem_access()
4938 max_access = &env->prog->aux->max_rdwr_access; in check_mem_access()
4941 err = check_buffer_access(env, reg, regno, off, size, false, in check_mem_access()
4945 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
4947 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
4948 reg_type_str(env, reg->type)); in check_mem_access()
4960 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) in check_atomic() argument
4978 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); in check_atomic()
4983 verbose(env, "invalid atomic operand size\n"); in check_atomic()
4988 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_atomic()
4993 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
5001 err = check_reg_arg(env, aux_reg, SRC_OP); in check_atomic()
5005 if (is_pointer_value(env, aux_reg)) { in check_atomic()
5006 verbose(env, "R%d leaks addr into mem\n", aux_reg); in check_atomic()
5011 if (is_pointer_value(env, insn->src_reg)) { in check_atomic()
5012 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); in check_atomic()
5016 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
5017 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
5018 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
5019 is_sk_reg(env, insn->dst_reg)) { in check_atomic()
5020 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", in check_atomic()
5022 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
5033 err = check_reg_arg(env, load_reg, DST_OP); in check_atomic()
5046 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5049 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5056 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5074 struct bpf_verifier_env *env, int regno, int off, in check_stack_range_initialized() argument
5078 struct bpf_reg_state *reg = reg_state(env, regno); in check_stack_range_initialized()
5079 struct bpf_func_state *state = func(env, reg); in check_stack_range_initialized()
5089 verbose(env, "invalid zero-sized read\n"); in check_stack_range_initialized()
5103 err = check_stack_access_within_bounds(env, regno, off, access_size, in check_stack_range_initialized()
5117 if (!env->bypass_spec_v1) { in check_stack_range_initialized()
5121 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", in check_stack_range_initialized()
5168 env->allow_ptr_leaks)) { in check_stack_range_initialized()
5170 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); in check_stack_range_initialized()
5179 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", in check_stack_range_initialized()
5185 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", in check_stack_range_initialized()
5193 mark_reg_read(env, &state->stack[spi].spilled_ptr, in check_stack_range_initialized()
5197 return update_stack_depth(env, state, min_off); in check_stack_range_initialized()
5200 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, in check_helper_mem_access() argument
5204 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_helper_mem_access()
5210 return check_packet_access(env, regno, reg->off, access_size, in check_helper_mem_access()
5214 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
5215 reg_type_str(env, reg->type)); in check_helper_mem_access()
5218 return check_mem_region_access(env, regno, reg->off, access_size, in check_helper_mem_access()
5221 if (check_map_access_type(env, regno, reg->off, access_size, in check_helper_mem_access()
5225 return check_map_access(env, regno, reg->off, access_size, in check_helper_mem_access()
5230 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
5231 reg_type_str(env, reg->type)); in check_helper_mem_access()
5235 return check_mem_region_access(env, regno, reg->off, in check_helper_mem_access()
5241 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
5242 reg_type_str(env, reg->type)); in check_helper_mem_access()
5246 max_access = &env->prog->aux->max_rdonly_access; in check_helper_mem_access()
5248 max_access = &env->prog->aux->max_rdwr_access; in check_helper_mem_access()
5250 return check_buffer_access(env, reg, regno, reg->off, in check_helper_mem_access()
5255 env, in check_helper_mem_access()
5264 if (!env->ops->convert_ctx_access) { in check_helper_mem_access()
5272 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, in check_helper_mem_access()
5283 verbose(env, "R%d type=%s ", regno, in check_helper_mem_access()
5284 reg_type_str(env, reg->type)); in check_helper_mem_access()
5285 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); in check_helper_mem_access()
5290 static int check_mem_size_reg(struct bpf_verifier_env *env, in check_mem_size_reg() argument
5319 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", in check_mem_size_reg()
5325 err = check_helper_mem_access(env, regno - 1, 0, in check_mem_size_reg()
5333 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", in check_mem_size_reg()
5337 err = check_helper_mem_access(env, regno - 1, in check_mem_size_reg()
5341 err = mark_chain_precision(env, regno); in check_mem_size_reg()
5345 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_mem_reg() argument
5366 err = check_helper_mem_access(env, regno, mem_size, true, &meta); in check_mem_reg()
5369 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); in check_mem_reg()
5377 int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_kfunc_mem_size_reg() argument
5380 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; in check_kfunc_mem_size_reg()
5395 err = check_mem_size_reg(env, reg, regno, true, &meta); in check_kfunc_mem_size_reg()
5398 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); in check_kfunc_mem_size_reg()
5424 static int process_spin_lock(struct bpf_verifier_env *env, int regno, in process_spin_lock() argument
5427 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_spin_lock()
5428 struct bpf_verifier_state *cur = env->cur_state; in process_spin_lock()
5434 verbose(env, in process_spin_lock()
5440 verbose(env, in process_spin_lock()
5447 verbose(env, in process_spin_lock()
5451 verbose(env, in process_spin_lock()
5455 verbose(env, in process_spin_lock()
5461 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", in process_spin_lock()
5467 verbose(env, in process_spin_lock()
5474 verbose(env, "bpf_spin_unlock without taking a lock\n"); in process_spin_lock()
5478 verbose(env, "bpf_spin_unlock of different lock\n"); in process_spin_lock()
5486 static int process_timer_func(struct bpf_verifier_env *env, int regno, in process_timer_func() argument
5489 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_timer_func()
5495 verbose(env, in process_timer_func()
5501 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", in process_timer_func()
5507 verbose(env, in process_timer_func()
5511 verbose(env, in process_timer_func()
5515 verbose(env, in process_timer_func()
5521 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", in process_timer_func()
5526 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); in process_timer_func()
5534 static int process_kptr_func(struct bpf_verifier_env *env, int regno, in process_kptr_func() argument
5537 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_kptr_func()
5544 verbose(env, in process_kptr_func()
5550 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", in process_kptr_func()
5557 verbose(env, "map '%s' has more than %d kptr\n", map_ptr->name, in process_kptr_func()
5560 verbose(env, "map '%s' has repeating kptr BTF tags\n", map_ptr->name); in process_kptr_func()
5562 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); in process_kptr_func()
5570 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); in process_kptr_func()
5574 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); in process_kptr_func()
5607 static int resolve_map_arg_type(struct bpf_verifier_env *env, in resolve_map_arg_type() argument
5613 verbose(env, "invalid map_ptr to access map->type\n"); in resolve_map_arg_type()
5623 verbose(env, "invalid arg_type for sockmap/sockhash\n"); in resolve_map_arg_type()
5745 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, in check_reg_type() argument
5750 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_reg_type()
5757 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); in check_reg_type()
5785 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); in check_reg_type()
5787 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); in check_reg_type()
5788 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); in check_reg_type()
5802 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); in check_reg_type()
5809 if (map_kptr_match_type(env, meta->kptr_off_desc, reg, regno)) in check_reg_type()
5813 verbose(env, "verifier internal error:"); in check_reg_type()
5814 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", in check_reg_type()
5819 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in check_reg_type()
5822 verbose(env, "R%d is of type %s but %s is expected\n", in check_reg_type()
5833 int check_func_arg_reg_off(struct bpf_verifier_env *env, in check_func_arg_reg_off() argument
5844 verbose(env, "cannot pass in dynptr at an offset\n"); in check_func_arg_reg_off()
5873 verbose(env, "R%d must have zero offset when passed to release func\n", in check_func_arg_reg_off()
5886 return __check_ptr_off_reg(env, reg, regno, fixed_off_ok); in check_func_arg_reg_off()
5889 static u32 stack_slot_get_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in stack_slot_get_id() argument
5891 struct bpf_func_state *state = func(env, reg); in stack_slot_get_id()
5897 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, in check_func_arg() argument
5902 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_func_arg()
5911 err = check_reg_arg(env, regno, SRC_OP); in check_func_arg()
5916 if (is_pointer_value(env, regno)) { in check_func_arg()
5917 verbose(env, "R%d leaks addr into helper function\n", in check_func_arg()
5925 !may_access_direct_pkt_data(env, meta, BPF_READ)) { in check_func_arg()
5926 verbose(env, "helper access to the packet is not allowed\n"); in check_func_arg()
5931 err = resolve_map_arg_type(env, meta, &arg_type); in check_func_arg()
5946 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); in check_func_arg()
5950 err = check_func_arg_reg_off(env, reg, regno, arg_type); in check_func_arg()
5957 struct bpf_func_state *state = func(env, reg); in check_func_arg()
5962 verbose(env, "arg %d is an unacquired reference\n", regno); in check_func_arg()
5966 verbose(env, "R%d must be referenced when passed to release function\n", in check_func_arg()
5971 verbose(env, "verifier internal error: more than one release argument\n"); in check_func_arg()
5979 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_func_arg()
6005 verbose(env, in check_func_arg()
6025 verbose(env, "invalid map_ptr to access map->key\n"); in check_func_arg()
6028 err = check_helper_mem_access(env, regno, in check_func_arg()
6041 verbose(env, "invalid map_ptr to access map->value\n"); in check_func_arg()
6045 err = check_helper_mem_access(env, regno, in check_func_arg()
6051 verbose(env, "Helper has invalid btf_id in R%d\n", regno); in check_func_arg()
6059 if (process_spin_lock(env, regno, true)) in check_func_arg()
6062 if (process_spin_lock(env, regno, false)) in check_func_arg()
6065 verbose(env, "verifier internal error\n"); in check_func_arg()
6070 if (process_timer_func(env, regno, meta)) in check_func_arg()
6082 err = check_helper_mem_access(env, regno, in check_func_arg()
6088 err = check_mem_size_reg(env, reg, regno, false, meta); in check_func_arg()
6091 err = check_mem_size_reg(env, reg, regno, true, meta); in check_func_arg()
6102 if (!is_dynptr_reg_valid_uninit(env, reg)) { in check_func_arg()
6103 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); in check_func_arg()
6111 verbose(env, "verifier internal error: multiple uninitialized dynptr args\n"); in check_func_arg()
6116 } else if (!is_dynptr_reg_valid_init(env, reg)) { in check_func_arg()
6117 verbose(env, in check_func_arg()
6121 } else if (!is_dynptr_type_expected(env, reg, arg_type)) { in check_func_arg()
6135 verbose(env, in check_func_arg()
6143 verbose(env, "R%d is not a known constant'\n", in check_func_arg()
6148 err = mark_chain_precision(env, regno); in check_func_arg()
6157 err = check_helper_mem_access(env, regno, size, false, meta); in check_func_arg()
6160 err = check_ptr_alignment(env, reg, 0, size, true); in check_func_arg()
6171 verbose(env, "R%d does not point to a readonly map'\n", regno); in check_func_arg()
6176 verbose(env, "R%d is not a constant address'\n", regno); in check_func_arg()
6181 verbose(env, "no direct value access support for this map type\n"); in check_func_arg()
6185 err = check_map_access(env, regno, reg->off, in check_func_arg()
6194 verbose(env, "direct value access on string failed\n"); in check_func_arg()
6200 verbose(env, "string is not zero-terminated\n"); in check_func_arg()
6206 if (process_kptr_func(env, regno, meta)) in check_func_arg()
6214 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) in may_update_sockmap() argument
6216 enum bpf_attach_type eatype = env->prog->expected_attach_type; in may_update_sockmap()
6217 enum bpf_prog_type type = resolve_prog_type(env->prog); in may_update_sockmap()
6242 verbose(env, "cannot update sockmap in this context\n"); in may_update_sockmap()
6246 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) in allow_tail_call_in_subprogs() argument
6248 return env->prog->jit_requested && in allow_tail_call_in_subprogs()
6252 static int check_map_func_compatibility(struct bpf_verifier_env *env, in check_map_func_compatibility() argument
6329 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
6339 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
6382 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { in check_map_func_compatibility()
6383 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in check_map_func_compatibility()
6486 verbose(env, "cannot pass map_type %d into func %s#%d\n", in check_map_func_compatibility()
6574 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) in clear_all_pkt_pointers() argument
6579 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in clear_all_pkt_pointers()
6581 __mark_reg_unknown(env, reg); in clear_all_pkt_pointers()
6614 static int release_reference(struct bpf_verifier_env *env, in release_reference() argument
6621 err = release_reference_state(cur_func(env), ref_obj_id); in release_reference()
6625 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in release_reference()
6627 if (!env->allow_ptr_leaks) in release_reference()
6628 __mark_reg_not_init(env, reg); in release_reference()
6630 __mark_reg_unknown(env, reg); in release_reference()
6637 static void clear_caller_saved_regs(struct bpf_verifier_env *env, in clear_caller_saved_regs() argument
6644 mark_reg_not_init(env, regs, caller_saved[i]); in clear_caller_saved_regs()
6645 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in clear_caller_saved_regs()
6649 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
6654 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in __check_func_call() argument
6658 struct bpf_verifier_state *state = env->cur_state; in __check_func_call()
6665 verbose(env, "the call stack of %d frames is too deep\n", in __check_func_call()
6672 verbose(env, "verifier bug. Frame %d already allocated\n", in __check_func_call()
6677 func_info_aux = env->prog->aux->func_info_aux; in __check_func_call()
6680 err = btf_check_subprog_call(env, subprog, caller->regs); in __check_func_call()
6685 verbose(env, "Caller passes invalid args into func#%d\n", in __check_func_call()
6689 if (env->log.level & BPF_LOG_LEVEL) in __check_func_call()
6690 verbose(env, in __check_func_call()
6693 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
6696 mark_reg_unknown(env, caller->regs, BPF_REG_0); in __check_func_call()
6710 env->subprog_info[subprog].is_async_cb = true; in __check_func_call()
6711 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in __check_func_call()
6719 err = set_callee_state_cb(env, caller, callee, *insn_idx); in __check_func_call()
6723 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
6724 mark_reg_unknown(env, caller->regs, BPF_REG_0); in __check_func_call()
6739 init_func_state(env, callee, in __check_func_call()
6750 err = set_callee_state_cb(env, caller, callee, *insn_idx); in __check_func_call()
6754 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
6760 *insn_idx = env->subprog_info[subprog].start - 1; in __check_func_call()
6762 if (env->log.level & BPF_LOG_LEVEL) { in __check_func_call()
6763 verbose(env, "caller:\n"); in __check_func_call()
6764 print_verifier_state(env, caller, true); in __check_func_call()
6765 verbose(env, "callee:\n"); in __check_func_call()
6766 print_verifier_state(env, callee, true); in __check_func_call()
6776 int map_set_for_each_callback_args(struct bpf_verifier_env *env, in map_set_for_each_callback_args() argument
6799 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in map_set_for_each_callback_args()
6803 static int set_callee_state(struct bpf_verifier_env *env, in set_callee_state() argument
6817 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_func_call() argument
6823 subprog = find_subprog(env, target_insn); in check_func_call()
6825 verbose(env, "verifier bug. No program starts at insn %d\n", in check_func_call()
6830 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); in check_func_call()
6833 static int set_map_elem_callback_state(struct bpf_verifier_env *env, in set_map_elem_callback_state() argument
6838 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; in set_map_elem_callback_state()
6843 verbose(env, "tail_call abusing map_ptr\n"); in set_map_elem_callback_state()
6850 verbose(env, "callback function not allowed for map\n"); in set_map_elem_callback_state()
6854 err = map->ops->map_set_for_each_callback_args(env, caller, callee); in set_map_elem_callback_state()
6863 static int set_loop_callback_state(struct bpf_verifier_env *env, in set_loop_callback_state() argument
6876 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_loop_callback_state()
6877 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_loop_callback_state()
6878 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_loop_callback_state()
6885 static int set_timer_callback_state(struct bpf_verifier_env *env, in set_timer_callback_state() argument
6908 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_timer_callback_state()
6909 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_timer_callback_state()
6915 static int set_find_vma_callback_state(struct bpf_verifier_env *env, in set_find_vma_callback_state() argument
6936 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_find_vma_callback_state()
6937 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_find_vma_callback_state()
6943 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, in set_user_ringbuf_callback_state() argument
6952 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); in set_user_ringbuf_callback_state()
6958 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_user_ringbuf_callback_state()
6959 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_user_ringbuf_callback_state()
6960 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_user_ringbuf_callback_state()
6967 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) in prepare_func_exit() argument
6969 struct bpf_verifier_state *state = env->cur_state; in prepare_func_exit()
6983 verbose(env, "cannot return stack pointer to the caller\n"); in prepare_func_exit()
6993 verbose(env, "R0 not a scalar value\n"); in prepare_func_exit()
6997 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); in prepare_func_exit()
7018 if (env->log.level & BPF_LOG_LEVEL) { in prepare_func_exit()
7019 verbose(env, "returning from callee:\n"); in prepare_func_exit()
7020 print_verifier_state(env, callee, true); in prepare_func_exit()
7021 verbose(env, "to caller at %d:\n", *insn_idx); in prepare_func_exit()
7022 print_verifier_state(env, caller, true); in prepare_func_exit()
7052 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_map() argument
7055 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map()
7071 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_map()
7084 verbose(env, "write into map forbidden\n"); in record_func_map()
7098 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_key() argument
7101 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_key()
7102 struct bpf_reg_state *regs = cur_regs(env), *reg; in record_func_key()
7110 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_key()
7123 err = mark_chain_precision(env, BPF_REG_3); in record_func_key()
7134 static int check_reference_leak(struct bpf_verifier_env *env) in check_reference_leak() argument
7136 struct bpf_func_state *state = cur_func(env); in check_reference_leak()
7146 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", in check_reference_leak()
7153 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, in check_bpf_snprintf_call() argument
7175 verbose(env, "verifier bug\n"); in check_bpf_snprintf_call()
7185 verbose(env, "Invalid format string\n"); in check_bpf_snprintf_call()
7190 static int check_get_func_ip(struct bpf_verifier_env *env) in check_get_func_ip() argument
7192 enum bpf_prog_type type = resolve_prog_type(env->prog); in check_get_func_ip()
7196 if (!bpf_prog_has_trampoline(env->prog)) { in check_get_func_ip()
7197 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", in check_get_func_ip()
7206 verbose(env, "func %s#%d not supported for program type %d\n", in check_get_func_ip()
7211 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) in cur_aux() argument
7213 return &env->insn_aux_data[env->insn_idx]; in cur_aux()
7216 static bool loop_flag_is_zero(struct bpf_verifier_env *env) in loop_flag_is_zero() argument
7218 struct bpf_reg_state *regs = cur_regs(env); in loop_flag_is_zero()
7223 mark_chain_precision(env, BPF_REG_4); in loop_flag_is_zero()
7228 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) in update_loop_inline_state() argument
7230 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; in update_loop_inline_state()
7234 state->fit_for_inline = loop_flag_is_zero(env); in update_loop_inline_state()
7242 state->fit_for_inline = (loop_flag_is_zero(env) && in update_loop_inline_state()
7246 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_helper_call() argument
7249 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_helper_call()
7262 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), in check_helper_call()
7267 if (env->ops->get_func_proto) in check_helper_call()
7268 fn = env->ops->get_func_proto(func_id, env->prog); in check_helper_call()
7270 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), in check_helper_call()
7276 if (!env->prog->gpl_compatible && fn->gpl_only) { in check_helper_call()
7277 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); in check_helper_call()
7281 if (fn->allowed && !fn->allowed(env->prog)) { in check_helper_call()
7282 verbose(env, "helper call is not allowed in probe\n"); in check_helper_call()
7289 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", in check_helper_call()
7299 verbose(env, "kernel subsystem misconfigured func %s#%d\n", in check_helper_call()
7307 err = check_func_arg(env, i, &meta, fn); in check_helper_call()
7312 err = record_func_map(env, &meta, func_id, insn_idx); in check_helper_call()
7316 err = record_func_key(env, &meta, func_id, insn_idx); in check_helper_call()
7324 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, in check_helper_call()
7330 regs = cur_regs(env); in check_helper_call()
7335 err = check_mem_access(env, insn_idx, meta.uninit_dynptr_regno, in check_helper_call()
7341 err = mark_stack_slots_dynptr(env, ®s[meta.uninit_dynptr_regno], in check_helper_call()
7351 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); in check_helper_call()
7353 err = release_reference(env, meta.ref_obj_id); in check_helper_call()
7360 verbose(env, "func %s#%d reference has not been acquired before\n", in check_helper_call()
7368 err = check_reference_leak(env); in check_helper_call()
7370 verbose(env, "tail_call would lead to reference leak\n"); in check_helper_call()
7379 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); in check_helper_call()
7384 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7388 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7392 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7396 err = check_bpf_snprintf_call(env, regs); in check_helper_call()
7399 update_loop_inline_state(env, meta.subprogno); in check_helper_call()
7400 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7405 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", in check_helper_call()
7406 reg_type_str(env, regs[BPF_REG_1].type)); in check_helper_call()
7412 env->prog->expected_attach_type == BPF_LSM_CGROUP) { in check_helper_call()
7413 if (!env->prog->aux->attach_func_proto->type) { in check_helper_call()
7417 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_helper_call()
7428 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); in check_helper_call()
7436 meta.ref_obj_id = stack_slot_get_id(env, reg); in check_helper_call()
7441 verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n"); in check_helper_call()
7446 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7456 mark_reg_not_init(env, regs, caller_saved[i]); in check_helper_call()
7457 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_helper_call()
7470 mark_reg_unknown(env, regs, BPF_REG_0); in check_helper_call()
7477 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7483 verbose(env, in check_helper_call()
7492 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
7496 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7500 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7504 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7508 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7516 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7527 verbose(env, "unable to resolve the size of type '%s': %ld\n", in check_helper_call()
7552 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7559 verbose(env, "verifier internal error:"); in check_helper_call()
7560 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", in check_helper_call()
7568 verbose(env, "invalid return type %u of func %s#%d\n", in check_helper_call()
7578 verbose(env, "unknown return type %u of func %s#%d\n", in check_helper_call()
7584 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
7587 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", in check_helper_call()
7596 int id = acquire_reference_state(env, insn_idx); in check_helper_call()
7608 err = check_map_func_compatibility(env, meta.map_ptr, func_id); in check_helper_call()
7614 !env->prog->has_callchain_buf) { in check_helper_call()
7625 verbose(env, err_str, func_id_name(func_id), func_id); in check_helper_call()
7629 env->prog->has_callchain_buf = true; in check_helper_call()
7633 env->prog->call_get_stack = true; in check_helper_call()
7636 if (check_get_func_ip(env)) in check_helper_call()
7638 env->prog->call_get_func_ip = true; in check_helper_call()
7642 clear_all_pkt_pointers(env); in check_helper_call()
7649 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, in mark_btf_func_reg_size() argument
7652 struct bpf_reg_state *reg = &cur_regs(env)[regno]; in mark_btf_func_reg_size()
7658 DEF_NOT_SUBREG : env->insn_idx + 1; in mark_btf_func_reg_size()
7662 mark_insn_zext(env, reg); in mark_btf_func_reg_size()
7663 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in mark_btf_func_reg_size()
7665 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); in mark_btf_func_reg_size()
7670 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_kfunc_call() argument
7674 struct bpf_reg_state *regs = cur_regs(env); in check_kfunc_call()
7688 desc_btf = find_kfunc_desc_btf(env, insn->off); in check_kfunc_call()
7697 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); in check_kfunc_call()
7699 verbose(env, "calling kernel function %s is not allowed\n", in check_kfunc_call()
7704 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capabilities\n"); in check_kfunc_call()
7713 err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, &meta); in check_kfunc_call()
7720 err = release_reference(env, regs[err].ref_obj_id); in check_kfunc_call()
7722 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
7729 mark_reg_not_init(env, regs, caller_saved[i]); in check_kfunc_call()
7735 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); in check_kfunc_call()
7740 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
7741 mark_btf_func_reg_size(env, BPF_REG_0, t->size); in check_kfunc_call()
7749 verbose(env, in check_kfunc_call()
7757 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
7768 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
7776 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
7778 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); in check_kfunc_call()
7780 int id = acquire_reference_state(env, insn_idx); in check_kfunc_call()
7796 mark_btf_func_reg_size(env, regno, sizeof(void *)); in check_kfunc_call()
7799 mark_btf_func_reg_size(env, regno, t->size); in check_kfunc_call()
7845 static bool check_reg_sane_offset(struct bpf_verifier_env *env, in check_reg_sane_offset() argument
7854 verbose(env, "math between %s pointer and %lld is not allowed\n", in check_reg_sane_offset()
7855 reg_type_str(env, type), val); in check_reg_sane_offset()
7860 verbose(env, "%s pointer offset %d is not allowed\n", in check_reg_sane_offset()
7861 reg_type_str(env, type), reg->off); in check_reg_sane_offset()
7866 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", in check_reg_sane_offset()
7867 reg_type_str(env, type)); in check_reg_sane_offset()
7872 verbose(env, "value %lld makes %s pointer be out of bounds\n", in check_reg_sane_offset()
7873 smin, reg_type_str(env, type)); in check_reg_sane_offset()
7919 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, in can_skip_alu_sanitation() argument
7922 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; in can_skip_alu_sanitation()
7942 static int sanitize_val_alu(struct bpf_verifier_env *env, in sanitize_val_alu() argument
7945 struct bpf_insn_aux_data *aux = cur_aux(env); in sanitize_val_alu()
7947 if (can_skip_alu_sanitation(env, insn)) in sanitize_val_alu()
7964 sanitize_speculative_path(struct bpf_verifier_env *env, in sanitize_speculative_path() argument
7971 branch = push_stack(env, next_idx, curr_idx, true); in sanitize_speculative_path()
7975 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
7977 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
7978 mark_reg_unknown(env, regs, insn->src_reg); in sanitize_speculative_path()
7984 static int sanitize_ptr_alu(struct bpf_verifier_env *env, in sanitize_ptr_alu() argument
7992 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; in sanitize_ptr_alu()
7993 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_ptr_alu()
8003 if (can_skip_alu_sanitation(env, insn)) in sanitize_ptr_alu()
8042 env->explore_alu_limits = true; in sanitize_ptr_alu()
8073 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, in sanitize_ptr_alu()
8074 env->insn_idx); in sanitize_ptr_alu()
8080 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) in sanitize_mark_insn_seen() argument
8082 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_mark_insn_seen()
8090 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; in sanitize_mark_insn_seen()
8093 static int sanitize_err(struct bpf_verifier_env *env, in sanitize_err() argument
8104 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", in sanitize_err()
8108 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", in sanitize_err()
8112 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", in sanitize_err()
8116 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", in sanitize_err()
8120 verbose(env, "R%d could not be pushed for speculative verification, %s\n", in sanitize_err()
8124 verbose(env, "verifier internal error: unknown reason (%d)\n", in sanitize_err()
8143 struct bpf_verifier_env *env, in check_stack_access_for_ptr_arithmetic() argument
8152 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", in check_stack_access_for_ptr_arithmetic()
8158 verbose(env, "R%d stack pointer arithmetic goes out of range, " in check_stack_access_for_ptr_arithmetic()
8166 static int sanitize_check_bounds(struct bpf_verifier_env *env, in sanitize_check_bounds() argument
8175 if (env->bypass_spec_v1) in sanitize_check_bounds()
8180 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
8185 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
8186 verbose(env, "R%d pointer arithmetic of map value goes out of range, " in sanitize_check_bounds()
8203 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, in adjust_ptr_min_max_vals() argument
8208 struct bpf_verifier_state *vstate = env->cur_state; in adjust_ptr_min_max_vals()
8228 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8234 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_ptr_min_max_vals()
8235 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8239 verbose(env, in adjust_ptr_min_max_vals()
8246 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", in adjust_ptr_min_max_vals()
8247 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
8262 verbose(env, "R%d pointer arithmetic on %s prohibited\n", in adjust_ptr_min_max_vals()
8263 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
8275 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || in adjust_ptr_min_max_vals()
8276 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
8283 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8286 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
8335 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8343 verbose(env, "R%d tried to subtract pointer from scalar\n", in adjust_ptr_min_max_vals()
8352 verbose(env, "R%d subtraction from stack pointer prohibited\n", in adjust_ptr_min_max_vals()
8394 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8404 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", in adjust_ptr_min_max_vals()
8409 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", in adjust_ptr_min_max_vals()
8414 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
8417 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
8420 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8423 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
9000 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, in adjust_scalar_min_max_vals() argument
9005 struct bpf_reg_state *regs = cur_regs(env); in adjust_scalar_min_max_vals()
9034 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9045 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9052 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9057 ret = sanitize_val_alu(env, insn); in adjust_scalar_min_max_vals()
9059 return sanitize_err(env, insn, ret, NULL, NULL); in adjust_scalar_min_max_vals()
9112 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9125 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9138 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9147 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9161 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, in adjust_reg_min_max_vals() argument
9164 struct bpf_verifier_state *vstate = env->cur_state; in adjust_reg_min_max_vals()
9188 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_reg_min_max_vals()
9189 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
9192 verbose(env, "R%d pointer %s pointer prohibited\n", in adjust_reg_min_max_vals()
9201 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
9204 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
9209 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
9212 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
9223 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
9229 print_verifier_state(env, state, true); in adjust_reg_min_max_vals()
9230 verbose(env, "verifier internal error: unexpected ptr_reg\n"); in adjust_reg_min_max_vals()
9234 print_verifier_state(env, state, true); in adjust_reg_min_max_vals()
9235 verbose(env, "verifier internal error: no src_reg\n"); in adjust_reg_min_max_vals()
9238 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
9242 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_alu_op() argument
9244 struct bpf_reg_state *regs = cur_regs(env); in check_alu_op()
9253 verbose(env, "BPF_NEG uses reserved fields\n"); in check_alu_op()
9260 verbose(env, "BPF_END uses reserved fields\n"); in check_alu_op()
9266 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9270 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
9271 verbose(env, "R%d pointer arithmetic prohibited\n", in check_alu_op()
9277 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
9285 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
9290 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
9295 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
9301 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9318 src_reg->id = ++env->id_gen; in check_alu_op()
9324 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
9325 verbose(env, in check_alu_op()
9337 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
9339 mark_reg_unknown(env, regs, in check_alu_op()
9350 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
9362 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); in check_alu_op()
9369 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
9373 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
9378 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
9384 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9390 verbose(env, "div by zero\n"); in check_alu_op()
9399 verbose(env, "invalid shift %d\n", insn->imm); in check_alu_op()
9405 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9409 return adjust_reg_min_max_vals(env, insn); in check_alu_op()
10130 static int check_cond_jmp_op(struct bpf_verifier_env *env, in check_cond_jmp_op() argument
10133 struct bpf_verifier_state *this_branch = env->cur_state; in check_cond_jmp_op()
10144 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); in check_cond_jmp_op()
10150 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
10155 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_cond_jmp_op()
10159 if (is_pointer_value(env, insn->src_reg)) { in check_cond_jmp_op()
10160 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
10167 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
10173 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
10205 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
10208 err = mark_chain_precision(env, insn->src_reg); in check_cond_jmp_op()
10218 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
10219 !sanitize_speculative_path(env, insn, *insn_idx + 1, in check_cond_jmp_op()
10229 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
10230 !sanitize_speculative_path(env, insn, in check_cond_jmp_op()
10237 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, in check_cond_jmp_op()
10312 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
10313 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
10317 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
10318 print_insn_state(env, this_branch->frame[this_branch->curframe]); in check_cond_jmp_op()
10323 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_imm() argument
10325 struct bpf_insn_aux_data *aux = cur_aux(env); in check_ld_imm()
10326 struct bpf_reg_state *regs = cur_regs(env); in check_ld_imm()
10332 verbose(env, "invalid BPF_LD_IMM insn\n"); in check_ld_imm()
10336 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); in check_ld_imm()
10340 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
10357 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
10370 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
10377 struct bpf_prog_aux *aux = env->prog->aux; in check_ld_imm()
10378 u32 subprogno = find_subprog(env, in check_ld_imm()
10379 env->insn_idx + insn->imm + 1); in check_ld_imm()
10382 verbose(env, "missing btf func_info\n"); in check_ld_imm()
10386 verbose(env, "callback function not static\n"); in check_ld_imm()
10395 map = env->used_maps[aux->map_index]; in check_ld_imm()
10403 dst_reg->id = ++env->id_gen; in check_ld_imm()
10408 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
10442 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_abs() argument
10444 struct bpf_reg_state *regs = cur_regs(env); in check_ld_abs()
10449 if (!may_access_skb(resolve_prog_type(env->prog))) { in check_ld_abs()
10450 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); in check_ld_abs()
10454 if (!env->ops->gen_ld_abs) { in check_ld_abs()
10455 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_abs()
10462 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); in check_ld_abs()
10467 err = check_reg_arg(env, ctx_reg, SRC_OP); in check_ld_abs()
10475 err = check_reference_leak(env); in check_ld_abs()
10477 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); in check_ld_abs()
10481 if (env->cur_state->active_spin_lock) { in check_ld_abs()
10482 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); in check_ld_abs()
10487 verbose(env, in check_ld_abs()
10494 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_ld_abs()
10499 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); in check_ld_abs()
10505 mark_reg_not_init(env, regs, caller_saved[i]); in check_ld_abs()
10506 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_ld_abs()
10513 mark_reg_unknown(env, regs, BPF_REG_0); in check_ld_abs()
10515 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; in check_ld_abs()
10519 static int check_return_code(struct bpf_verifier_env *env) in check_return_code() argument
10522 const struct bpf_prog *prog = env->prog; in check_return_code()
10525 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_return_code()
10527 struct bpf_func_state *frame = env->cur_state->frame[0]; in check_return_code()
10553 err = check_reg_arg(env, BPF_REG_0, SRC_OP); in check_return_code()
10557 if (is_pointer_value(env, BPF_REG_0)) { in check_return_code()
10558 verbose(env, "R0 leaks addr as return value\n"); in check_return_code()
10562 reg = cur_regs(env) + BPF_REG_0; in check_return_code()
10567 verbose(env, "In async callback the register R0 is not a known value (%s)\n", in check_return_code()
10568 reg_type_str(env, reg->type)); in check_return_code()
10573 verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); in check_return_code()
10581 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", in check_return_code()
10582 reg_type_str(env, reg->type)); in check_return_code()
10590 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || in check_return_code()
10591 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || in check_return_code()
10592 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || in check_return_code()
10593 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || in check_return_code()
10594 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || in check_return_code()
10595 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) in check_return_code()
10597 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || in check_return_code()
10598 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) in check_return_code()
10602 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { in check_return_code()
10614 if (!env->prog->aux->attach_btf_id) in check_return_code()
10619 switch (env->prog->expected_attach_type) { in check_return_code()
10638 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { in check_return_code()
10644 if (!env->prog->aux->attach_func_proto->type) { in check_return_code()
10661 verbose(env, "At program exit the register R0 is not a known value (%s)\n", in check_return_code()
10662 reg_type_str(env, reg->type)); in check_return_code()
10667 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); in check_return_code()
10671 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_return_code()
10677 env->prog->enforce_expected_attach_type = 1; in check_return_code()
10721 static u32 state_htab_size(struct bpf_verifier_env *env) in state_htab_size() argument
10723 return env->prog->len; in state_htab_size()
10727 struct bpf_verifier_env *env, in explored_state() argument
10730 struct bpf_verifier_state *cur = env->cur_state; in explored_state()
10733 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; in explored_state()
10736 static void init_explored_state(struct bpf_verifier_env *env, int idx) in init_explored_state() argument
10738 env->insn_aux_data[idx].prune_point = true; in init_explored_state()
10751 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, in push_insn() argument
10754 int *insn_stack = env->cfg.insn_stack; in push_insn()
10755 int *insn_state = env->cfg.insn_state; in push_insn()
10763 if (w < 0 || w >= env->prog->len) { in push_insn()
10764 verbose_linfo(env, t, "%d: ", t); in push_insn()
10765 verbose(env, "jump out of range from insn %d to %d\n", t, w); in push_insn()
10771 init_explored_state(env, w); in push_insn()
10777 if (env->cfg.cur_stack >= env->prog->len) in push_insn()
10779 insn_stack[env->cfg.cur_stack++] = w; in push_insn()
10782 if (loop_ok && env->bpf_capable) in push_insn()
10784 verbose_linfo(env, t, "%d: ", t); in push_insn()
10785 verbose_linfo(env, w, "%d: ", w); in push_insn()
10786 verbose(env, "back-edge from insn %d to %d\n", t, w); in push_insn()
10792 verbose(env, "insn state internal bug\n"); in push_insn()
10800 struct bpf_verifier_env *env, in visit_func_call_insn() argument
10805 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); in visit_func_call_insn()
10810 init_explored_state(env, t + 1); in visit_func_call_insn()
10812 init_explored_state(env, t); in visit_func_call_insn()
10813 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, in visit_func_call_insn()
10828 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) in visit_insn() argument
10830 struct bpf_insn *insns = env->prog->insnsi; in visit_insn()
10834 return visit_func_call_insn(t, insn_cnt, insns, env, true); in visit_insn()
10839 return push_insn(t, t + 1, FALLTHROUGH, env, false); in visit_insn()
10852 init_explored_state(env, t); in visit_insn()
10853 return visit_func_call_insn(t, insn_cnt, insns, env, in visit_insn()
10861 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, in visit_insn()
10870 init_explored_state(env, t + insns[t].off + 1); in visit_insn()
10875 init_explored_state(env, t + 1); in visit_insn()
10881 init_explored_state(env, t); in visit_insn()
10882 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); in visit_insn()
10886 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); in visit_insn()
10893 static int check_cfg(struct bpf_verifier_env *env) in check_cfg() argument
10895 int insn_cnt = env->prog->len; in check_cfg()
10900 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
10904 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
10912 env->cfg.cur_stack = 1; in check_cfg()
10914 while (env->cfg.cur_stack > 0) { in check_cfg()
10915 int t = insn_stack[env->cfg.cur_stack - 1]; in check_cfg()
10917 ret = visit_insn(t, insn_cnt, env); in check_cfg()
10921 env->cfg.cur_stack--; in check_cfg()
10927 verbose(env, "visit_insn internal bug\n"); in check_cfg()
10934 if (env->cfg.cur_stack < 0) { in check_cfg()
10935 verbose(env, "pop stack internal bug\n"); in check_cfg()
10942 verbose(env, "unreachable insn %d\n", i); in check_cfg()
10952 env->cfg.insn_state = env->cfg.insn_stack = NULL; in check_cfg()
10956 static int check_abnormal_return(struct bpf_verifier_env *env) in check_abnormal_return() argument
10960 for (i = 1; i < env->subprog_cnt; i++) { in check_abnormal_return()
10961 if (env->subprog_info[i].has_ld_abs) { in check_abnormal_return()
10962 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); in check_abnormal_return()
10965 if (env->subprog_info[i].has_tail_call) { in check_abnormal_return()
10966 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); in check_abnormal_return()
10977 static int check_btf_func(struct bpf_verifier_env *env, in check_btf_func() argument
10995 if (check_abnormal_return(env)) in check_btf_func()
11000 if (nfuncs != env->subprog_cnt) { in check_btf_func()
11001 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); in check_btf_func()
11009 verbose(env, "invalid func info rec size %u\n", urec_size); in check_btf_func()
11013 prog = env->prog; in check_btf_func()
11030 verbose(env, "nonzero tailing record in func info"); in check_btf_func()
11051 verbose(env, in check_btf_func()
11057 verbose(env, in check_btf_func()
11063 if (env->subprog_info[i].start != krecord[i].insn_off) { in check_btf_func()
11064 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); in check_btf_func()
11071 verbose(env, "invalid type id %d in func info", in check_btf_func()
11084 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { in check_btf_func()
11085 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); in check_btf_func()
11088 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { in check_btf_func()
11089 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); in check_btf_func()
11108 static void adjust_btf_func(struct bpf_verifier_env *env) in adjust_btf_func() argument
11110 struct bpf_prog_aux *aux = env->prog->aux; in adjust_btf_func()
11116 for (i = 0; i < env->subprog_cnt; i++) in adjust_btf_func()
11117 aux->func_info[i].insn_off = env->subprog_info[i].start; in adjust_btf_func()
11123 static int check_btf_line(struct bpf_verifier_env *env, in check_btf_line() argument
11155 prog = env->prog; in check_btf_line()
11159 sub = env->subprog_info; in check_btf_line()
11167 verbose(env, "nonzero tailing record in line_info"); in check_btf_line()
11194 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", in check_btf_line()
11202 verbose(env, in check_btf_line()
11211 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); in check_btf_line()
11216 if (s != env->subprog_cnt) { in check_btf_line()
11221 verbose(env, "missing bpf_line_info for func#%u\n", s); in check_btf_line()
11231 if (s != env->subprog_cnt) { in check_btf_line()
11232 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", in check_btf_line()
11233 env->subprog_cnt - s, s); in check_btf_line()
11251 static int check_core_relo(struct bpf_verifier_env *env, in check_core_relo() argument
11257 struct bpf_prog *prog = env->prog; in check_core_relo()
11260 .log = &env->log, in check_core_relo()
11290 verbose(env, "nonzero tailing record in core_relo"); in check_core_relo()
11305 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", in check_core_relo()
11320 static int check_btf_info(struct bpf_verifier_env *env, in check_btf_info() argument
11328 if (check_abnormal_return(env)) in check_btf_info()
11340 env->prog->aux->btf = btf; in check_btf_info()
11342 err = check_btf_func(env, attr, uattr); in check_btf_info()
11346 err = check_btf_line(env, attr, uattr); in check_btf_info()
11350 err = check_core_relo(env, attr, uattr); in check_btf_info()
11400 static void clean_func_state(struct bpf_verifier_env *env, in clean_func_state() argument
11414 __mark_reg_not_init(env, &st->regs[i]); in clean_func_state()
11422 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); in clean_func_state()
11429 static void clean_verifier_state(struct bpf_verifier_env *env, in clean_verifier_state() argument
11439 clean_func_state(env, st->frame[i]); in clean_verifier_state()
11474 static void clean_live_states(struct bpf_verifier_env *env, int insn, in clean_live_states() argument
11480 sl = *explored_state(env, insn); in clean_live_states()
11490 clean_verifier_state(env, &sl->state); in clean_live_states()
11497 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, in regsafe() argument
11524 if (env->explore_alu_limits) in regsafe()
11615 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, in stacksafe() argument
11661 if (!regsafe(env, &old->stack[spi].spilled_ptr, in stacksafe()
11712 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, in func_states_equal() argument
11717 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); in func_states_equal()
11719 if (!regsafe(env, &old->regs[i], &cur->regs[i], in func_states_equal()
11720 env->idmap_scratch)) in func_states_equal()
11723 if (!stacksafe(env, old, cur, env->idmap_scratch)) in func_states_equal()
11732 static bool states_equal(struct bpf_verifier_env *env, in states_equal() argument
11756 if (!func_states_equal(env, old->frame[i], cur->frame[i])) in states_equal()
11765 static int propagate_liveness_reg(struct bpf_verifier_env *env, in propagate_liveness_reg() argument
11784 err = mark_reg_read(env, reg, parent_reg, flag); in propagate_liveness_reg()
11798 static int propagate_liveness(struct bpf_verifier_env *env, in propagate_liveness() argument
11820 err = propagate_liveness_reg(env, &state_reg[i], in propagate_liveness()
11825 mark_insn_zext(env, &parent_reg[i]); in propagate_liveness()
11833 err = propagate_liveness_reg(env, state_reg, in propagate_liveness()
11845 static int propagate_precision(struct bpf_verifier_env *env, in propagate_precision() argument
11858 if (env->log.level & BPF_LOG_LEVEL2) in propagate_precision()
11859 verbose(env, "propagating r%d\n", i); in propagate_precision()
11860 err = mark_chain_precision(env, i); in propagate_precision()
11872 if (env->log.level & BPF_LOG_LEVEL2) in propagate_precision()
11873 verbose(env, "propagating fp%d\n", in propagate_precision()
11875 err = mark_chain_precision_stack(env, i); in propagate_precision()
11901 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) in is_state_visited() argument
11905 struct bpf_verifier_state *cur = env->cur_state, *new; in is_state_visited()
11907 bool add_new_state = env->test_state_freq ? true : false; in is_state_visited()
11909 cur->last_insn_idx = env->prev_insn_idx; in is_state_visited()
11910 if (!env->insn_aux_data[insn_idx].prune_point) in is_state_visited()
11924 if (env->jmps_processed - env->prev_jmps_processed >= 2 && in is_state_visited()
11925 env->insn_processed - env->prev_insn_processed >= 8) in is_state_visited()
11928 pprev = explored_state(env, insn_idx); in is_state_visited()
11931 clean_live_states(env, insn_idx, cur); in is_state_visited()
11955 states_equal(env, &sl->state, cur)) { in is_state_visited()
11956 verbose_linfo(env, insn_idx, "; "); in is_state_visited()
11957 verbose(env, "infinite loop detected at insn %d\n", insn_idx); in is_state_visited()
11972 if (env->jmps_processed - env->prev_jmps_processed < 20 && in is_state_visited()
11973 env->insn_processed - env->prev_insn_processed < 100) in is_state_visited()
11977 if (states_equal(env, &sl->state, cur)) { in is_state_visited()
11989 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
11996 err = err ? : push_jmp_history(env, cur); in is_state_visited()
11997 err = err ? : propagate_precision(env, &sl->state); in is_state_visited()
12029 env->peak_states--; in is_state_visited()
12035 sl->next = env->free_list; in is_state_visited()
12036 env->free_list = sl; in is_state_visited()
12046 if (env->max_states_per_insn < states_cnt) in is_state_visited()
12047 env->max_states_per_insn = states_cnt; in is_state_visited()
12049 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) in is_state_visited()
12050 return push_jmp_history(env, cur); in is_state_visited()
12053 return push_jmp_history(env, cur); in is_state_visited()
12067 env->total_states++; in is_state_visited()
12068 env->peak_states++; in is_state_visited()
12069 env->prev_jmps_processed = env->jmps_processed; in is_state_visited()
12070 env->prev_insn_processed = env->insn_processed; in is_state_visited()
12087 new_sl->next = *explored_state(env, insn_idx); in is_state_visited()
12088 *explored_state(env, insn_idx) = new_sl; in is_state_visited()
12157 static int do_check(struct bpf_verifier_env *env) in do_check() argument
12159 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check()
12160 struct bpf_verifier_state *state = env->cur_state; in do_check()
12161 struct bpf_insn *insns = env->prog->insnsi; in do_check()
12163 int insn_cnt = env->prog->len; in do_check()
12172 env->prev_insn_idx = prev_insn_idx; in do_check()
12173 if (env->insn_idx >= insn_cnt) { in do_check()
12174 verbose(env, "invalid insn idx %d insn_cnt %d\n", in do_check()
12175 env->insn_idx, insn_cnt); in do_check()
12179 insn = &insns[env->insn_idx]; in do_check()
12182 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { in do_check()
12183 verbose(env, in do_check()
12185 env->insn_processed); in do_check()
12189 err = is_state_visited(env, env->insn_idx); in do_check()
12194 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
12196 verbose(env, "\nfrom %d to %d%s: safe\n", in do_check()
12197 env->prev_insn_idx, env->insn_idx, in do_check()
12198 env->cur_state->speculative ? in do_check()
12201 verbose(env, "%d: safe\n", env->insn_idx); in do_check()
12212 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { in do_check()
12213 verbose(env, "\nfrom %d to %d%s:", in do_check()
12214 env->prev_insn_idx, env->insn_idx, in do_check()
12215 env->cur_state->speculative ? in do_check()
12217 print_verifier_state(env, state->frame[state->curframe], true); in do_check()
12221 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
12225 .private_data = env, in do_check()
12228 if (verifier_state_scratched(env)) in do_check()
12229 print_insn_state(env, state->frame[state->curframe]); in do_check()
12231 verbose_linfo(env, env->insn_idx, "; "); in do_check()
12232 env->prev_log_len = env->log.len_used; in do_check()
12233 verbose(env, "%d: ", env->insn_idx); in do_check()
12234 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in do_check()
12235 env->prev_insn_print_len = env->log.len_used - env->prev_log_len; in do_check()
12236 env->prev_log_len = env->log.len_used; in do_check()
12239 if (bpf_prog_is_dev_bound(env->prog->aux)) { in do_check()
12240 err = bpf_prog_offload_verify_insn(env, env->insn_idx, in do_check()
12241 env->prev_insn_idx); in do_check()
12246 regs = cur_regs(env); in do_check()
12247 sanitize_mark_insn_seen(env); in do_check()
12248 prev_insn_idx = env->insn_idx; in do_check()
12251 err = check_alu_op(env, insn); in do_check()
12261 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
12265 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
12274 err = check_mem_access(env, env->insn_idx, insn->src_reg, in do_check()
12280 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; in do_check()
12297 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
12305 err = check_atomic(env, env->insn_idx, insn); in do_check()
12308 env->insn_idx++; in do_check()
12313 verbose(env, "BPF_STX uses reserved fields\n"); in do_check()
12318 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
12322 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12329 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12335 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; in do_check()
12340 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
12347 verbose(env, "BPF_ST uses reserved fields\n"); in do_check()
12351 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12355 if (is_ctx_reg(env, insn->dst_reg)) { in do_check()
12356 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", in do_check()
12358 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in do_check()
12363 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12372 env->jmps_processed++; in do_check()
12382 verbose(env, "BPF_CALL uses reserved fields\n"); in do_check()
12386 if (env->cur_state->active_spin_lock && in do_check()
12389 verbose(env, "function calls are not allowed while holding a lock\n"); in do_check()
12393 err = check_func_call(env, insn, &env->insn_idx); in do_check()
12395 err = check_kfunc_call(env, insn, &env->insn_idx); in do_check()
12397 err = check_helper_call(env, insn, &env->insn_idx); in do_check()
12406 verbose(env, "BPF_JA uses reserved fields\n"); in do_check()
12410 env->insn_idx += insn->off + 1; in do_check()
12419 verbose(env, "BPF_EXIT uses reserved fields\n"); in do_check()
12423 if (env->cur_state->active_spin_lock) { in do_check()
12424 verbose(env, "bpf_spin_unlock is missing\n"); in do_check()
12434 err = check_reference_leak(env); in do_check()
12440 err = prepare_func_exit(env, &env->insn_idx); in do_check()
12447 err = check_return_code(env); in do_check()
12451 mark_verifier_state_scratched(env); in do_check()
12452 update_branch_counts(env, env->cur_state); in do_check()
12453 err = pop_stack(env, &prev_insn_idx, in do_check()
12454 &env->insn_idx, pop_log); in do_check()
12464 err = check_cond_jmp_op(env, insn, &env->insn_idx); in do_check()
12472 err = check_ld_abs(env, insn); in do_check()
12477 err = check_ld_imm(env, insn); in do_check()
12481 env->insn_idx++; in do_check()
12482 sanitize_mark_insn_seen(env); in do_check()
12484 verbose(env, "invalid BPF_LD mode\n"); in do_check()
12488 verbose(env, "unknown insn class %d\n", class); in do_check()
12492 env->insn_idx++; in do_check()
12529 static int check_pseudo_btf_id(struct bpf_verifier_env *env, in check_pseudo_btf_id() argument
12549 verbose(env, "invalid module BTF object FD specified.\n"); in check_pseudo_btf_id()
12554 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"… in check_pseudo_btf_id()
12563 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); in check_pseudo_btf_id()
12569 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); in check_pseudo_btf_id()
12577 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", in check_pseudo_btf_id()
12612 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", in check_pseudo_btf_id()
12626 for (i = 0; i < env->used_btf_cnt; i++) { in check_pseudo_btf_id()
12627 if (env->used_btfs[i].btf == btf) { in check_pseudo_btf_id()
12633 if (env->used_btf_cnt >= MAX_USED_BTFS) { in check_pseudo_btf_id()
12638 btf_mod = &env->used_btfs[env->used_btf_cnt]; in check_pseudo_btf_id()
12651 env->used_btf_cnt++; in check_pseudo_btf_id()
12673 static int check_map_prog_compatibility(struct bpf_verifier_env *env, in check_map_prog_compatibility() argument
12682 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
12687 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
12692 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
12699 verbose(env, "tracing progs cannot use bpf_timer yet\n"); in check_map_prog_compatibility()
12706 verbose(env, "offload device mismatch between prog and map\n"); in check_map_prog_compatibility()
12711 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); in check_map_prog_compatibility()
12732 verbose(env, in check_map_prog_compatibility()
12753 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) in resolve_pseudo_ldimm64() argument
12755 struct bpf_insn *insn = env->prog->insnsi; in resolve_pseudo_ldimm64()
12756 int insn_cnt = env->prog->len; in resolve_pseudo_ldimm64()
12759 err = bpf_prog_calc_tag(env->prog); in resolve_pseudo_ldimm64()
12766 verbose(env, "BPF_LDX uses reserved fields\n"); in resolve_pseudo_ldimm64()
12780 verbose(env, "invalid bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
12789 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
12790 err = check_pseudo_btf_id(env, insn, aux); in resolve_pseudo_ldimm64()
12797 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
12815 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
12822 if (bpfptr_is_null(env->fd_array)) { in resolve_pseudo_ldimm64()
12823 verbose(env, "fd_idx without fd_array is invalid\n"); in resolve_pseudo_ldimm64()
12826 if (copy_from_bpfptr_offset(&fd, env->fd_array, in resolve_pseudo_ldimm64()
12839 verbose(env, "fd %d is not pointing to valid bpf_map\n", in resolve_pseudo_ldimm64()
12844 err = check_map_prog_compatibility(env, map, env->prog); in resolve_pseudo_ldimm64()
12850 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
12858 verbose(env, "direct value offset of %u is not allowed\n", off); in resolve_pseudo_ldimm64()
12864 verbose(env, "no direct value access support for this map type\n"); in resolve_pseudo_ldimm64()
12871 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", in resolve_pseudo_ldimm64()
12885 for (j = 0; j < env->used_map_cnt; j++) { in resolve_pseudo_ldimm64()
12886 if (env->used_maps[j] == map) { in resolve_pseudo_ldimm64()
12893 if (env->used_map_cnt >= MAX_USED_MAPS) { in resolve_pseudo_ldimm64()
12905 aux->map_index = env->used_map_cnt; in resolve_pseudo_ldimm64()
12906 env->used_maps[env->used_map_cnt++] = map; in resolve_pseudo_ldimm64()
12909 bpf_cgroup_storage_assign(env->prog->aux, map)) { in resolve_pseudo_ldimm64()
12910 verbose(env, "only one cgroup storage of each type is allowed\n"); in resolve_pseudo_ldimm64()
12924 verbose(env, "unknown opcode %02x\n", insn->code); in resolve_pseudo_ldimm64()
12937 static void release_maps(struct bpf_verifier_env *env) in release_maps() argument
12939 __bpf_free_used_maps(env->prog->aux, env->used_maps, in release_maps()
12940 env->used_map_cnt); in release_maps()
12944 static void release_btfs(struct bpf_verifier_env *env) in release_btfs() argument
12946 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, in release_btfs()
12947 env->used_btf_cnt); in release_btfs()
12951 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) in convert_pseudo_ld_imm64() argument
12953 struct bpf_insn *insn = env->prog->insnsi; in convert_pseudo_ld_imm64()
12954 int insn_cnt = env->prog->len; in convert_pseudo_ld_imm64()
12970 static void adjust_insn_aux_data(struct bpf_verifier_env *env, in adjust_insn_aux_data() argument
12974 struct bpf_insn_aux_data *old_data = env->insn_aux_data; in adjust_insn_aux_data()
12984 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); in adjust_insn_aux_data()
12996 new_data[i].zext_dst = insn_has_def32(env, insn + i); in adjust_insn_aux_data()
12998 env->insn_aux_data = new_data; in adjust_insn_aux_data()
13002 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) in adjust_subprog_starts() argument
13009 for (i = 0; i <= env->subprog_cnt; i++) { in adjust_subprog_starts()
13010 if (env->subprog_info[i].start <= off) in adjust_subprog_starts()
13012 env->subprog_info[i].start += len - 1; in adjust_subprog_starts()
13030 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, in bpf_patch_insn_data() argument
13037 new_data = vzalloc(array_size(env->prog->len + len - 1, in bpf_patch_insn_data()
13043 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); in bpf_patch_insn_data()
13046 verbose(env, in bpf_patch_insn_data()
13048 env->insn_aux_data[off].orig_idx); in bpf_patch_insn_data()
13052 adjust_insn_aux_data(env, new_data, new_prog, off, len); in bpf_patch_insn_data()
13053 adjust_subprog_starts(env, off, len); in bpf_patch_insn_data()
13058 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, in adjust_subprog_starts_after_remove() argument
13064 for (i = 0; i < env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
13065 if (env->subprog_info[i].start >= off) in adjust_subprog_starts_after_remove()
13068 for (j = i; j < env->subprog_cnt; j++) in adjust_subprog_starts_after_remove()
13069 if (env->subprog_info[j].start >= off + cnt) in adjust_subprog_starts_after_remove()
13074 if (env->subprog_info[j].start != off + cnt) in adjust_subprog_starts_after_remove()
13078 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove()
13082 move = env->subprog_cnt + 1 - j; in adjust_subprog_starts_after_remove()
13084 memmove(env->subprog_info + i, in adjust_subprog_starts_after_remove()
13085 env->subprog_info + j, in adjust_subprog_starts_after_remove()
13086 sizeof(*env->subprog_info) * move); in adjust_subprog_starts_after_remove()
13087 env->subprog_cnt -= j - i; in adjust_subprog_starts_after_remove()
13103 if (env->subprog_info[i].start == off) in adjust_subprog_starts_after_remove()
13108 for (; i <= env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
13109 env->subprog_info[i].start -= cnt; in adjust_subprog_starts_after_remove()
13114 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, in bpf_adj_linfo_after_remove() argument
13117 struct bpf_prog *prog = env->prog; in bpf_adj_linfo_after_remove()
13164 for (i = 0; i <= env->subprog_cnt; i++) in bpf_adj_linfo_after_remove()
13165 if (env->subprog_info[i].linfo_idx > l_off) { in bpf_adj_linfo_after_remove()
13169 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) in bpf_adj_linfo_after_remove()
13170 env->subprog_info[i].linfo_idx -= l_cnt; in bpf_adj_linfo_after_remove()
13172 env->subprog_info[i].linfo_idx = l_off; in bpf_adj_linfo_after_remove()
13178 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) in verifier_remove_insns() argument
13180 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in verifier_remove_insns()
13181 unsigned int orig_prog_len = env->prog->len; in verifier_remove_insns()
13184 if (bpf_prog_is_dev_bound(env->prog->aux)) in verifier_remove_insns()
13185 bpf_prog_offload_remove_insns(env, off, cnt); in verifier_remove_insns()
13187 err = bpf_remove_insns(env->prog, off, cnt); in verifier_remove_insns()
13191 err = adjust_subprog_starts_after_remove(env, off, cnt); in verifier_remove_insns()
13195 err = bpf_adj_linfo_after_remove(env, off, cnt); in verifier_remove_insns()
13216 static void sanitize_dead_code(struct bpf_verifier_env *env) in sanitize_dead_code() argument
13218 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in sanitize_dead_code()
13220 struct bpf_insn *insn = env->prog->insnsi; in sanitize_dead_code()
13221 const int insn_cnt = env->prog->len; in sanitize_dead_code()
13246 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) in opt_hard_wire_dead_code_branches() argument
13248 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_hard_wire_dead_code_branches()
13250 struct bpf_insn *insn = env->prog->insnsi; in opt_hard_wire_dead_code_branches()
13251 const int insn_cnt = env->prog->len; in opt_hard_wire_dead_code_branches()
13265 if (bpf_prog_is_dev_bound(env->prog->aux)) in opt_hard_wire_dead_code_branches()
13266 bpf_prog_offload_replace_insn(env, i, &ja); in opt_hard_wire_dead_code_branches()
13272 static int opt_remove_dead_code(struct bpf_verifier_env *env) in opt_remove_dead_code() argument
13274 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_remove_dead_code()
13275 int insn_cnt = env->prog->len; in opt_remove_dead_code()
13287 err = verifier_remove_insns(env, i, j); in opt_remove_dead_code()
13290 insn_cnt = env->prog->len; in opt_remove_dead_code()
13296 static int opt_remove_nops(struct bpf_verifier_env *env) in opt_remove_nops() argument
13299 struct bpf_insn *insn = env->prog->insnsi; in opt_remove_nops()
13300 int insn_cnt = env->prog->len; in opt_remove_nops()
13307 err = verifier_remove_insns(env, i, 1); in opt_remove_nops()
13317 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, in opt_subreg_zext_lo32_rnd_hi32() argument
13321 struct bpf_insn_aux_data *aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
13322 int i, patch_len, delta = 0, len = env->prog->len; in opt_subreg_zext_lo32_rnd_hi32()
13323 struct bpf_insn *insns = env->prog->insnsi; in opt_subreg_zext_lo32_rnd_hi32()
13355 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { in opt_subreg_zext_lo32_rnd_hi32()
13390 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); in opt_subreg_zext_lo32_rnd_hi32()
13400 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); in opt_subreg_zext_lo32_rnd_hi32()
13403 env->prog = new_prog; in opt_subreg_zext_lo32_rnd_hi32()
13405 aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
13417 static int convert_ctx_accesses(struct bpf_verifier_env *env) in convert_ctx_accesses() argument
13419 const struct bpf_verifier_ops *ops = env->ops; in convert_ctx_accesses()
13421 const int insn_cnt = env->prog->len; in convert_ctx_accesses()
13428 if (ops->gen_prologue || env->seen_direct_write) { in convert_ctx_accesses()
13430 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
13433 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in convert_ctx_accesses()
13434 env->prog); in convert_ctx_accesses()
13436 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
13439 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
13443 env->prog = new_prog; in convert_ctx_accesses()
13448 if (bpf_prog_is_dev_bound(env->prog->aux)) in convert_ctx_accesses()
13451 insn = env->prog->insnsi + delta; in convert_ctx_accesses()
13478 env->insn_aux_data[i + delta].sanitize_stack_spill) { in convert_ctx_accesses()
13485 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); in convert_ctx_accesses()
13490 env->prog = new_prog; in convert_ctx_accesses()
13498 switch ((int)env->insn_aux_data[i + delta].ptr_type) { in convert_ctx_accesses()
13519 env->prog->aux->num_exentries++; in convert_ctx_accesses()
13526 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; in convert_ctx_accesses()
13541 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); in convert_ctx_accesses()
13556 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in convert_ctx_accesses()
13560 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
13568 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); in convert_ctx_accesses()
13588 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in convert_ctx_accesses()
13595 env->prog = new_prog; in convert_ctx_accesses()
13602 static int jit_subprogs(struct bpf_verifier_env *env) in jit_subprogs() argument
13604 struct bpf_prog *prog = env->prog, **func, *tmp; in jit_subprogs()
13611 if (env->subprog_cnt <= 1) in jit_subprogs()
13622 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
13635 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
13651 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); in jit_subprogs()
13655 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
13657 subprog_end = env->subprog_info[i + 1].start; in jit_subprogs()
13693 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
13701 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
13710 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
13723 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
13750 func[i]->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
13752 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
13756 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); in jit_subprogs()
13766 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
13777 insn[0].imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
13784 insn->off = env->insn_aux_data[i].call_imm; in jit_subprogs()
13785 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
13793 prog->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
13809 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
13824 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
13830 static int fixup_call_args(struct bpf_verifier_env *env) in fixup_call_args() argument
13833 struct bpf_prog *prog = env->prog; in fixup_call_args()
13840 if (env->prog->jit_requested && in fixup_call_args()
13841 !bpf_prog_is_dev_bound(env->prog->aux)) { in fixup_call_args()
13842 err = jit_subprogs(env); in fixup_call_args()
13850 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); in fixup_call_args()
13853 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in fixup_call_args()
13857 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in fixup_call_args()
13865 verbose(env, "callbacks are not allowed in non-JITed programs\n"); in fixup_call_args()
13871 depth = get_callee_stack_depth(env, insn, i); in fixup_call_args()
13881 static int fixup_kfunc_call(struct bpf_verifier_env *env, in fixup_kfunc_call() argument
13887 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); in fixup_kfunc_call()
13894 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); in fixup_kfunc_call()
13896 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", in fixup_kfunc_call()
13909 static int do_misc_fixups(struct bpf_verifier_env *env) in do_misc_fixups() argument
13911 struct bpf_prog *prog = env->prog; in do_misc_fixups()
13956 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in do_misc_fixups()
13961 env->prog = prog = new_prog; in do_misc_fixups()
13970 cnt = env->ops->gen_ld_abs(insn, insn_buf); in do_misc_fixups()
13972 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
13976 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
13981 env->prog = prog = new_prog; in do_misc_fixups()
13995 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
14029 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14034 env->prog = prog = new_prog; in do_misc_fixups()
14044 ret = fixup_kfunc_call(env, insn); in do_misc_fixups()
14063 if (!allow_tail_call_in_subprogs(env)) in do_misc_fixups()
14075 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
14076 if (env->bpf_capable && !prog->blinding_requested && in do_misc_fixups()
14090 verbose(env, "adding tail call poke descriptor failed\n"); in do_misc_fixups()
14108 verbose(env, "tail_call abusing map_ptr\n"); in do_misc_fixups()
14121 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14126 env->prog = prog = new_prog; in do_misc_fixups()
14154 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14159 env->prog = prog = new_prog; in do_misc_fixups()
14167 if (env->prog->aux->sleepable) in do_misc_fixups()
14174 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14179 env->prog = prog = new_prog; in do_misc_fixups()
14198 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
14210 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
14214 new_prog = bpf_patch_insn_data(env, i + delta, in do_misc_fixups()
14220 env->prog = prog = new_prog; in do_misc_fixups()
14297 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, in do_misc_fixups()
14303 env->prog = prog = new_prog; in do_misc_fixups()
14323 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14328 env->prog = prog = new_prog; in do_misc_fixups()
14351 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14356 env->prog = prog = new_prog; in do_misc_fixups()
14367 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
14371 env->prog = prog = new_prog; in do_misc_fixups()
14382 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
14386 env->prog = prog = new_prog; in do_misc_fixups()
14392 fn = env->ops->get_func_proto(insn->imm, env->prog); in do_misc_fixups()
14397 verbose(env, in do_misc_fixups()
14411 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
14417 verbose(env, "tracking tail call prog failed\n"); in do_misc_fixups()
14422 sort_kfunc_descs_by_imm(env->prog); in do_misc_fixups()
14427 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, in inline_bpf_loop() argument
14488 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); in inline_bpf_loop()
14493 callback_start = env->subprog_info[callback_subprogno].start; in inline_bpf_loop()
14518 static int optimize_bpf_loop(struct bpf_verifier_env *env) in optimize_bpf_loop() argument
14520 struct bpf_subprog_info *subprogs = env->subprog_info; in optimize_bpf_loop()
14522 struct bpf_insn *insn = env->prog->insnsi; in optimize_bpf_loop()
14523 int insn_cnt = env->prog->len; in optimize_bpf_loop()
14530 &env->insn_aux_data[i + delta].loop_inline_state; in optimize_bpf_loop()
14536 new_prog = inline_bpf_loop(env, in optimize_bpf_loop()
14545 env->prog = new_prog; in optimize_bpf_loop()
14558 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in optimize_bpf_loop()
14563 static void free_states(struct bpf_verifier_env *env) in free_states() argument
14568 sl = env->free_list; in free_states()
14575 env->free_list = NULL; in free_states()
14577 if (!env->explored_states) in free_states()
14580 for (i = 0; i < state_htab_size(env); i++) { in free_states()
14581 sl = env->explored_states[i]; in free_states()
14589 env->explored_states[i] = NULL; in free_states()
14593 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
14595 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check_common()
14600 env->prev_linfo = NULL; in do_check_common()
14601 env->pass_cnt++; in do_check_common()
14614 env->cur_state = state; in do_check_common()
14615 init_func_state(env, state->frame[0], in do_check_common()
14621 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
14622 ret = btf_prepare_func_args(env, subprog, regs); in do_check_common()
14627 mark_reg_known_zero(env, regs, i); in do_check_common()
14629 mark_reg_unknown(env, regs, i); in do_check_common()
14633 mark_reg_known_zero(env, regs, i); in do_check_common()
14635 regs[i].id = ++env->id_gen; in do_check_common()
14641 mark_reg_known_zero(env, regs, BPF_REG_1); in do_check_common()
14642 ret = btf_check_subprog_arg_match(env, subprog, regs); in do_check_common()
14656 ret = do_check(env); in do_check_common()
14661 if (env->cur_state) { in do_check_common()
14662 free_verifier_state(env->cur_state, true); in do_check_common()
14663 env->cur_state = NULL; in do_check_common()
14665 while (!pop_stack(env, NULL, NULL, false)); in do_check_common()
14667 bpf_vlog_reset(&env->log, 0); in do_check_common()
14668 free_states(env); in do_check_common()
14689 static int do_check_subprogs(struct bpf_verifier_env *env) in do_check_subprogs() argument
14691 struct bpf_prog_aux *aux = env->prog->aux; in do_check_subprogs()
14697 for (i = 1; i < env->subprog_cnt; i++) { in do_check_subprogs()
14700 env->insn_idx = env->subprog_info[i].start; in do_check_subprogs()
14701 WARN_ON_ONCE(env->insn_idx == 0); in do_check_subprogs()
14702 ret = do_check_common(env, i); in do_check_subprogs()
14705 } else if (env->log.level & BPF_LOG_LEVEL) { in do_check_subprogs()
14706 verbose(env, in do_check_subprogs()
14714 static int do_check_main(struct bpf_verifier_env *env) in do_check_main() argument
14718 env->insn_idx = 0; in do_check_main()
14719 ret = do_check_common(env, 0); in do_check_main()
14721 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check_main()
14726 static void print_verification_stats(struct bpf_verifier_env *env) in print_verification_stats() argument
14730 if (env->log.level & BPF_LOG_STATS) { in print_verification_stats()
14731 verbose(env, "verification time %lld usec\n", in print_verification_stats()
14732 div_u64(env->verification_time, 1000)); in print_verification_stats()
14733 verbose(env, "stack depth "); in print_verification_stats()
14734 for (i = 0; i < env->subprog_cnt; i++) { in print_verification_stats()
14735 u32 depth = env->subprog_info[i].stack_depth; in print_verification_stats()
14737 verbose(env, "%d", depth); in print_verification_stats()
14738 if (i + 1 < env->subprog_cnt) in print_verification_stats()
14739 verbose(env, "+"); in print_verification_stats()
14741 verbose(env, "\n"); in print_verification_stats()
14743 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " in print_verification_stats()
14745 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, in print_verification_stats()
14746 env->max_states_per_insn, env->total_states, in print_verification_stats()
14747 env->peak_states, env->longest_mark_read_walk); in print_verification_stats()
14750 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) in check_struct_ops_btf_id() argument
14755 struct bpf_prog *prog = env->prog; in check_struct_ops_btf_id()
14760 verbose(env, "struct ops programs must have a GPL compatible license\n"); in check_struct_ops_btf_id()
14767 verbose(env, "attach_btf_id %u is not a supported struct\n", in check_struct_ops_btf_id()
14775 verbose(env, "attach to invalid member idx %u of struct %s\n", in check_struct_ops_btf_id()
14785 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", in check_struct_ops_btf_id()
14794 verbose(env, "attach to unsupported member %s of struct %s\n", in check_struct_ops_btf_id()
14802 env->ops = st_ops->verifier_ops; in check_struct_ops_btf_id()
15080 static int check_attach_btf_id(struct bpf_verifier_env *env) in BTF_SET_START()
15082 struct bpf_prog *prog = env->prog; in BTF_SET_START()
15094 verbose(env, "Syscall programs can only be sleepable\n"); in BTF_SET_START()
15100 verbose(env, "Only fentry/fexit/fmod_ret, lsm, and kprobe/uprobe programs can be sleepable\n"); in BTF_SET_START()
15105 return check_struct_ops_btf_id(env); in BTF_SET_START()
15112 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); in BTF_SET_START()
15121 env->ops = bpf_verifier_ops[tgt_prog->type]; in BTF_SET_START()
15144 ret = bpf_lsm_verify_prog(&env->log, prog); in BTF_SET_START()
15175 struct bpf_verifier_env *env; in bpf_check() local
15187 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); in bpf_check()
15188 if (!env) in bpf_check()
15190 log = &env->log; in bpf_check()
15193 env->insn_aux_data = in bpf_check()
15196 if (!env->insn_aux_data) in bpf_check()
15199 env->insn_aux_data[i].orig_idx = i; in bpf_check()
15200 env->prog = *prog; in bpf_check()
15201 env->ops = bpf_verifier_ops[env->prog->type]; in bpf_check()
15202 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); in bpf_check()
15226 mark_verifier_state_clean(env); in bpf_check()
15230 verbose(env, "in-kernel BTF is malformed\n"); in bpf_check()
15235 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); in bpf_check()
15237 env->strict_alignment = true; in bpf_check()
15239 env->strict_alignment = false; in bpf_check()
15241 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); in bpf_check()
15242 env->allow_uninit_stack = bpf_allow_uninit_stack(); in bpf_check()
15243 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); in bpf_check()
15244 env->bypass_spec_v1 = bpf_bypass_spec_v1(); in bpf_check()
15245 env->bypass_spec_v4 = bpf_bypass_spec_v4(); in bpf_check()
15246 env->bpf_capable = bpf_capable(); in bpf_check()
15249 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; in bpf_check()
15251 env->explored_states = kvcalloc(state_htab_size(env), in bpf_check()
15255 if (!env->explored_states) in bpf_check()
15258 ret = add_subprog_and_kfunc(env); in bpf_check()
15262 ret = check_subprogs(env); in bpf_check()
15266 ret = check_btf_info(env, attr, uattr); in bpf_check()
15270 ret = check_attach_btf_id(env); in bpf_check()
15274 ret = resolve_pseudo_ldimm64(env); in bpf_check()
15278 if (bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
15279 ret = bpf_prog_offload_verifier_prep(env->prog); in bpf_check()
15284 ret = check_cfg(env); in bpf_check()
15288 ret = do_check_subprogs(env); in bpf_check()
15289 ret = ret ?: do_check_main(env); in bpf_check()
15291 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) in bpf_check()
15292 ret = bpf_prog_offload_finalize(env); in bpf_check()
15295 kvfree(env->explored_states); in bpf_check()
15298 ret = check_max_stack_depth(env); in bpf_check()
15302 ret = optimize_bpf_loop(env); in bpf_check()
15306 opt_hard_wire_dead_code_branches(env); in bpf_check()
15308 ret = opt_remove_dead_code(env); in bpf_check()
15310 ret = opt_remove_nops(env); in bpf_check()
15313 sanitize_dead_code(env); in bpf_check()
15318 ret = convert_ctx_accesses(env); in bpf_check()
15321 ret = do_misc_fixups(env); in bpf_check()
15326 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
15327 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); in bpf_check()
15328 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret in bpf_check()
15333 ret = fixup_call_args(env); in bpf_check()
15335 env->verification_time = ktime_get_ns() - start_time; in bpf_check()
15336 print_verification_stats(env); in bpf_check()
15337 env->prog->aux->verified_insns = env->insn_processed; in bpf_check()
15349 if (env->used_map_cnt) { in bpf_check()
15351 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, in bpf_check()
15352 sizeof(env->used_maps[0]), in bpf_check()
15355 if (!env->prog->aux->used_maps) { in bpf_check()
15360 memcpy(env->prog->aux->used_maps, env->used_maps, in bpf_check()
15361 sizeof(env->used_maps[0]) * env->used_map_cnt); in bpf_check()
15362 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
15364 if (env->used_btf_cnt) { in bpf_check()
15366 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, in bpf_check()
15367 sizeof(env->used_btfs[0]), in bpf_check()
15369 if (!env->prog->aux->used_btfs) { in bpf_check()
15374 memcpy(env->prog->aux->used_btfs, env->used_btfs, in bpf_check()
15375 sizeof(env->used_btfs[0]) * env->used_btf_cnt); in bpf_check()
15376 env->prog->aux->used_btf_cnt = env->used_btf_cnt; in bpf_check()
15378 if (env->used_map_cnt || env->used_btf_cnt) { in bpf_check()
15382 convert_pseudo_ld_imm64(env); in bpf_check()
15385 adjust_btf_func(env); in bpf_check()
15388 if (!env->prog->aux->used_maps) in bpf_check()
15392 release_maps(env); in bpf_check()
15393 if (!env->prog->aux->used_btfs) in bpf_check()
15394 release_btfs(env); in bpf_check()
15399 if (env->prog->type == BPF_PROG_TYPE_EXT) in bpf_check()
15400 env->prog->expected_attach_type = 0; in bpf_check()
15402 *prog = env->prog; in bpf_check()
15406 vfree(env->insn_aux_data); in bpf_check()
15408 kfree(env); in bpf_check()