Lines Matching refs:env
219 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, in bpf_verifier_log_write() argument
224 if (!bpf_verifier_log_needed(&env->log)) in bpf_verifier_log_write()
228 bpf_verifier_vlog(&env->log, fmt, args); in bpf_verifier_log_write()
235 struct bpf_verifier_env *env = private_data; in verbose() local
238 if (!bpf_verifier_log_needed(&env->log)) in verbose()
242 bpf_verifier_vlog(&env->log, fmt, args); in verbose()
266 static void print_liveness(struct bpf_verifier_env *env, in print_liveness() argument
270 verbose(env, "_"); in print_liveness()
272 verbose(env, "r"); in print_liveness()
274 verbose(env, "w"); in print_liveness()
277 static struct bpf_func_state *func(struct bpf_verifier_env *env, in func() argument
280 struct bpf_verifier_state *cur = env->cur_state; in func()
285 static void print_verifier_state(struct bpf_verifier_env *env, in print_verifier_state() argument
293 verbose(env, " frame%d:", state->frameno); in print_verifier_state()
299 verbose(env, " R%d", i); in print_verifier_state()
300 print_liveness(env, reg->live); in print_verifier_state()
301 verbose(env, "=%s", reg_type_str[t]); in print_verifier_state()
305 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
307 verbose(env, ",call_%d", func(env, reg)->callsite); in print_verifier_state()
309 verbose(env, "(id=%d", reg->id); in print_verifier_state()
311 verbose(env, ",off=%d", reg->off); in print_verifier_state()
313 verbose(env, ",r=%d", reg->range); in print_verifier_state()
317 verbose(env, ",ks=%d,vs=%d", in print_verifier_state()
325 verbose(env, ",imm=%llx", reg->var_off.value); in print_verifier_state()
329 verbose(env, ",smin_value=%lld", in print_verifier_state()
333 verbose(env, ",smax_value=%lld", in print_verifier_state()
336 verbose(env, ",umin_value=%llu", in print_verifier_state()
339 verbose(env, ",umax_value=%llu", in print_verifier_state()
345 verbose(env, ",var_off=%s", tn_buf); in print_verifier_state()
348 verbose(env, ")"); in print_verifier_state()
353 verbose(env, " fp%d", in print_verifier_state()
355 print_liveness(env, state->stack[i].spilled_ptr.live); in print_verifier_state()
356 verbose(env, "=%s", in print_verifier_state()
360 verbose(env, " fp%d=0", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
362 verbose(env, "\n"); in print_verifier_state()
485 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, in pop_stack() argument
488 struct bpf_verifier_state *cur = env->cur_state; in pop_stack()
489 struct bpf_verifier_stack_elem *elem, *head = env->head; in pop_stack()
492 if (env->head == NULL) in pop_stack()
507 env->head = elem; in pop_stack()
508 env->stack_size--; in pop_stack()
512 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, in push_stack() argument
515 struct bpf_verifier_state *cur = env->cur_state; in push_stack()
525 elem->next = env->head; in push_stack()
526 env->head = elem; in push_stack()
527 env->stack_size++; in push_stack()
531 if (env->stack_size > BPF_COMPLEXITY_LIMIT_STACK) { in push_stack()
532 verbose(env, "BPF program is too complex\n"); in push_stack()
537 free_verifier_state(env->cur_state, true); in push_stack()
538 env->cur_state = NULL; in push_stack()
540 while (!pop_stack(env, NULL, NULL)); in push_stack()
579 static void mark_reg_known_zero(struct bpf_verifier_env *env, in mark_reg_known_zero() argument
583 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); in mark_reg_known_zero()
694 static void mark_reg_unknown(struct bpf_verifier_env *env, in mark_reg_unknown() argument
698 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); in mark_reg_unknown()
713 static void mark_reg_not_init(struct bpf_verifier_env *env, in mark_reg_not_init() argument
717 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); in mark_reg_not_init()
726 static void init_reg_state(struct bpf_verifier_env *env, in init_reg_state() argument
733 mark_reg_not_init(env, regs, i); in init_reg_state()
739 mark_reg_known_zero(env, regs, BPF_REG_FP); in init_reg_state()
744 mark_reg_known_zero(env, regs, BPF_REG_1); in init_reg_state()
748 static void init_func_state(struct bpf_verifier_env *env, in init_func_state() argument
755 init_reg_state(env, state); in init_func_state()
770 static int find_subprog(struct bpf_verifier_env *env, int off) in find_subprog() argument
774 p = bsearch(&off, env->subprog_info, env->subprog_cnt, in find_subprog()
775 sizeof(env->subprog_info[0]), cmp_subprogs); in find_subprog()
778 return p - env->subprog_info; in find_subprog()
782 static int add_subprog(struct bpf_verifier_env *env, int off) in add_subprog() argument
784 int insn_cnt = env->prog->len; in add_subprog()
788 verbose(env, "call to invalid destination\n"); in add_subprog()
791 ret = find_subprog(env, off); in add_subprog()
794 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { in add_subprog()
795 verbose(env, "too many subprograms\n"); in add_subprog()
798 env->subprog_info[env->subprog_cnt++].start = off; in add_subprog()
799 sort(env->subprog_info, env->subprog_cnt, in add_subprog()
800 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); in add_subprog()
804 static int check_subprogs(struct bpf_verifier_env *env) in check_subprogs() argument
807 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs()
808 struct bpf_insn *insn = env->prog->insnsi; in check_subprogs()
809 int insn_cnt = env->prog->len; in check_subprogs()
812 ret = add_subprog(env, 0); in check_subprogs()
822 if (!env->allow_ptr_leaks) { in check_subprogs()
823 verbose(env, "function calls to other bpf functions are allowed for root only\n"); in check_subprogs()
826 if (bpf_prog_is_dev_bound(env->prog->aux)) { in check_subprogs()
827 verbose(env, "function calls in offloaded programs are not supported yet\n"); in check_subprogs()
830 ret = add_subprog(env, i + insn[i].imm + 1); in check_subprogs()
838 subprog[env->subprog_cnt].start = insn_cnt; in check_subprogs()
840 if (env->log.level > 1) in check_subprogs()
841 for (i = 0; i < env->subprog_cnt; i++) in check_subprogs()
842 verbose(env, "func#%d @%d\n", i, subprog[i].start); in check_subprogs()
856 verbose(env, "jump out of range from insn %d to %d\n", i, off); in check_subprogs()
867 verbose(env, "last insn is not an exit or jmp\n"); in check_subprogs()
872 if (cur_subprog < env->subprog_cnt) in check_subprogs()
880 struct bpf_verifier_state *skip_callee(struct bpf_verifier_env *env, in skip_callee() argument
921 verbose(env, "verifier bug regno %d tmp %p\n", regno, tmp); in skip_callee()
922 verbose(env, "regno %d parent frame %d current frame %d\n", in skip_callee()
927 static int mark_reg_read(struct bpf_verifier_env *env, in mark_reg_read() argument
942 parent = skip_callee(env, state, parent, regno); in mark_reg_read()
954 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, in check_reg_arg() argument
957 struct bpf_verifier_state *vstate = env->cur_state; in check_reg_arg()
962 verbose(env, "R%d is invalid\n", regno); in check_reg_arg()
969 verbose(env, "R%d !read_ok\n", regno); in check_reg_arg()
972 return mark_reg_read(env, vstate, vstate->parent, regno); in check_reg_arg()
976 verbose(env, "frame pointer is read only\n"); in check_reg_arg()
981 mark_reg_unknown(env, regs, regno); in check_reg_arg()
1012 static int check_stack_write(struct bpf_verifier_env *env, in check_stack_write() argument
1027 if (!env->allow_ptr_leaks && in check_stack_write()
1030 verbose(env, "attempt to corrupt spilled pointer on stack\n"); in check_stack_write()
1034 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write()
1040 verbose(env, "invalid size of register spill\n"); in check_stack_write()
1045 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); in check_stack_write()
1055 !env->allow_ptr_leaks) { in check_stack_write()
1056 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; in check_stack_write()
1071 verbose(env, in check_stack_write()
1137 static void mark_stack_slot_read(struct bpf_verifier_env *env, in mark_stack_slot_read() argument
1164 static int check_stack_read(struct bpf_verifier_env *env, in check_stack_read() argument
1168 struct bpf_verifier_state *vstate = env->cur_state; in check_stack_read()
1174 verbose(env, "invalid read from stack off %d+0 size %d\n", in check_stack_read()
1182 verbose(env, "invalid size of register spill\n"); in check_stack_read()
1187 verbose(env, "corrupted spill memory\n"); in check_stack_read()
1201 mark_stack_slot_read(env, vstate, vstate->parent, spi, in check_stack_read()
1214 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read()
1218 mark_stack_slot_read(env, vstate, vstate->parent, spi, in check_stack_read()
1228 mark_reg_unknown(env, state->regs, value_regno); in check_stack_read()
1237 static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off, in __check_map_access() argument
1240 struct bpf_reg_state *regs = cur_regs(env); in __check_map_access()
1245 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", in __check_map_access()
1253 static int check_map_access(struct bpf_verifier_env *env, u32 regno, in check_map_access() argument
1256 struct bpf_verifier_state *vstate = env->cur_state; in check_map_access()
1265 if (env->log.level) in check_map_access()
1266 print_verifier_state(env, state); in check_map_access()
1274 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_map_access()
1278 err = __check_map_access(env, regno, reg->smin_value + off, size, in check_map_access()
1281 verbose(env, "R%d min value is outside of the array range\n", in check_map_access()
1291 …verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n… in check_map_access()
1295 err = __check_map_access(env, regno, reg->umax_value + off, size, in check_map_access()
1298 verbose(env, "R%d max value is outside of the array range\n", in check_map_access()
1305 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, in may_access_direct_pkt_data() argument
1309 switch (env->prog->type) { in may_access_direct_pkt_data()
1327 env->seen_direct_write = true; in may_access_direct_pkt_data()
1334 static int __check_packet_access(struct bpf_verifier_env *env, u32 regno, in __check_packet_access() argument
1337 struct bpf_reg_state *regs = cur_regs(env); in __check_packet_access()
1342 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", in __check_packet_access()
1349 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, in check_packet_access() argument
1352 struct bpf_reg_state *regs = cur_regs(env); in check_packet_access()
1365 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_packet_access()
1369 err = __check_packet_access(env, regno, off, size, zero_size_allowed); in check_packet_access()
1371 verbose(env, "R%d offset is outside of the packet\n", regno); in check_packet_access()
1378 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, in check_ctx_access() argument
1385 if (env->ops->is_valid_access && in check_ctx_access()
1386 env->ops->is_valid_access(off, size, t, env->prog, &info)) { in check_ctx_access()
1396 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; in check_ctx_access()
1398 if (env->prog->aux->max_ctx_offset < off + size) in check_ctx_access()
1399 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
1403 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); in check_ctx_access()
1416 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) in is_pointer_value() argument
1418 return __is_pointer_value(env->allow_ptr_leaks, cur_regs(env) + regno); in is_pointer_value()
1421 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) in is_ctx_reg() argument
1423 const struct bpf_reg_state *reg = cur_regs(env) + regno; in is_ctx_reg()
1428 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) in is_pkt_reg() argument
1430 const struct bpf_reg_state *reg = cur_regs(env) + regno; in is_pkt_reg()
1435 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, in check_pkt_ptr_alignment() argument
1461 verbose(env, in check_pkt_ptr_alignment()
1470 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, in check_generic_ptr_alignment() argument
1486 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", in check_generic_ptr_alignment()
1494 static int check_ptr_alignment(struct bpf_verifier_env *env, in check_ptr_alignment() argument
1498 bool strict = env->strict_alignment || strict_alignment_once; in check_ptr_alignment()
1507 return check_pkt_ptr_alignment(env, reg, off, size, strict); in check_ptr_alignment()
1525 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, in check_ptr_alignment()
1529 static int update_stack_depth(struct bpf_verifier_env *env, in update_stack_depth() argument
1533 u16 stack = env->subprog_info[func->subprogno].stack_depth; in update_stack_depth()
1539 env->subprog_info[func->subprogno].stack_depth = -off; in update_stack_depth()
1549 static int check_max_stack_depth(struct bpf_verifier_env *env) in check_max_stack_depth() argument
1552 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth()
1553 struct bpf_insn *insn = env->prog->insnsi; in check_max_stack_depth()
1563 verbose(env, "combined stack size of %d calls is %d. Too large\n", in check_max_stack_depth()
1580 idx = find_subprog(env, i); in check_max_stack_depth()
1606 static int get_callee_stack_depth(struct bpf_verifier_env *env, in get_callee_stack_depth() argument
1611 subprog = find_subprog(env, start); in get_callee_stack_depth()
1617 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
1621 static int check_ctx_reg(struct bpf_verifier_env *env, in check_ctx_reg() argument
1629 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", in check_ctx_reg()
1638 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); in check_ctx_reg()
1674 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, in check_mem_access() argument
1678 struct bpf_reg_state *regs = cur_regs(env); in check_mem_access()
1688 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); in check_mem_access()
1697 is_pointer_value(env, value_regno)) { in check_mem_access()
1698 verbose(env, "R%d leaks addr into map\n", value_regno); in check_mem_access()
1702 err = check_map_access(env, regno, off, size, false); in check_mem_access()
1704 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
1710 is_pointer_value(env, value_regno)) { in check_mem_access()
1711 verbose(env, "R%d leaks addr into ctx\n", value_regno); in check_mem_access()
1715 err = check_ctx_reg(env, reg, regno); in check_mem_access()
1719 err = check_ctx_access(env, insn_idx, off, size, t, ®_type); in check_mem_access()
1726 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
1728 mark_reg_known_zero(env, regs, in check_mem_access()
1745 verbose(env, "variable stack access var_off=%s off=%d size=%d", in check_mem_access()
1751 verbose(env, "invalid stack off=%d size=%d\n", off, in check_mem_access()
1756 state = func(env, reg); in check_mem_access()
1757 err = update_stack_depth(env, state, off); in check_mem_access()
1762 err = check_stack_write(env, state, off, size, in check_mem_access()
1765 err = check_stack_read(env, state, off, size, in check_mem_access()
1768 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { in check_mem_access()
1769 verbose(env, "cannot write into packet\n"); in check_mem_access()
1773 is_pointer_value(env, value_regno)) { in check_mem_access()
1774 verbose(env, "R%d leaks addr into packet\n", in check_mem_access()
1778 err = check_packet_access(env, regno, off, size, false); in check_mem_access()
1780 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
1782 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
1795 static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) in check_xadd() argument
1801 verbose(env, "BPF_XADD uses reserved fields\n"); in check_xadd()
1806 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_xadd()
1811 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_xadd()
1815 if (is_pointer_value(env, insn->src_reg)) { in check_xadd()
1816 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); in check_xadd()
1820 if (is_ctx_reg(env, insn->dst_reg) || in check_xadd()
1821 is_pkt_reg(env, insn->dst_reg)) { in check_xadd()
1822 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", in check_xadd()
1823 insn->dst_reg, is_ctx_reg(env, insn->dst_reg) ? in check_xadd()
1829 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_xadd()
1835 return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_xadd()
1845 static int check_stack_boundary(struct bpf_verifier_env *env, int regno, in check_stack_boundary() argument
1849 struct bpf_reg_state *reg = cur_regs(env) + regno; in check_stack_boundary()
1850 struct bpf_func_state *state = func(env, reg); in check_stack_boundary()
1859 verbose(env, "R%d type=%s expected=%s\n", regno, in check_stack_boundary()
1870 verbose(env, "invalid variable stack read R%d var_off=%s\n", in check_stack_boundary()
1877 verbose(env, "invalid stack type R%d off=%d access_size=%d\n", in check_stack_boundary()
1904 verbose(env, "invalid indirect read from stack off %d+%d size %d\n", in check_stack_boundary()
1911 mark_stack_slot_read(env, env->cur_state, env->cur_state->parent, in check_stack_boundary()
1914 return update_stack_depth(env, state, off); in check_stack_boundary()
1917 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, in check_helper_mem_access() argument
1921 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_helper_mem_access()
1926 return check_packet_access(env, regno, reg->off, access_size, in check_helper_mem_access()
1929 return check_map_access(env, regno, reg->off, access_size, in check_helper_mem_access()
1932 return check_stack_boundary(env, regno, access_size, in check_helper_mem_access()
1950 static int check_func_arg(struct bpf_verifier_env *env, u32 regno, in check_func_arg() argument
1954 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_func_arg()
1961 err = check_reg_arg(env, regno, SRC_OP); in check_func_arg()
1966 if (is_pointer_value(env, regno)) { in check_func_arg()
1967 verbose(env, "R%d leaks addr into helper function\n", in check_func_arg()
1975 !may_access_direct_pkt_data(env, meta, BPF_READ)) { in check_func_arg()
1976 verbose(env, "helper access to the packet is not allowed\n"); in check_func_arg()
1999 err = check_ctx_reg(env, reg, regno); in check_func_arg()
2017 verbose(env, "unsupported arg_type %d\n", arg_type); in check_func_arg()
2035 verbose(env, "invalid map_ptr to access map->key\n"); in check_func_arg()
2038 err = check_helper_mem_access(env, regno, in check_func_arg()
2047 verbose(env, "invalid map_ptr to access map->value\n"); in check_func_arg()
2050 err = check_helper_mem_access(env, regno, in check_func_arg()
2074 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", in check_func_arg()
2080 err = check_helper_mem_access(env, regno - 1, 0, in check_func_arg()
2088 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", in check_func_arg()
2092 err = check_helper_mem_access(env, regno - 1, in check_func_arg()
2099 verbose(env, "R%d type=%s expected=%s\n", regno, in check_func_arg()
2104 static int check_map_func_compatibility(struct bpf_verifier_env *env, in check_map_func_compatibility() argument
2183 if (env->subprog_cnt > 1) { in check_map_func_compatibility()
2184 verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n"); in check_map_func_compatibility()
2235 verbose(env, "cannot pass map_type %d into func %s#%d\n", in check_map_func_compatibility()
2298 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, in __clear_all_pkt_pointers() argument
2306 mark_reg_unknown(env, regs, i); in __clear_all_pkt_pointers()
2317 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) in clear_all_pkt_pointers() argument
2319 struct bpf_verifier_state *vstate = env->cur_state; in clear_all_pkt_pointers()
2323 __clear_all_pkt_pointers(env, vstate->frame[i]); in clear_all_pkt_pointers()
2326 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_func_call() argument
2329 struct bpf_verifier_state *state = env->cur_state; in check_func_call()
2334 verbose(env, "the call stack of %d frames is too deep\n", in check_func_call()
2340 subprog = find_subprog(env, target_insn + 1); in check_func_call()
2342 verbose(env, "verifier bug. No program starts at insn %d\n", in check_func_call()
2349 verbose(env, "verifier bug. Frame %d already allocated\n", in check_func_call()
2363 init_func_state(env, callee, in check_func_call()
2375 mark_reg_not_init(env, caller->regs, caller_saved[i]); in check_func_call()
2376 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_func_call()
2385 if (env->log.level) { in check_func_call()
2386 verbose(env, "caller:\n"); in check_func_call()
2387 print_verifier_state(env, caller); in check_func_call()
2388 verbose(env, "callee:\n"); in check_func_call()
2389 print_verifier_state(env, callee); in check_func_call()
2394 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) in prepare_func_exit() argument
2396 struct bpf_verifier_state *state = env->cur_state; in prepare_func_exit()
2409 verbose(env, "cannot return stack pointer to the caller\n"); in prepare_func_exit()
2419 if (env->log.level) { in prepare_func_exit()
2420 verbose(env, "returning from callee:\n"); in prepare_func_exit()
2421 print_verifier_state(env, callee); in prepare_func_exit()
2422 verbose(env, "to caller at %d:\n", *insn_idx); in prepare_func_exit()
2423 print_verifier_state(env, caller); in prepare_func_exit()
2449 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_map() argument
2452 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map()
2461 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_map()
2474 static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) in check_helper_call() argument
2484 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), in check_helper_call()
2489 if (env->ops->get_func_proto) in check_helper_call()
2490 fn = env->ops->get_func_proto(func_id, env->prog); in check_helper_call()
2492 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), in check_helper_call()
2498 if (!env->prog->gpl_compatible && fn->gpl_only) { in check_helper_call()
2499 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); in check_helper_call()
2506 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", in check_helper_call()
2516 verbose(env, "kernel subsystem misconfigured func %s#%d\n", in check_helper_call()
2522 err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta); in check_helper_call()
2525 err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta); in check_helper_call()
2528 err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta); in check_helper_call()
2531 err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta); in check_helper_call()
2534 err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta); in check_helper_call()
2538 err = record_func_map(env, &meta, func_id, insn_idx); in check_helper_call()
2546 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, in check_helper_call()
2552 regs = cur_regs(env); in check_helper_call()
2559 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); in check_helper_call()
2565 mark_reg_not_init(env, regs, caller_saved[i]); in check_helper_call()
2566 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_helper_call()
2572 mark_reg_unknown(env, regs, BPF_REG_0); in check_helper_call()
2582 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
2589 verbose(env, in check_helper_call()
2594 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
2596 verbose(env, "unknown return type %d of func %s#%d\n", in check_helper_call()
2603 err = check_map_func_compatibility(env, meta.map_ptr, func_id); in check_helper_call()
2607 if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) { in check_helper_call()
2618 verbose(env, err_str, func_id_name(func_id), func_id); in check_helper_call()
2622 env->prog->has_callchain_buf = true; in check_helper_call()
2626 clear_all_pkt_pointers(env); in check_helper_call()
2650 static bool check_reg_sane_offset(struct bpf_verifier_env *env, in check_reg_sane_offset() argument
2659 verbose(env, "math between %s pointer and %lld is not allowed\n", in check_reg_sane_offset()
2665 verbose(env, "%s pointer offset %d is not allowed\n", in check_reg_sane_offset()
2671 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", in check_reg_sane_offset()
2677 verbose(env, "value %lld makes %s pointer be out of bounds\n", in check_reg_sane_offset()
2690 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, in adjust_ptr_min_max_vals() argument
2695 struct bpf_verifier_state *vstate = env->cur_state; in adjust_ptr_min_max_vals()
2719 verbose(env, in adjust_ptr_min_max_vals()
2726 …verbose(env, "R%d pointer arithmetic on PTR_TO_MAP_VALUE_OR_NULL prohibited, null-check it first\n… in adjust_ptr_min_max_vals()
2731 verbose(env, "R%d pointer arithmetic on CONST_PTR_TO_MAP prohibited\n", in adjust_ptr_min_max_vals()
2736 verbose(env, "R%d pointer arithmetic on PTR_TO_PACKET_END prohibited\n", in adjust_ptr_min_max_vals()
2747 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || in adjust_ptr_min_max_vals()
2748 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
2796 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
2804 verbose(env, "R%d tried to subtract pointer from scalar\n", in adjust_ptr_min_max_vals()
2813 verbose(env, "R%d subtraction from stack pointer prohibited\n", in adjust_ptr_min_max_vals()
2854 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
2864 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", in adjust_ptr_min_max_vals()
2869 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", in adjust_ptr_min_max_vals()
2874 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
2887 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, in adjust_scalar_min_max_vals() argument
2892 struct bpf_reg_state *regs = cur_regs(env); in adjust_scalar_min_max_vals()
3062 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
3087 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
3117 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
3136 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
3153 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, in adjust_reg_min_max_vals() argument
3156 struct bpf_verifier_state *vstate = env->cur_state; in adjust_reg_min_max_vals()
3174 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_reg_min_max_vals()
3175 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
3178 verbose(env, "R%d pointer %s pointer prohibited\n", in adjust_reg_min_max_vals()
3187 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
3192 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
3203 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
3209 print_verifier_state(env, state); in adjust_reg_min_max_vals()
3210 verbose(env, "verifier internal error: unexpected ptr_reg\n"); in adjust_reg_min_max_vals()
3214 print_verifier_state(env, state); in adjust_reg_min_max_vals()
3215 verbose(env, "verifier internal error: no src_reg\n"); in adjust_reg_min_max_vals()
3218 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
3222 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_alu_op() argument
3224 struct bpf_reg_state *regs = cur_regs(env); in check_alu_op()
3233 verbose(env, "BPF_NEG uses reserved fields\n"); in check_alu_op()
3240 verbose(env, "BPF_END uses reserved fields\n"); in check_alu_op()
3246 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
3250 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
3251 verbose(env, "R%d pointer arithmetic prohibited\n", in check_alu_op()
3257 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
3265 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
3270 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
3275 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
3281 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
3294 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
3295 verbose(env, in check_alu_op()
3300 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
3308 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
3320 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); in check_alu_op()
3327 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
3331 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
3336 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
3342 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
3348 verbose(env, "div by zero\n"); in check_alu_op()
3353 verbose(env, "BPF_ARSH not supported for 32 bit ALU\n"); in check_alu_op()
3362 verbose(env, "invalid shift %d\n", insn->imm); in check_alu_op()
3368 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
3372 return adjust_reg_min_max_vals(env, insn); in check_alu_op()
3816 static int check_cond_jmp_op(struct bpf_verifier_env *env, in check_cond_jmp_op() argument
3819 struct bpf_verifier_state *this_branch = env->cur_state; in check_cond_jmp_op()
3827 verbose(env, "invalid BPF_JMP opcode %x\n", opcode); in check_cond_jmp_op()
3833 verbose(env, "BPF_JMP uses reserved fields\n"); in check_cond_jmp_op()
3838 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_cond_jmp_op()
3842 if (is_pointer_value(env, insn->src_reg)) { in check_cond_jmp_op()
3843 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
3849 verbose(env, "BPF_JMP uses reserved fields\n"); in check_cond_jmp_op()
3855 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
3882 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx); in check_cond_jmp_op()
3928 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
3929 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
3933 if (env->log.level) in check_cond_jmp_op()
3934 print_verifier_state(env, this_branch->frame[this_branch->curframe]); in check_cond_jmp_op()
3947 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_imm() argument
3949 struct bpf_reg_state *regs = cur_regs(env); in check_ld_imm()
3953 verbose(env, "invalid BPF_LD_IMM insn\n"); in check_ld_imm()
3957 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); in check_ld_imm()
3961 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
4008 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_abs() argument
4010 struct bpf_reg_state *regs = cur_regs(env); in check_ld_abs()
4014 if (!may_access_skb(env->prog->type)) { in check_ld_abs()
4015 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); in check_ld_abs()
4019 if (!env->ops->gen_ld_abs) { in check_ld_abs()
4020 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_abs()
4024 if (env->subprog_cnt > 1) { in check_ld_abs()
4032 verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n"); in check_ld_abs()
4039 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); in check_ld_abs()
4044 err = check_reg_arg(env, BPF_REG_6, SRC_OP); in check_ld_abs()
4049 verbose(env, in check_ld_abs()
4056 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_ld_abs()
4063 mark_reg_not_init(env, regs, caller_saved[i]); in check_ld_abs()
4064 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_ld_abs()
4071 mark_reg_unknown(env, regs, BPF_REG_0); in check_ld_abs()
4075 static int check_return_code(struct bpf_verifier_env *env) in check_return_code() argument
4080 switch (env->prog->type) { in check_return_code()
4091 reg = cur_regs(env) + BPF_REG_0; in check_return_code()
4093 verbose(env, "At program exit the register R0 is not a known value (%s)\n", in check_return_code()
4099 verbose(env, "At program exit the register R0 "); in check_return_code()
4104 verbose(env, "has value %s", tn_buf); in check_return_code()
4106 verbose(env, "has unknown scalar value"); in check_return_code()
4108 verbose(env, " should have been 0 or 1\n"); in check_return_code()
4165 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) in push_insn() argument
4173 if (w < 0 || w >= env->prog->len) { in push_insn()
4174 verbose(env, "jump out of range from insn %d to %d\n", t, w); in push_insn()
4180 env->explored_states[w] = STATE_LIST_MARK; in push_insn()
4186 if (cur_stack >= env->prog->len) in push_insn()
4191 verbose(env, "back-edge from insn %d to %d\n", t, w); in push_insn()
4197 verbose(env, "insn state internal bug\n"); in push_insn()
4206 static int check_cfg(struct bpf_verifier_env *env) in check_cfg() argument
4208 struct bpf_insn *insns = env->prog->insnsi; in check_cfg()
4209 int insn_cnt = env->prog->len; in check_cfg()
4213 ret = check_subprogs(env); in check_cfg()
4242 ret = push_insn(t, t + 1, FALLTHROUGH, env); in check_cfg()
4248 env->explored_states[t + 1] = STATE_LIST_MARK; in check_cfg()
4250 env->explored_states[t] = STATE_LIST_MARK; in check_cfg()
4251 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env); in check_cfg()
4264 FALLTHROUGH, env); in check_cfg()
4273 env->explored_states[t + 1] = STATE_LIST_MARK; in check_cfg()
4276 env->explored_states[t] = STATE_LIST_MARK; in check_cfg()
4277 ret = push_insn(t, t + 1, FALLTHROUGH, env); in check_cfg()
4283 ret = push_insn(t, t + insns[t].off + 1, BRANCH, env); in check_cfg()
4293 ret = push_insn(t, t + 1, FALLTHROUGH, env); in check_cfg()
4303 verbose(env, "pop stack internal bug\n"); in check_cfg()
4312 verbose(env, "unreachable insn %d\n", i); in check_cfg()
4588 static bool states_equal(struct bpf_verifier_env *env, in states_equal() argument
4616 static int propagate_liveness(struct bpf_verifier_env *env, in propagate_liveness() argument
4635 err = mark_reg_read(env, vstate, vparent, i); in propagate_liveness()
4650 mark_stack_slot_read(env, vstate, vparent, i, frame); in propagate_liveness()
4656 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) in is_state_visited() argument
4660 struct bpf_verifier_state *cur = env->cur_state; in is_state_visited()
4663 sl = env->explored_states[insn_idx]; in is_state_visited()
4671 if (states_equal(env, &sl->state, cur)) { in is_state_visited()
4682 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
4708 new_sl->next = env->explored_states[insn_idx]; in is_state_visited()
4709 env->explored_states[insn_idx] = new_sl; in is_state_visited()
4731 static int do_check(struct bpf_verifier_env *env) in do_check() argument
4734 struct bpf_insn *insns = env->prog->insnsi; in do_check()
4736 int insn_cnt = env->prog->len, i; in do_check()
4751 env->cur_state = state; in do_check()
4752 init_func_state(env, state->frame[0], in do_check()
4763 verbose(env, "invalid insn idx %d insn_cnt %d\n", in do_check()
4772 verbose(env, in do_check()
4778 err = is_state_visited(env, insn_idx); in do_check()
4783 if (env->log.level) { in do_check()
4785 verbose(env, "\nfrom %d to %d: safe\n", in do_check()
4788 verbose(env, "%d: safe\n", insn_idx); in do_check()
4796 if (env->log.level > 1 || (env->log.level && do_print_state)) { in do_check()
4797 if (env->log.level > 1) in do_check()
4798 verbose(env, "%d:", insn_idx); in do_check()
4800 verbose(env, "\nfrom %d to %d:", in do_check()
4802 print_verifier_state(env, state->frame[state->curframe]); in do_check()
4806 if (env->log.level) { in do_check()
4809 .private_data = env, in do_check()
4812 verbose(env, "%d: ", insn_idx); in do_check()
4813 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in do_check()
4816 if (bpf_prog_is_dev_bound(env->prog->aux)) { in do_check()
4817 err = bpf_prog_offload_verify_insn(env, insn_idx, in do_check()
4823 regs = cur_regs(env); in do_check()
4824 env->insn_aux_data[insn_idx].seen = true; in do_check()
4826 err = check_alu_op(env, insn); in do_check()
4836 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
4840 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
4849 err = check_mem_access(env, insn_idx, insn->src_reg, insn->off, in do_check()
4855 prev_src_type = &env->insn_aux_data[insn_idx].ptr_type; in do_check()
4874 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
4882 err = check_xadd(env, insn_idx, insn); in do_check()
4890 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
4894 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
4901 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in do_check()
4907 prev_dst_type = &env->insn_aux_data[insn_idx].ptr_type; in do_check()
4914 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
4921 verbose(env, "BPF_ST uses reserved fields\n"); in do_check()
4925 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
4929 if (is_ctx_reg(env, insn->dst_reg)) { in do_check()
4930 verbose(env, "BPF_ST stores into R%d context is not allowed\n", in do_check()
4936 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in do_check()
4951 verbose(env, "BPF_CALL uses reserved fields\n"); in do_check()
4956 err = check_func_call(env, insn, &insn_idx); in do_check()
4958 err = check_helper_call(env, insn->imm, insn_idx); in do_check()
4967 verbose(env, "BPF_JA uses reserved fields\n"); in do_check()
4979 verbose(env, "BPF_EXIT uses reserved fields\n"); in do_check()
4986 err = prepare_func_exit(env, &insn_idx); in do_check()
4999 err = check_reg_arg(env, BPF_REG_0, SRC_OP); in do_check()
5003 if (is_pointer_value(env, BPF_REG_0)) { in do_check()
5004 verbose(env, "R0 leaks addr as return value\n"); in do_check()
5008 err = check_return_code(env); in do_check()
5012 err = pop_stack(env, &prev_insn_idx, &insn_idx); in do_check()
5022 err = check_cond_jmp_op(env, insn, &insn_idx); in do_check()
5030 err = check_ld_abs(env, insn); in do_check()
5035 err = check_ld_imm(env, insn); in do_check()
5040 env->insn_aux_data[insn_idx].seen = true; in do_check()
5042 verbose(env, "invalid BPF_LD mode\n"); in do_check()
5046 verbose(env, "unknown insn class %d\n", class); in do_check()
5053 verbose(env, "processed %d insns (limit %d), stack depth ", in do_check()
5055 for (i = 0; i < env->subprog_cnt; i++) { in do_check()
5056 u32 depth = env->subprog_info[i].stack_depth; in do_check()
5058 verbose(env, "%d", depth); in do_check()
5059 if (i + 1 < env->subprog_cnt) in do_check()
5060 verbose(env, "+"); in do_check()
5062 verbose(env, "\n"); in do_check()
5063 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check()
5075 static int check_map_prog_compatibility(struct bpf_verifier_env *env, in check_map_prog_compatibility() argument
5087 verbose(env, "perf_event programs can only use preallocated hash map\n"); in check_map_prog_compatibility()
5092 verbose(env, "perf_event programs can only use preallocated inner hash map\n"); in check_map_prog_compatibility()
5099 verbose(env, "offload device mismatch between prog and map\n"); in check_map_prog_compatibility()
5109 static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) in replace_map_fd_with_map_ptr() argument
5111 struct bpf_insn *insn = env->prog->insnsi; in replace_map_fd_with_map_ptr()
5112 int insn_cnt = env->prog->len; in replace_map_fd_with_map_ptr()
5115 err = bpf_prog_calc_tag(env->prog); in replace_map_fd_with_map_ptr()
5122 verbose(env, "BPF_LDX uses reserved fields\n"); in replace_map_fd_with_map_ptr()
5129 verbose(env, "BPF_STX uses reserved fields\n"); in replace_map_fd_with_map_ptr()
5140 verbose(env, "invalid bpf_ld_imm64 insn\n"); in replace_map_fd_with_map_ptr()
5149 verbose(env, in replace_map_fd_with_map_ptr()
5157 verbose(env, "fd %d is not pointing to valid bpf_map\n", in replace_map_fd_with_map_ptr()
5162 err = check_map_prog_compatibility(env, map, env->prog); in replace_map_fd_with_map_ptr()
5173 for (j = 0; j < env->used_map_cnt; j++) in replace_map_fd_with_map_ptr()
5174 if (env->used_maps[j] == map) { in replace_map_fd_with_map_ptr()
5179 if (env->used_map_cnt >= MAX_USED_MAPS) { in replace_map_fd_with_map_ptr()
5194 env->used_maps[env->used_map_cnt++] = map; in replace_map_fd_with_map_ptr()
5197 bpf_cgroup_storage_assign(env->prog, map)) { in replace_map_fd_with_map_ptr()
5198 verbose(env, in replace_map_fd_with_map_ptr()
5213 verbose(env, "unknown opcode %02x\n", insn->code); in replace_map_fd_with_map_ptr()
5226 static void release_maps(struct bpf_verifier_env *env) in release_maps() argument
5230 if (env->prog->aux->cgroup_storage) in release_maps()
5231 bpf_cgroup_storage_release(env->prog, in release_maps()
5232 env->prog->aux->cgroup_storage); in release_maps()
5234 for (i = 0; i < env->used_map_cnt; i++) in release_maps()
5235 bpf_map_put(env->used_maps[i]); in release_maps()
5239 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) in convert_pseudo_ld_imm64() argument
5241 struct bpf_insn *insn = env->prog->insnsi; in convert_pseudo_ld_imm64()
5242 int insn_cnt = env->prog->len; in convert_pseudo_ld_imm64()
5254 static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len, in adjust_insn_aux_data() argument
5257 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; in adjust_insn_aux_data()
5271 env->insn_aux_data = new_data; in adjust_insn_aux_data()
5276 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) in adjust_subprog_starts() argument
5283 for (i = 0; i <= env->subprog_cnt; i++) { in adjust_subprog_starts()
5284 if (env->subprog_info[i].start < off) in adjust_subprog_starts()
5286 env->subprog_info[i].start += len - 1; in adjust_subprog_starts()
5290 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, in bpf_patch_insn_data() argument
5295 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); in bpf_patch_insn_data()
5298 if (adjust_insn_aux_data(env, new_prog->len, off, len)) in bpf_patch_insn_data()
5300 adjust_subprog_starts(env, off, len); in bpf_patch_insn_data()
5315 static void sanitize_dead_code(struct bpf_verifier_env *env) in sanitize_dead_code() argument
5317 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in sanitize_dead_code()
5319 struct bpf_insn *insn = env->prog->insnsi; in sanitize_dead_code()
5320 const int insn_cnt = env->prog->len; in sanitize_dead_code()
5333 static int convert_ctx_accesses(struct bpf_verifier_env *env) in convert_ctx_accesses() argument
5335 const struct bpf_verifier_ops *ops = env->ops; in convert_ctx_accesses()
5337 const int insn_cnt = env->prog->len; in convert_ctx_accesses()
5345 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in convert_ctx_accesses()
5346 env->prog); in convert_ctx_accesses()
5348 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
5351 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
5355 env->prog = new_prog; in convert_ctx_accesses()
5360 if (!ops->convert_ctx_access || bpf_prog_is_dev_bound(env->prog->aux)) in convert_ctx_accesses()
5363 insn = env->prog->insnsi + delta; in convert_ctx_accesses()
5380 env->insn_aux_data[i + delta].sanitize_stack_off) { in convert_ctx_accesses()
5388 env->insn_aux_data[i + delta].sanitize_stack_off, in convert_ctx_accesses()
5397 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); in convert_ctx_accesses()
5402 env->prog = new_prog; in convert_ctx_accesses()
5407 if (env->insn_aux_data[i + delta].ptr_type != PTR_TO_CTX) in convert_ctx_accesses()
5410 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; in convert_ctx_accesses()
5425 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); in convert_ctx_accesses()
5440 cnt = ops->convert_ctx_access(type, insn, insn_buf, env->prog, in convert_ctx_accesses()
5444 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
5457 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in convert_ctx_accesses()
5464 env->prog = new_prog; in convert_ctx_accesses()
5471 static int jit_subprogs(struct bpf_verifier_env *env) in jit_subprogs() argument
5473 struct bpf_prog *prog = env->prog, **func, *tmp; in jit_subprogs()
5479 if (env->subprog_cnt <= 1) in jit_subprogs()
5490 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
5503 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
5508 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); in jit_subprogs()
5512 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
5514 subprog_end = env->subprog_info[i + 1].start; in jit_subprogs()
5531 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
5544 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
5568 func[i]->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
5570 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
5574 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); in jit_subprogs()
5584 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
5597 insn->off = env->insn_aux_data[i].call_imm; in jit_subprogs()
5598 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
5605 prog->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
5608 for (i = 0; i < env->subprog_cnt; i++) in jit_subprogs()
5620 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
5625 static int fixup_call_args(struct bpf_verifier_env *env) in fixup_call_args() argument
5628 struct bpf_prog *prog = env->prog; in fixup_call_args()
5635 if (env->prog->jit_requested) { in fixup_call_args()
5636 err = jit_subprogs(env); in fixup_call_args()
5647 depth = get_callee_stack_depth(env, insn, i); in fixup_call_args()
5662 static int fixup_bpf_calls(struct bpf_verifier_env *env) in fixup_bpf_calls() argument
5664 struct bpf_prog *prog = env->prog; in fixup_bpf_calls()
5706 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in fixup_bpf_calls()
5711 env->prog = prog = new_prog; in fixup_bpf_calls()
5719 cnt = env->ops->gen_ld_abs(insn, insn_buf); in fixup_bpf_calls()
5721 verbose(env, "bpf verifier is misconfigured\n"); in fixup_bpf_calls()
5725 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in fixup_bpf_calls()
5730 env->prog = prog = new_prog; in fixup_bpf_calls()
5753 env->prog->aux->stack_depth = MAX_BPF_STACK; in fixup_bpf_calls()
5763 aux = &env->insn_aux_data[i + delta]; in fixup_bpf_calls()
5774 verbose(env, "tail_call abusing map_ptr\n"); in fixup_bpf_calls()
5787 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in fixup_bpf_calls()
5792 env->prog = prog = new_prog; in fixup_bpf_calls()
5805 aux = &env->insn_aux_data[i + delta]; in fixup_bpf_calls()
5815 verbose(env, "bpf verifier is misconfigured\n"); in fixup_bpf_calls()
5819 new_prog = bpf_patch_insn_data(env, i + delta, in fixup_bpf_calls()
5825 env->prog = prog = new_prog; in fixup_bpf_calls()
5856 fn = env->ops->get_func_proto(insn->imm, env->prog); in fixup_bpf_calls()
5861 verbose(env, in fixup_bpf_calls()
5872 static void free_states(struct bpf_verifier_env *env) in free_states() argument
5877 if (!env->explored_states) in free_states()
5880 for (i = 0; i < env->prog->len; i++) { in free_states()
5881 sl = env->explored_states[i]; in free_states()
5892 kfree(env->explored_states); in free_states()
5897 struct bpf_verifier_env *env; in bpf_check() local
5908 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); in bpf_check()
5909 if (!env) in bpf_check()
5911 log = &env->log; in bpf_check()
5913 env->insn_aux_data = in bpf_check()
5917 if (!env->insn_aux_data) in bpf_check()
5919 env->prog = *prog; in bpf_check()
5920 env->ops = bpf_verifier_ops[env->prog->type]; in bpf_check()
5940 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); in bpf_check()
5942 env->strict_alignment = true; in bpf_check()
5944 ret = replace_map_fd_with_map_ptr(env); in bpf_check()
5948 if (bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
5949 ret = bpf_prog_offload_verifier_prep(env); in bpf_check()
5954 env->explored_states = kcalloc(env->prog->len, in bpf_check()
5958 if (!env->explored_states) in bpf_check()
5961 env->allow_ptr_leaks = capable(CAP_SYS_ADMIN); in bpf_check()
5963 ret = check_cfg(env); in bpf_check()
5967 ret = do_check(env); in bpf_check()
5968 if (env->cur_state) { in bpf_check()
5969 free_verifier_state(env->cur_state, true); in bpf_check()
5970 env->cur_state = NULL; in bpf_check()
5974 while (!pop_stack(env, NULL, NULL)); in bpf_check()
5975 free_states(env); in bpf_check()
5978 sanitize_dead_code(env); in bpf_check()
5981 ret = check_max_stack_depth(env); in bpf_check()
5985 ret = convert_ctx_accesses(env); in bpf_check()
5988 ret = fixup_bpf_calls(env); in bpf_check()
5991 ret = fixup_call_args(env); in bpf_check()
6000 if (ret == 0 && env->used_map_cnt) { in bpf_check()
6002 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, in bpf_check()
6003 sizeof(env->used_maps[0]), in bpf_check()
6006 if (!env->prog->aux->used_maps) { in bpf_check()
6011 memcpy(env->prog->aux->used_maps, env->used_maps, in bpf_check()
6012 sizeof(env->used_maps[0]) * env->used_map_cnt); in bpf_check()
6013 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
6018 convert_pseudo_ld_imm64(env); in bpf_check()
6022 if (!env->prog->aux->used_maps) in bpf_check()
6026 release_maps(env); in bpf_check()
6027 *prog = env->prog; in bpf_check()
6030 vfree(env->insn_aux_data); in bpf_check()
6032 kfree(env); in bpf_check()