272 find_linfo(const struct bpf_verifier_env *env, u32 insn_off)  in find_linfo()  argument
278 	prog = env->prog;  in find_linfo()
331 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,  in bpf_verifier_log_write()  argument
336 	if (!bpf_verifier_log_needed(&env->log))  in bpf_verifier_log_write()
340 	bpf_verifier_vlog(&env->log, fmt, args);  in bpf_verifier_log_write()
347 	struct bpf_verifier_env *env = private_data;  in verbose()  local
350 	if (!bpf_verifier_log_needed(&env->log))  in verbose()
354 	bpf_verifier_vlog(&env->log, fmt, args);  in verbose()
379 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,  in verbose_linfo()  argument
385 	if (!bpf_verifier_log_needed(&env->log))  in verbose_linfo()
388 	linfo = find_linfo(env, insn_off);  in verbose_linfo()
389 	if (!linfo || linfo == env->prev_linfo)  in verbose_linfo()
396 		bpf_verifier_vlog(&env->log, prefix_fmt, args);  in verbose_linfo()
400 	verbose(env, "%s\n",  in verbose_linfo()
401 		ltrim(btf_name_by_offset(env->prog->aux->btf,  in verbose_linfo()
404 	env->prev_linfo = linfo;  in verbose_linfo()
407 static void verbose_invalid_scalar(struct bpf_verifier_env *env,  in verbose_invalid_scalar()  argument
414 	verbose(env, "At %s the register %s ", ctx, reg_name);  in verbose_invalid_scalar()
417 		verbose(env, "has value %s", tn_buf);  in verbose_invalid_scalar()
419 		verbose(env, "has unknown scalar value");  in verbose_invalid_scalar()
422 	verbose(env, " should have been in %s\n", tn_buf);  in verbose_invalid_scalar()
589 static void print_liveness(struct bpf_verifier_env *env,  in print_liveness()  argument
593 	    verbose(env, "_");  in print_liveness()
595 		verbose(env, "r");  in print_liveness()
597 		verbose(env, "w");  in print_liveness()
599 		verbose(env, "D");  in print_liveness()
602 static struct bpf_func_state *func(struct bpf_verifier_env *env,  in func()  argument
605 	struct bpf_verifier_state *cur = env->cur_state;  in func()
615 static void print_verifier_state(struct bpf_verifier_env *env,  in print_verifier_state()  argument
623 		verbose(env, " frame%d:", state->frameno);  in print_verifier_state()
629 		verbose(env, " R%d", i);  in print_verifier_state()
630 		print_liveness(env, reg->live);  in print_verifier_state()
631 		verbose(env, "=%s", reg_type_str[t]);  in print_verifier_state()
633 			verbose(env, "P");  in print_verifier_state()
637 			verbose(env, "%lld", reg->var_off.value + reg->off);  in print_verifier_state()
642 				verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));  in print_verifier_state()
643 			verbose(env, "(id=%d", reg->id);  in print_verifier_state()
645 				verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);  in print_verifier_state()
647 				verbose(env, ",off=%d", reg->off);  in print_verifier_state()
649 				verbose(env, ",r=%d", reg->range);  in print_verifier_state()
654 				verbose(env, ",ks=%d,vs=%d",  in print_verifier_state()
662 				verbose(env, ",imm=%llx", reg->var_off.value);  in print_verifier_state()
666 					verbose(env, ",smin_value=%lld",  in print_verifier_state()
670 					verbose(env, ",smax_value=%lld",  in print_verifier_state()
673 					verbose(env, ",umin_value=%llu",  in print_verifier_state()
676 					verbose(env, ",umax_value=%llu",  in print_verifier_state()
682 					verbose(env, ",var_off=%s", tn_buf);  in print_verifier_state()
686 					verbose(env, ",s32_min_value=%d",  in print_verifier_state()
690 					verbose(env, ",s32_max_value=%d",  in print_verifier_state()
694 					verbose(env, ",u32_min_value=%d",  in print_verifier_state()
698 					verbose(env, ",u32_max_value=%d",  in print_verifier_state()
701 			verbose(env, ")");  in print_verifier_state()
718 		verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);  in print_verifier_state()
719 		print_liveness(env, state->stack[i].spilled_ptr.live);  in print_verifier_state()
723 			verbose(env, "=%s", reg_type_str[t]);  in print_verifier_state()
725 				verbose(env, "P");  in print_verifier_state()
727 				verbose(env, "%lld", reg->var_off.value + reg->off);  in print_verifier_state()
729 			verbose(env, "=%s", types_buf);  in print_verifier_state()
733 		verbose(env, " refs=%d", state->refs[0].id);  in print_verifier_state()
736 				verbose(env, ",%d", state->refs[i].id);  in print_verifier_state()
739 		verbose(env, " cb");  in print_verifier_state()
741 		verbose(env, " async_cb");  in print_verifier_state()
742 	verbose(env, "\n");  in print_verifier_state()
850 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)  in acquire_reference_state()  argument
852 	struct bpf_func_state *state = cur_func(env);  in acquire_reference_state()
859 	id = ++env->id_gen;  in acquire_reference_state()
970 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)  in update_branch_counts()  argument
987 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,  in pop_stack()  argument
990 	struct bpf_verifier_state *cur = env->cur_state;  in pop_stack()
991 	struct bpf_verifier_stack_elem *elem, *head = env->head;  in pop_stack()
994 	if (env->head == NULL)  in pop_stack()
1003 		bpf_vlog_reset(&env->log, head->log_pos);  in pop_stack()
1011 	env->head = elem;  in pop_stack()
1012 	env->stack_size--;  in pop_stack()
1016 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,  in push_stack()  argument
1020 	struct bpf_verifier_state *cur = env->cur_state;  in push_stack()
1030 	elem->next = env->head;  in push_stack()
1031 	elem->log_pos = env->log.len_used;  in push_stack()
1032 	env->head = elem;  in push_stack()
1033 	env->stack_size++;  in push_stack()
1038 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {  in push_stack()
1039 		verbose(env, "The sequence of %d jumps is too complex.\n",  in push_stack()
1040 			env->stack_size);  in push_stack()
1057 	free_verifier_state(env->cur_state, true);  in push_stack()
1058 	env->cur_state = NULL;  in push_stack()
1060 	while (!pop_stack(env, NULL, NULL, false));  in push_stack()
1069 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1121 static void mark_reg_known_zero(struct bpf_verifier_env *env,  in mark_reg_known_zero()  argument
1125 		verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);  in mark_reg_known_zero()
1128 			__mark_reg_not_init(env, regs + regno);  in mark_reg_known_zero()
1440 static void __mark_reg_unknown(const struct bpf_verifier_env *env,  in __mark_reg_unknown()  argument
1451 	reg->precise = env->subprog_cnt > 1 || !env->bpf_capable;  in __mark_reg_unknown()
1455 static void mark_reg_unknown(struct bpf_verifier_env *env,  in mark_reg_unknown()  argument
1459 		verbose(env, "mark_reg_unknown(regs, %u)\n", regno);  in mark_reg_unknown()
1462 			__mark_reg_not_init(env, regs + regno);  in mark_reg_unknown()
1465 	__mark_reg_unknown(env, regs + regno);  in mark_reg_unknown()
1468 static void __mark_reg_not_init(const struct bpf_verifier_env *env,  in __mark_reg_not_init()  argument
1471 	__mark_reg_unknown(env, reg);  in __mark_reg_not_init()
1475 static void mark_reg_not_init(struct bpf_verifier_env *env,  in mark_reg_not_init()  argument
1479 		verbose(env, "mark_reg_not_init(regs, %u)\n", regno);  in mark_reg_not_init()
1482 			__mark_reg_not_init(env, regs + regno);  in mark_reg_not_init()
1485 	__mark_reg_not_init(env, regs + regno);  in mark_reg_not_init()
1488 static void mark_btf_ld_reg(struct bpf_verifier_env *env,  in mark_btf_ld_reg()  argument
1494 		mark_reg_unknown(env, regs, regno);  in mark_btf_ld_reg()
1497 	mark_reg_known_zero(env, regs, regno);  in mark_btf_ld_reg()
1504 static void init_reg_state(struct bpf_verifier_env *env,  in init_reg_state()  argument
1511 		mark_reg_not_init(env, regs, i);  in init_reg_state()
1519 	mark_reg_known_zero(env, regs, BPF_REG_FP);  in init_reg_state()
1524 static void init_func_state(struct bpf_verifier_env *env,  in init_func_state()  argument
1531 	init_reg_state(env, state);  in init_func_state()
1535 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,  in push_async_cb()  argument
1548 	elem->next = env->head;  in push_async_cb()
1549 	elem->log_pos = env->log.len_used;  in push_async_cb()
1550 	env->head = elem;  in push_async_cb()
1551 	env->stack_size++;  in push_async_cb()
1552 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {  in push_async_cb()
1553 		verbose(env,  in push_async_cb()
1555 			env->stack_size);  in push_async_cb()
1567 	init_func_state(env, frame,  in push_async_cb()
1574 	free_verifier_state(env->cur_state, true);  in push_async_cb()
1575 	env->cur_state = NULL;  in push_async_cb()
1577 	while (!pop_stack(env, NULL, NULL, false));  in push_async_cb()
1594 static int find_subprog(struct bpf_verifier_env *env, int off)  in find_subprog()  argument
1598 	p = bsearch(&off, env->subprog_info, env->subprog_cnt,  in find_subprog()
1599 		    sizeof(env->subprog_info[0]), cmp_subprogs);  in find_subprog()
1602 	return p - env->subprog_info;  in find_subprog()
1606 static int add_subprog(struct bpf_verifier_env *env, int off)  in add_subprog()  argument
1608 	int insn_cnt = env->prog->len;  in add_subprog()
1612 		verbose(env, "call to invalid destination\n");  in add_subprog()
1615 	ret = find_subprog(env, off);  in add_subprog()
1618 	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {  in add_subprog()
1619 		verbose(env, "too many subprograms\n");  in add_subprog()
1623 	env->subprog_info[env->subprog_cnt++].start = off;  in add_subprog()
1624 	sort(env->subprog_info, env->subprog_cnt,  in add_subprog()
1625 	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);  in add_subprog()
1626 	return env->subprog_cnt - 1;  in add_subprog()
1663 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)  in add_kfunc_call()  argument
1673 	prog_aux = env->prog->aux;  in add_kfunc_call()
1677 			verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");  in add_kfunc_call()
1681 		if (!env->prog->jit_requested) {  in add_kfunc_call()
1682 			verbose(env, "JIT is required for calling kernel function\n");  in add_kfunc_call()
1687 			verbose(env, "JIT does not support calling kernel function\n");  in add_kfunc_call()
1691 		if (!env->prog->gpl_compatible) {  in add_kfunc_call()
1692 			verbose(env, "cannot call kernel function from non-GPL compatible program\n");  in add_kfunc_call()
1702 	if (find_kfunc_desc(env->prog, func_id))  in add_kfunc_call()
1706 		verbose(env, "too many different kernel function calls\n");  in add_kfunc_call()
1712 		verbose(env, "kernel btf_id %u is not a function\n",  in add_kfunc_call()
1718 		verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",  in add_kfunc_call()
1726 		verbose(env, "cannot find address for kernel function %s\n",  in add_kfunc_call()
1734 	err = btf_distill_func_proto(&env->log, btf_vmlinux,  in add_kfunc_call()
1789 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)  in add_subprog_and_kfunc()  argument
1791 	struct bpf_subprog_info *subprog = env->subprog_info;  in add_subprog_and_kfunc()
1792 	struct bpf_insn *insn = env->prog->insnsi;  in add_subprog_and_kfunc()
1793 	int i, ret, insn_cnt = env->prog->len;  in add_subprog_and_kfunc()
1796 	ret = add_subprog(env, 0);  in add_subprog_and_kfunc()
1805 		if (!env->bpf_capable) {  in add_subprog_and_kfunc()
1806 …verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_AD…  in add_subprog_and_kfunc()
1811 			ret = add_subprog(env, i + insn->imm + 1);  in add_subprog_and_kfunc()
1816 			ret = add_subprog(env, i + insn->imm + 1);  in add_subprog_and_kfunc()
1818 			ret = add_kfunc_call(env, insn->imm);  in add_subprog_and_kfunc()
1828 	subprog[env->subprog_cnt].start = insn_cnt;  in add_subprog_and_kfunc()
1830 	if (env->log.level & BPF_LOG_LEVEL2)  in add_subprog_and_kfunc()
1831 		for (i = 0; i < env->subprog_cnt; i++)  in add_subprog_and_kfunc()
1832 			verbose(env, "func#%d @%d\n", i, subprog[i].start);  in add_subprog_and_kfunc()
1837 static int check_subprogs(struct bpf_verifier_env *env)  in check_subprogs()  argument
1840 	struct bpf_subprog_info *subprog = env->subprog_info;  in check_subprogs()
1841 	struct bpf_insn *insn = env->prog->insnsi;  in check_subprogs()
1842 	int insn_cnt = env->prog->len;  in check_subprogs()
1863 			verbose(env, "jump out of range from insn %d to %d\n", i, off);  in check_subprogs()
1874 				verbose(env, "last insn is not an exit or jmp\n");  in check_subprogs()
1879 			if (cur_subprog < env->subprog_cnt)  in check_subprogs()
1889 static int mark_reg_read(struct bpf_verifier_env *env,  in mark_reg_read()  argument
1901 			verbose(env, "verifier BUG type %s var_off %lld off %d\n",  in mark_reg_read()
1932 	if (env->longest_mark_read_walk < cnt)  in mark_reg_read()
1933 		env->longest_mark_read_walk = cnt;  in mark_reg_read()
1941 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,  in is_reg64()  argument
2049 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)  in insn_has_def32()  argument
2056 	return !is_reg64(env, insn, dst_reg, NULL, DST_OP);  in insn_has_def32()
2059 static void mark_insn_zext(struct bpf_verifier_env *env,  in mark_insn_zext()  argument
2067 	env->insn_aux_data[def_idx - 1].zext_dst = true;  in mark_insn_zext()
2072 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,  in check_reg_arg()  argument
2075 	struct bpf_verifier_state *vstate = env->cur_state;  in check_reg_arg()
2077 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;  in check_reg_arg()
2082 		verbose(env, "R%d is invalid\n", regno);  in check_reg_arg()
2087 	rw64 = is_reg64(env, insn, regno, reg, t);  in check_reg_arg()
2091 			verbose(env, "R%d !read_ok\n", regno);  in check_reg_arg()
2099 			mark_insn_zext(env, reg);  in check_reg_arg()
2101 		return mark_reg_read(env, reg, reg->parent,  in check_reg_arg()
2106 			verbose(env, "frame pointer is read only\n");  in check_reg_arg()
2110 		reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;  in check_reg_arg()
2112 			mark_reg_unknown(env, regs, regno);  in check_reg_arg()
2118 static int push_jmp_history(struct bpf_verifier_env *env,  in push_jmp_history()  argument
2128 	p[cnt - 1].idx = env->insn_idx;  in push_jmp_history()
2129 	p[cnt - 1].prev_idx = env->prev_insn_idx;  in push_jmp_history()
2167 static int backtrack_insn(struct bpf_verifier_env *env, int idx,  in backtrack_insn()  argument
2173 		.private_data	= env,  in backtrack_insn()
2175 	struct bpf_insn *insn = env->prog->insnsi + idx;  in backtrack_insn()
2185 	if (env->log.level & BPF_LOG_LEVEL) {  in backtrack_insn()
2186 		verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);  in backtrack_insn()
2187 		verbose(env, "%d: ", idx);  in backtrack_insn()
2188 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);  in backtrack_insn()
2244 			verbose(env, "BUG spi %d\n", spi);  in backtrack_insn()
2263 			verbose(env, "BUG spi %d\n", spi);  in backtrack_insn()
2282 				verbose(env, "BUG regs %x\n", *reg_mask);  in backtrack_insn()
2356 static void mark_all_scalars_precise(struct bpf_verifier_env *env,  in mark_all_scalars_precise()  argument
2386 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,  in __mark_chain_precision()  argument
2389 	struct bpf_verifier_state *st = env->cur_state;  in __mark_chain_precision()
2391 	int last_idx = env->insn_idx;  in __mark_chain_precision()
2400 	if (!env->bpf_capable)  in __mark_chain_precision()
2443 		if (env->log.level & BPF_LOG_LEVEL)  in __mark_chain_precision()
2444 			verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);  in __mark_chain_precision()
2450 				err = backtrack_insn(env, i, &reg_mask, &stack_mask);  in __mark_chain_precision()
2453 				mark_all_scalars_precise(env, st);  in __mark_chain_precision()
2467 			if (i >= env->prog->len) {  in __mark_chain_precision()
2474 				verbose(env, "BUG backtracking idx %d\n", i);  in __mark_chain_precision()
2513 				mark_all_scalars_precise(env, st);  in __mark_chain_precision()
2530 		if (env->log.level & BPF_LOG_LEVEL) {  in __mark_chain_precision()
2531 			print_verifier_state(env, func);  in __mark_chain_precision()
2532 			verbose(env, "parent %s regs=%x stack=%llx marks\n",  in __mark_chain_precision()
2548 static int mark_chain_precision(struct bpf_verifier_env *env, int regno)  in mark_chain_precision()  argument
2550 	return __mark_chain_precision(env, regno, -1);  in mark_chain_precision()
2553 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)  in mark_chain_precision_stack()  argument
2555 	return __mark_chain_precision(env, -1, spi);  in mark_chain_precision_stack()
2643 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,  in check_stack_write_fixed_off()  argument
2651 	u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg;  in check_stack_write_fixed_off()
2660 	if (!env->allow_ptr_leaks &&  in check_stack_write_fixed_off()
2663 		verbose(env, "attempt to corrupt spilled pointer on stack\n");  in check_stack_write_fixed_off()
2667 	cur = env->cur_state->frame[env->cur_state->curframe];  in check_stack_write_fixed_off()
2670 	if (!env->bypass_spec_v4) {  in check_stack_write_fixed_off()
2681 			env->insn_aux_data[insn_idx].sanitize_stack_spill = true;  in check_stack_write_fixed_off()
2685 	    !register_is_null(reg) && env->bpf_capable) {  in check_stack_write_fixed_off()
2693 			err = mark_chain_precision(env, value_regno);  in check_stack_write_fixed_off()
2701 			verbose_linfo(env, insn_idx, "; ");  in check_stack_write_fixed_off()
2702 			verbose(env, "invalid size of register spill\n");  in check_stack_write_fixed_off()
2706 			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");  in check_stack_write_fixed_off()
2734 			err = mark_chain_precision(env, value_regno);  in check_stack_write_fixed_off()
2767 static int check_stack_write_var_off(struct bpf_verifier_env *env,  in check_stack_write_var_off()  argument
2783 	cur = env->cur_state->frame[env->cur_state->curframe];  in check_stack_write_var_off()
2806 		if (!env->allow_ptr_leaks  in check_stack_write_var_off()
2815 			verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",  in check_stack_write_var_off()
2837 		if (*stype == STACK_INVALID && !env->allow_uninit_stack) {  in check_stack_write_var_off()
2838 			verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",  in check_stack_write_var_off()
2846 		err = mark_chain_precision(env, value_regno);  in check_stack_write_var_off()
2861 static void mark_reg_stack_read(struct bpf_verifier_env *env,  in mark_reg_stack_read()  argument
2866 	struct bpf_verifier_state *vstate = env->cur_state;  in mark_reg_stack_read()
2898 		mark_reg_unknown(env, state->regs, dst_regno);  in mark_reg_stack_read()
2912 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,  in check_stack_read_fixed_off()  argument
2917 	struct bpf_verifier_state *vstate = env->cur_state;  in check_stack_read_fixed_off()
2929 				verbose_linfo(env, env->insn_idx, "; ");  in check_stack_read_fixed_off()
2930 				verbose(env, "invalid size of register fill\n");  in check_stack_read_fixed_off()
2934 				mark_reg_unknown(env, state->regs, dst_regno);  in check_stack_read_fixed_off()
2937 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in check_stack_read_fixed_off()
2942 				verbose(env, "corrupted spill memory\n");  in check_stack_read_fixed_off()
2955 		} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {  in check_stack_read_fixed_off()
2962 			verbose(env, "leaking pointer from stack off %d\n",  in check_stack_read_fixed_off()
2966 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in check_stack_read_fixed_off()
2976 			verbose(env, "invalid read from stack off %d+%d size %d\n",  in check_stack_read_fixed_off()
2980 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in check_stack_read_fixed_off()
2982 			mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);  in check_stack_read_fixed_off()
2992 static int check_stack_range_initialized(struct bpf_verifier_env *env,
2998 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)  in reg_state()  argument
3000 	return cur_regs(env) + regno;  in reg_state()
3016 static int check_stack_read_var_off(struct bpf_verifier_env *env,  in check_stack_read_var_off()  argument
3020 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read_var_off()
3021 	struct bpf_func_state *ptr_state = func(env, reg);  in check_stack_read_var_off()
3027 	err = check_stack_range_initialized(env, ptr_regno, off, size,  in check_stack_read_var_off()
3034 	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);  in check_stack_read_var_off()
3047 static int check_stack_read(struct bpf_verifier_env *env,  in check_stack_read()  argument
3051 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read()
3052 	struct bpf_func_state *state = func(env, reg);  in check_stack_read()
3065 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=…  in check_stack_read()
3073 	if (!env->bypass_spec_v1 && var_off) {  in check_stack_read()
3077 		verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",  in check_stack_read()
3084 		err = check_stack_read_fixed_off(env, state, off, size,  in check_stack_read()
3091 		err = check_stack_read_var_off(env, ptr_regno, off, size,  in check_stack_read()
3108 static int check_stack_write(struct bpf_verifier_env *env,  in check_stack_write()  argument
3112 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_write()
3113 	struct bpf_func_state *state = func(env, reg);  in check_stack_write()
3118 		err = check_stack_write_fixed_off(env, state, off, size,  in check_stack_write()
3124 		err = check_stack_write_var_off(env, state,  in check_stack_write()
3131 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,  in check_map_access_type()  argument
3134 	struct bpf_reg_state *regs = cur_regs(env);  in check_map_access_type()
3139 		verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",  in check_map_access_type()
3145 		verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",  in check_map_access_type()
3154 static int __check_mem_access(struct bpf_verifier_env *env, int regno,  in __check_mem_access()  argument
3164 	reg = &cur_regs(env)[regno];  in __check_mem_access()
3167 		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",  in __check_mem_access()
3171 		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",  in __check_mem_access()
3177 		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",  in __check_mem_access()
3182 		verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",  in __check_mem_access()
3190 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,  in check_mem_region_access()  argument
3194 	struct bpf_verifier_state *vstate = env->cur_state;  in check_mem_region_access()
3203 	if (env->log.level & BPF_LOG_LEVEL)  in check_mem_region_access()
3204 		print_verifier_state(env, state);  in check_mem_region_access()
3216 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n…  in check_mem_region_access()
3220 	err = __check_mem_access(env, regno, reg->smin_value + off, size,  in check_mem_region_access()
3223 		verbose(env, "R%d min value is outside of the allowed memory range\n",  in check_mem_region_access()
3233 		verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",  in check_mem_region_access()
3237 	err = __check_mem_access(env, regno, reg->umax_value + off, size,  in check_mem_region_access()
3240 		verbose(env, "R%d max value is outside of the allowed memory range\n",  in check_mem_region_access()
3249 static int check_map_access(struct bpf_verifier_env *env, u32 regno,  in check_map_access()  argument
3252 	struct bpf_verifier_state *vstate = env->cur_state;  in check_map_access()
3258 	err = check_mem_region_access(env, regno, off, size, map->value_size,  in check_map_access()
3273 			verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");  in check_map_access()
3282 			verbose(env, "bpf_timer cannot be accessed directly by load/store\n");  in check_map_access()
3296 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,  in may_access_direct_pkt_data()  argument
3300 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);  in may_access_direct_pkt_data()
3324 		env->seen_direct_write = true;  in may_access_direct_pkt_data()
3329 			env->seen_direct_write = true;  in may_access_direct_pkt_data()
3338 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,  in check_packet_access()  argument
3341 	struct bpf_reg_state *regs = cur_regs(env);  in check_packet_access()
3354 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n…  in check_packet_access()
3360 	      __check_mem_access(env, regno, off, size, reg->range,  in check_packet_access()
3363 		verbose(env, "R%d offset is outside of the packet\n", regno);  in check_packet_access()
3373 	env->prog->aux->max_pkt_offset =  in check_packet_access()
3374 		max_t(u32, env->prog->aux->max_pkt_offset,  in check_packet_access()
3381 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,  in check_ctx_access()  argument
3387 		.log = &env->log,  in check_ctx_access()
3390 	if (env->ops->is_valid_access &&  in check_ctx_access()
3391 	    env->ops->is_valid_access(off, size, t, env->prog, &info)) {  in check_ctx_access()
3405 			env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;  in check_ctx_access()
3408 		if (env->prog->aux->max_ctx_offset < off + size)  in check_ctx_access()
3409 			env->prog->aux->max_ctx_offset = off + size;  in check_ctx_access()
3413 	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);  in check_ctx_access()
3417 static int check_flow_keys_access(struct bpf_verifier_env *env, int off,  in check_flow_keys_access()  argument
3422 		verbose(env, "invalid access to flow keys off=%d size=%d\n",  in check_flow_keys_access()
3429 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,  in check_sock_access()  argument
3433 	struct bpf_reg_state *regs = cur_regs(env);  in check_sock_access()
3439 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n…  in check_sock_access()
3463 		env->insn_aux_data[insn_idx].ctx_field_size =  in check_sock_access()
3468 	verbose(env, "R%d invalid %s access off=%d size=%d\n",  in check_sock_access()
3474 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)  in is_pointer_value()  argument
3476 	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));  in is_pointer_value()
3479 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)  in is_ctx_reg()  argument
3481 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_ctx_reg()
3486 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)  in is_sk_reg()  argument
3488 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_sk_reg()
3493 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)  in is_pkt_reg()  argument
3495 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_pkt_reg()
3500 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)  in is_flow_key_reg()  argument
3502 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_flow_key_reg()
3508 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,  in check_pkt_ptr_alignment()  argument
3534 		verbose(env,  in check_pkt_ptr_alignment()
3543 static int check_generic_ptr_alignment(struct bpf_verifier_env *env,  in check_generic_ptr_alignment()  argument
3559 		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",  in check_generic_ptr_alignment()
3567 static int check_ptr_alignment(struct bpf_verifier_env *env,  in check_ptr_alignment()  argument
3571 	bool strict = env->strict_alignment || strict_alignment_once;  in check_ptr_alignment()
3580 		return check_pkt_ptr_alignment(env, reg, off, size, strict);  in check_ptr_alignment()
3616 	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,  in check_ptr_alignment()
3620 static int update_stack_depth(struct bpf_verifier_env *env,  in update_stack_depth()  argument
3624 	u16 stack = env->subprog_info[func->subprogno].stack_depth;  in update_stack_depth()
3630 	env->subprog_info[func->subprogno].stack_depth = -off;  in update_stack_depth()
3640 static int check_max_stack_depth(struct bpf_verifier_env *env)  in check_max_stack_depth()  argument
3643 	struct bpf_subprog_info *subprog = env->subprog_info;  in check_max_stack_depth()
3644 	struct bpf_insn *insn = env->prog->insnsi;  in check_max_stack_depth()
3671 		verbose(env,  in check_max_stack_depth()
3681 		verbose(env, "combined stack size of %d calls is %d. Too large\n",  in check_max_stack_depth()
3698 		idx = find_subprog(env, next_insn);  in check_max_stack_depth()
3706 				verbose(env, "verifier bug. subprog has tail_call and async cb\n");  in check_max_stack_depth()
3719 			verbose(env, "the call stack of %d frames is too deep !\n",  in check_max_stack_depth()
3734 		env->prog->aux->tail_call_reachable = true;  in check_max_stack_depth()
3749 static int get_callee_stack_depth(struct bpf_verifier_env *env,  in get_callee_stack_depth()  argument
3754 	subprog = find_subprog(env, start);  in get_callee_stack_depth()
3760 	return env->subprog_info[subprog].stack_depth;  in get_callee_stack_depth()
3764 int check_ctx_reg(struct bpf_verifier_env *env,  in check_ctx_reg()  argument
3772 		verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n",  in check_ctx_reg()
3781 		verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf);  in check_ctx_reg()
3788 static int __check_buffer_access(struct bpf_verifier_env *env,  in __check_buffer_access()  argument
3794 		verbose(env,  in __check_buffer_access()
3803 		verbose(env,  in __check_buffer_access()
3812 static int check_tp_buffer_access(struct bpf_verifier_env *env,  in check_tp_buffer_access()  argument
3818 	err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);  in check_tp_buffer_access()
3822 	if (off + size > env->prog->aux->max_tp_access)  in check_tp_buffer_access()
3823 		env->prog->aux->max_tp_access = off + size;  in check_tp_buffer_access()
3828 static int check_buffer_access(struct bpf_verifier_env *env,  in check_buffer_access()  argument
3837 	err = __check_buffer_access(env, buf_info, reg, regno, off, size);  in check_buffer_access()
3920 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,  in check_ptr_to_btf_access()  argument
3933 		verbose(env,  in check_ptr_to_btf_access()
3942 		verbose(env,  in check_ptr_to_btf_access()
3948 	if (env->ops->btf_struct_access) {  in check_ptr_to_btf_access()
3949 		ret = env->ops->btf_struct_access(&env->log, reg->btf, t,  in check_ptr_to_btf_access()
3953 			verbose(env, "only read is supported\n");  in check_ptr_to_btf_access()
3957 		ret = btf_struct_access(&env->log, reg->btf, t, off, size,  in check_ptr_to_btf_access()
3965 		mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id);  in check_ptr_to_btf_access()
3970 static int check_ptr_to_map_access(struct bpf_verifier_env *env,  in check_ptr_to_map_access()  argument
3984 		verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");  in check_ptr_to_map_access()
3989 		verbose(env, "map_ptr access not supported for map type %d\n",  in check_ptr_to_map_access()
3997 	if (!env->allow_ptr_to_map_access) {  in check_ptr_to_map_access()
3998 		verbose(env,  in check_ptr_to_map_access()
4005 		verbose(env, "R%d is %s invalid negative access: off=%d\n",  in check_ptr_to_map_access()
4011 		verbose(env, "only read from %s is supported\n", tname);  in check_ptr_to_map_access()
4015 	ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id);  in check_ptr_to_map_access()
4020 		mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id);  in check_ptr_to_map_access()
4053 		struct bpf_verifier_env *env,  in check_stack_access_within_bounds()  argument
4057 	struct bpf_reg_state *regs = cur_regs(env);  in check_stack_access_within_bounds()
4059 	struct bpf_func_state *state = func(env, reg);  in check_stack_access_within_bounds()
4081 			verbose(env, "invalid unbounded variable-offset%s stack R%d\n",  in check_stack_access_within_bounds()
4098 			verbose(env, "invalid%s stack R%d off=%d size=%d\n",  in check_stack_access_within_bounds()
4104 			verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",  in check_stack_access_within_bounds()
4117 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,  in check_mem_access()  argument
4121 	struct bpf_reg_state *regs = cur_regs(env);  in check_mem_access()
4131 	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);  in check_mem_access()
4140 			verbose(env, "write to change key R%d not allowed\n", regno);  in check_mem_access()
4144 		err = check_mem_region_access(env, regno, off, size,  in check_mem_access()
4149 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4152 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
4153 			verbose(env, "R%d leaks addr into map\n", value_regno);  in check_mem_access()
4156 		err = check_map_access_type(env, regno, off, size, t);  in check_mem_access()
4159 		err = check_map_access(env, regno, off, size, false);  in check_mem_access()
4178 				mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4183 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
4184 			verbose(env, "R%d leaks addr into mem\n", value_regno);  in check_mem_access()
4187 		err = check_mem_region_access(env, regno, off, size,  in check_mem_access()
4190 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4197 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
4198 			verbose(env, "R%d leaks addr into ctx\n", value_regno);  in check_mem_access()
4202 		err = check_ctx_reg(env, reg, regno);  in check_mem_access()
4206 		err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf, &btf_id);  in check_mem_access()
4208 			verbose_linfo(env, insn_idx, "; ");  in check_mem_access()
4215 				mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4217 				mark_reg_known_zero(env, regs,  in check_mem_access()
4220 					regs[value_regno].id = ++env->id_gen;  in check_mem_access()
4238 		err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);  in check_mem_access()
4242 		state = func(env, reg);  in check_mem_access()
4243 		err = update_stack_depth(env, state, off);  in check_mem_access()
4248 			err = check_stack_read(env, regno, off, size,  in check_mem_access()
4251 			err = check_stack_write(env, regno, off, size,  in check_mem_access()
4254 		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {  in check_mem_access()
4255 			verbose(env, "cannot write into packet\n");  in check_mem_access()
4259 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
4260 			verbose(env, "R%d leaks addr into packet\n",  in check_mem_access()
4264 		err = check_packet_access(env, regno, off, size, false);  in check_mem_access()
4266 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4269 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
4270 			verbose(env, "R%d leaks addr into flow keys\n",  in check_mem_access()
4275 		err = check_flow_keys_access(env, off, size);  in check_mem_access()
4277 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4280 			verbose(env, "R%d cannot write into %s\n",  in check_mem_access()
4284 		err = check_sock_access(env, insn_idx, regno, off, size, t);  in check_mem_access()
4286 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4288 		err = check_tp_buffer_access(env, reg, regno, off, size);  in check_mem_access()
4290 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4292 		err = check_ptr_to_btf_access(env, regs, regno, off, size, t,  in check_mem_access()
4295 		err = check_ptr_to_map_access(env, regs, regno, off, size, t,  in check_mem_access()
4299 			verbose(env, "R%d cannot write into %s\n",  in check_mem_access()
4303 		err = check_buffer_access(env, reg, regno, off, size, false,  in check_mem_access()
4305 					  &env->prog->aux->max_rdonly_access);  in check_mem_access()
4307 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4309 		err = check_buffer_access(env, reg, regno, off, size, false,  in check_mem_access()
4311 					  &env->prog->aux->max_rdwr_access);  in check_mem_access()
4313 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
4315 		verbose(env, "R%d invalid mem access '%s'\n", regno,  in check_mem_access()
4328 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)  in check_atomic()  argument
4346 		verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);  in check_atomic()
4351 		verbose(env, "invalid atomic operand size\n");  in check_atomic()
4356 	err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_atomic()
4361 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_atomic()
4367 		err = check_reg_arg(env, BPF_REG_0, SRC_OP);  in check_atomic()
4372 	if (is_pointer_value(env, insn->src_reg)) {  in check_atomic()
4373 		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);  in check_atomic()
4377 	if (is_ctx_reg(env, insn->dst_reg) ||  in check_atomic()
4378 	    is_pkt_reg(env, insn->dst_reg) ||  in check_atomic()
4379 	    is_flow_key_reg(env, insn->dst_reg) ||  in check_atomic()
4380 	    is_sk_reg(env, insn->dst_reg)) {  in check_atomic()
4381 		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",  in check_atomic()
4383 			reg_type_str[reg_state(env, insn->dst_reg)->type]);  in check_atomic()
4394 		err = check_reg_arg(env, load_reg, DST_OP);  in check_atomic()
4405 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,  in check_atomic()
4411 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,  in check_atomic()
4429 		struct bpf_verifier_env *env, int regno, int off,  in check_stack_range_initialized()  argument
4433 	struct bpf_reg_state *reg = reg_state(env, regno);  in check_stack_range_initialized()
4434 	struct bpf_func_state *state = func(env, reg);  in check_stack_range_initialized()
4444 		verbose(env, "invalid zero-sized read\n");  in check_stack_range_initialized()
4458 	err = check_stack_access_within_bounds(env, regno, off, access_size,  in check_stack_range_initialized()
4472 		if (!env->bypass_spec_v1) {  in check_stack_range_initialized()
4476 			verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",  in check_stack_range_initialized()
4523 		     env->allow_ptr_leaks)) {  in check_stack_range_initialized()
4525 				__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);  in check_stack_range_initialized()
4534 			verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",  in check_stack_range_initialized()
4540 			verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",  in check_stack_range_initialized()
4548 		mark_reg_read(env, &state->stack[spi].spilled_ptr,  in check_stack_range_initialized()
4552 	return update_stack_depth(env, state, min_off);  in check_stack_range_initialized()
4555 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,  in check_helper_mem_access()  argument
4559 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_helper_mem_access()
4564 		return check_packet_access(env, regno, reg->off, access_size,  in check_helper_mem_access()
4567 		return check_mem_region_access(env, regno, reg->off, access_size,  in check_helper_mem_access()
4570 		if (check_map_access_type(env, regno, reg->off, access_size,  in check_helper_mem_access()
4574 		return check_map_access(env, regno, reg->off, access_size,  in check_helper_mem_access()
4577 		return check_mem_region_access(env, regno, reg->off,  in check_helper_mem_access()
4583 		return check_buffer_access(env, reg, regno, reg->off,  in check_helper_mem_access()
4586 					   &env->prog->aux->max_rdonly_access);  in check_helper_mem_access()
4588 		return check_buffer_access(env, reg, regno, reg->off,  in check_helper_mem_access()
4591 					   &env->prog->aux->max_rdwr_access);  in check_helper_mem_access()
4594 				env,  in check_helper_mem_access()
4603 		verbose(env, "R%d type=%s expected=%s\n", regno,  in check_helper_mem_access()
4610 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_mem_reg()  argument
4625 		rv = check_helper_mem_access(env, regno, mem_size, true, NULL);  in check_mem_reg()
4630 	return check_helper_mem_access(env, regno, mem_size, true, NULL);  in check_mem_reg()
4652 static int process_spin_lock(struct bpf_verifier_env *env, int regno,  in process_spin_lock()  argument
4655 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_spin_lock()
4656 	struct bpf_verifier_state *cur = env->cur_state;  in process_spin_lock()
4662 		verbose(env,  in process_spin_lock()
4668 		verbose(env,  in process_spin_lock()
4675 			verbose(env,  in process_spin_lock()
4679 			verbose(env,  in process_spin_lock()
4683 			verbose(env,  in process_spin_lock()
4689 		verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",  in process_spin_lock()
4695 			verbose(env,  in process_spin_lock()
4702 			verbose(env, "bpf_spin_unlock without taking a lock\n");  in process_spin_lock()
4706 			verbose(env, "bpf_spin_unlock of different lock\n");  in process_spin_lock()
4714 static int process_timer_func(struct bpf_verifier_env *env, int regno,  in process_timer_func()  argument
4717 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_timer_func()
4723 		verbose(env,  in process_timer_func()
4729 		verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n",  in process_timer_func()
4735 			verbose(env,  in process_timer_func()
4739 			verbose(env,  in process_timer_func()
4743 			verbose(env,  in process_timer_func()
4749 		verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n",  in process_timer_func()
4754 		verbose(env, "verifier bug. Two map pointers in a timer helper\n");  in process_timer_func()
4796 static int resolve_map_arg_type(struct bpf_verifier_env *env,  in resolve_map_arg_type()  argument
4802 		verbose(env, "invalid map_ptr to access map->type\n");  in resolve_map_arg_type()
4812 			verbose(env, "invalid arg_type for sockmap/sockhash\n");  in resolve_map_arg_type()
4929 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,  in check_reg_type()  argument
4933 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_reg_type()
4940 		verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);  in check_reg_type()
4953 	verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);  in check_reg_type()
4955 		verbose(env, "%s, ", reg_type_str[compatible->types[j]]);  in check_reg_type()
4956 	verbose(env, "%s\n", reg_type_str[compatible->types[j]]);  in check_reg_type()
4963 				verbose(env, "verifier internal error: missing arg compatible BTF ID\n");  in check_reg_type()
4969 		if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,  in check_reg_type()
4971 			verbose(env, "R%d is of type %s but %s is expected\n",  in check_reg_type()
4978 			verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",  in check_reg_type()
4987 static int check_func_arg(struct bpf_verifier_env *env, u32 arg,  in check_func_arg()  argument
4992 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_func_arg()
5000 	err = check_reg_arg(env, regno, SRC_OP);  in check_func_arg()
5005 		if (is_pointer_value(env, regno)) {  in check_func_arg()
5006 			verbose(env, "R%d leaks addr into helper function\n",  in check_func_arg()
5014 	    !may_access_direct_pkt_data(env, meta, BPF_READ)) {  in check_func_arg()
5015 		verbose(env, "helper access to the packet is not allowed\n");  in check_func_arg()
5022 		err = resolve_map_arg_type(env, meta, &arg_type);  in check_func_arg()
5033 	err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]);  in check_func_arg()
5038 		err = check_ctx_reg(env, reg, regno);  in check_func_arg()
5046 			verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",  in check_func_arg()
5071 				verbose(env,  in check_func_arg()
5090 			verbose(env, "invalid map_ptr to access map->key\n");  in check_func_arg()
5093 		err = check_helper_mem_access(env, regno,  in check_func_arg()
5105 			verbose(env, "invalid map_ptr to access map->value\n");  in check_func_arg()
5109 		err = check_helper_mem_access(env, regno,  in check_func_arg()
5114 			verbose(env, "Helper has invalid btf_id in R%d\n", regno);  in check_func_arg()
5121 			if (process_spin_lock(env, regno, true))  in check_func_arg()
5124 			if (process_spin_lock(env, regno, false))  in check_func_arg()
5127 			verbose(env, "verifier internal error\n");  in check_func_arg()
5131 		if (process_timer_func(env, regno, meta))  in check_func_arg()
5165 			verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",  in check_func_arg()
5171 			err = check_helper_mem_access(env, regno - 1, 0,  in check_func_arg()
5179 			verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",  in check_func_arg()
5183 		err = check_helper_mem_access(env, regno - 1,  in check_func_arg()
5187 			err = mark_chain_precision(env, regno);  in check_func_arg()
5190 			verbose(env, "R%d is not a known constant'\n",  in check_func_arg()
5198 		err = check_helper_mem_access(env, regno, size, false, meta);  in check_func_arg()
5201 		err = check_ptr_alignment(env, reg, 0, size, true);  in check_func_arg()
5209 			verbose(env, "R%d does not point to a readonly map'\n", regno);  in check_func_arg()
5214 			verbose(env, "R%d is not a constant address'\n", regno);  in check_func_arg()
5219 			verbose(env, "no direct value access support for this map type\n");  in check_func_arg()
5223 		err = check_map_access(env, regno, reg->off,  in check_func_arg()
5231 			verbose(env, "direct value access on string failed\n");  in check_func_arg()
5237 			verbose(env, "string is not zero-terminated\n");  in check_func_arg()
5245 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)  in may_update_sockmap()  argument
5247 	enum bpf_attach_type eatype = env->prog->expected_attach_type;  in may_update_sockmap()
5248 	enum bpf_prog_type type = resolve_prog_type(env->prog);  in may_update_sockmap()
5273 	verbose(env, "cannot update sockmap in this context\n");  in may_update_sockmap()
5277 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)  in allow_tail_call_in_subprogs()  argument
5279 	return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64);  in allow_tail_call_in_subprogs()
5282 static int check_map_func_compatibility(struct bpf_verifier_env *env,  in check_map_func_compatibility()  argument
5352 		    !may_update_sockmap(env, func_id))  in check_map_func_compatibility()
5362 		    !may_update_sockmap(env, func_id))  in check_map_func_compatibility()
5400 		if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {  in check_map_func_compatibility()
5401 			verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");  in check_map_func_compatibility()
5486 	verbose(env, "cannot pass map_type %d into func %s#%d\n",  in check_map_func_compatibility()
5593 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,  in __clear_all_pkt_pointers()  argument
5601 			mark_reg_unknown(env, regs, i);  in __clear_all_pkt_pointers()
5607 			__mark_reg_unknown(env, reg);  in __clear_all_pkt_pointers()
5611 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)  in clear_all_pkt_pointers()  argument
5613 	struct bpf_verifier_state *vstate = env->cur_state;  in clear_all_pkt_pointers()
5617 		__clear_all_pkt_pointers(env, vstate->frame[i]);  in clear_all_pkt_pointers()
5646 static void release_reg_references(struct bpf_verifier_env *env,  in release_reg_references()  argument
5655 			mark_reg_unknown(env, regs, i);  in release_reg_references()
5661 			__mark_reg_unknown(env, reg);  in release_reg_references()
5668 static int release_reference(struct bpf_verifier_env *env,  in release_reference()  argument
5671 	struct bpf_verifier_state *vstate = env->cur_state;  in release_reference()
5675 	err = release_reference_state(cur_func(env), ref_obj_id);  in release_reference()
5680 		release_reg_references(env, vstate->frame[i], ref_obj_id);  in release_reference()
5685 static void clear_caller_saved_regs(struct bpf_verifier_env *env,  in clear_caller_saved_regs()  argument
5692 		mark_reg_not_init(env, regs, caller_saved[i]);  in clear_caller_saved_regs()
5693 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);  in clear_caller_saved_regs()
5697 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
5702 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in __check_func_call()  argument
5706 	struct bpf_verifier_state *state = env->cur_state;  in __check_func_call()
5713 		verbose(env, "the call stack of %d frames is too deep\n",  in __check_func_call()
5720 		verbose(env, "verifier bug. Frame %d already allocated\n",  in __check_func_call()
5725 	func_info_aux = env->prog->aux->func_info_aux;  in __check_func_call()
5728 	err = btf_check_subprog_arg_match(env, subprog, caller->regs);  in __check_func_call()
5733 			verbose(env, "Caller passes invalid args into func#%d\n",  in __check_func_call()
5737 			if (env->log.level & BPF_LOG_LEVEL)  in __check_func_call()
5738 				verbose(env,  in __check_func_call()
5741 			clear_caller_saved_regs(env, caller->regs);  in __check_func_call()
5744 			mark_reg_unknown(env, caller->regs, BPF_REG_0);  in __check_func_call()
5757 		env->subprog_info[subprog].is_async_cb = true;  in __check_func_call()
5758 		async_cb = push_async_cb(env, env->subprog_info[subprog].start,  in __check_func_call()
5766 		err = set_callee_state_cb(env, caller, callee, *insn_idx);  in __check_func_call()
5770 		clear_caller_saved_regs(env, caller->regs);  in __check_func_call()
5771 		mark_reg_unknown(env, caller->regs, BPF_REG_0);  in __check_func_call()
5786 	init_func_state(env, callee,  in __check_func_call()
5797 	err = set_callee_state_cb(env, caller, callee, *insn_idx);  in __check_func_call()
5801 	clear_caller_saved_regs(env, caller->regs);  in __check_func_call()
5807 	*insn_idx = env->subprog_info[subprog].start - 1;  in __check_func_call()
5809 	if (env->log.level & BPF_LOG_LEVEL) {  in __check_func_call()
5810 		verbose(env, "caller:\n");  in __check_func_call()
5811 		print_verifier_state(env, caller);  in __check_func_call()
5812 		verbose(env, "callee:\n");  in __check_func_call()
5813 		print_verifier_state(env, callee);  in __check_func_call()
5818 int map_set_for_each_callback_args(struct bpf_verifier_env *env,  in map_set_for_each_callback_args()  argument
5841 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in map_set_for_each_callback_args()
5845 static int set_callee_state(struct bpf_verifier_env *env,  in set_callee_state()  argument
5859 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_func_call()  argument
5865 	subprog = find_subprog(env, target_insn);  in check_func_call()
5867 		verbose(env, "verifier bug. No program starts at insn %d\n",  in check_func_call()
5872 	return __check_func_call(env, insn, insn_idx, subprog, set_callee_state);  in check_func_call()
5875 static int set_map_elem_callback_state(struct bpf_verifier_env *env,  in set_map_elem_callback_state()  argument
5880 	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];  in set_map_elem_callback_state()
5885 		verbose(env, "tail_call abusing map_ptr\n");  in set_map_elem_callback_state()
5892 		verbose(env, "callback function not allowed for map\n");  in set_map_elem_callback_state()
5896 	err = map->ops->map_set_for_each_callback_args(env, caller, callee);  in set_map_elem_callback_state()
5904 static int set_timer_callback_state(struct bpf_verifier_env *env,  in set_timer_callback_state()  argument
5927 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);  in set_timer_callback_state()
5928 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in set_timer_callback_state()
5933 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)  in prepare_func_exit()  argument
5935 	struct bpf_verifier_state *state = env->cur_state;  in prepare_func_exit()
5949 		verbose(env, "cannot return stack pointer to the caller\n");  in prepare_func_exit()
5960 			verbose(env, "R0 not a scalar value\n");  in prepare_func_exit()
5964 			verbose_invalid_scalar(env, r0, &range, "callback return", "R0");  in prepare_func_exit()
5978 	if (env->log.level & BPF_LOG_LEVEL) {  in prepare_func_exit()
5979 		verbose(env, "returning from callee:\n");  in prepare_func_exit()
5980 		print_verifier_state(env, callee);  in prepare_func_exit()
5981 		verbose(env, "to caller at %d:\n", *insn_idx);  in prepare_func_exit()
5982 		print_verifier_state(env, caller);  in prepare_func_exit()
6014 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,  in record_func_map()  argument
6017 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];  in record_func_map()
6032 		verbose(env, "kernel subsystem misconfigured verifier\n");  in record_func_map()
6045 		verbose(env, "write into map forbidden\n");  in record_func_map()
6059 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,  in record_func_key()  argument
6062 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];  in record_func_key()
6063 	struct bpf_reg_state *regs = cur_regs(env), *reg;  in record_func_key()
6072 		verbose(env, "kernel subsystem misconfigured verifier\n");  in record_func_key()
6084 	err = mark_chain_precision(env, BPF_REG_3);  in record_func_key()
6097 static int check_reference_leak(struct bpf_verifier_env *env)  in check_reference_leak()  argument
6099 	struct bpf_func_state *state = cur_func(env);  in check_reference_leak()
6103 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",  in check_reference_leak()
6109 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,  in check_bpf_snprintf_call()  argument
6131 		verbose(env, "verifier bug\n");  in check_bpf_snprintf_call()
6141 		verbose(env, "Invalid format string\n");  in check_bpf_snprintf_call()
6146 static int check_get_func_ip(struct bpf_verifier_env *env)  in check_get_func_ip()  argument
6148 	enum bpf_attach_type eatype = env->prog->expected_attach_type;  in check_get_func_ip()
6149 	enum bpf_prog_type type = resolve_prog_type(env->prog);  in check_get_func_ip()
6155 			verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",  in check_get_func_ip()
6164 	verbose(env, "func %s#%d not supported for program type %d\n",  in check_get_func_ip()
6169 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_helper_call()  argument
6182 		verbose(env, "invalid func %s#%d\n", func_id_name(func_id),  in check_helper_call()
6187 	if (env->ops->get_func_proto)  in check_helper_call()
6188 		fn = env->ops->get_func_proto(func_id, env->prog);  in check_helper_call()
6190 		verbose(env, "unknown func %s#%d\n", func_id_name(func_id),  in check_helper_call()
6196 	if (!env->prog->gpl_compatible && fn->gpl_only) {  in check_helper_call()
6197 		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");  in check_helper_call()
6201 	if (fn->allowed && !fn->allowed(env->prog)) {  in check_helper_call()
6202 		verbose(env, "helper call is not allowed in probe\n");  in check_helper_call()
6209 		verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",  in check_helper_call()
6219 		verbose(env, "kernel subsystem misconfigured func %s#%d\n",  in check_helper_call()
6227 		err = check_func_arg(env, i, &meta, fn);  in check_helper_call()
6232 	err = record_func_map(env, &meta, func_id, insn_idx);  in check_helper_call()
6236 	err = record_func_key(env, &meta, func_id, insn_idx);  in check_helper_call()
6244 		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,  in check_helper_call()
6251 		err = check_reference_leak(env);  in check_helper_call()
6253 			verbose(env, "tail_call would lead to reference leak\n");  in check_helper_call()
6257 		err = release_reference(env, meta.ref_obj_id);  in check_helper_call()
6259 			verbose(env, "func %s#%d reference has not been acquired before\n",  in check_helper_call()
6265 	regs = cur_regs(env);  in check_helper_call()
6272 		verbose(env, "get_local_storage() doesn't support non-zero flags\n");  in check_helper_call()
6277 		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,  in check_helper_call()
6284 		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,  in check_helper_call()
6291 		err = check_bpf_snprintf_call(env, regs);  in check_helper_call()
6298 		mark_reg_not_init(env, regs, caller_saved[i]);  in check_helper_call()
6299 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);  in check_helper_call()
6308 		mark_reg_unknown(env, regs, BPF_REG_0);  in check_helper_call()
6314 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
6320 			verbose(env,  in check_helper_call()
6329 				regs[BPF_REG_0].id = ++env->id_gen;  in check_helper_call()
6334 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
6337 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
6340 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
6343 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
6350 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
6361 				verbose(env, "unable to resolve the size of type '%s': %ld\n",  in check_helper_call()
6380 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
6386 			verbose(env, "invalid return type %d of func %s#%d\n",  in check_helper_call()
6396 		verbose(env, "unknown return type %d of func %s#%d\n",  in check_helper_call()
6402 		regs[BPF_REG_0].id = ++env->id_gen;  in check_helper_call()
6408 		int id = acquire_reference_state(env, insn_idx);  in check_helper_call()
6420 	err = check_map_func_compatibility(env, meta.map_ptr, func_id);  in check_helper_call()
6426 	    !env->prog->has_callchain_buf) {  in check_helper_call()
6437 			verbose(env, err_str, func_id_name(func_id), func_id);  in check_helper_call()
6441 		env->prog->has_callchain_buf = true;  in check_helper_call()
6445 		env->prog->call_get_stack = true;  in check_helper_call()
6448 		if (check_get_func_ip(env))  in check_helper_call()
6450 		env->prog->call_get_func_ip = true;  in check_helper_call()
6454 		clear_all_pkt_pointers(env);  in check_helper_call()
6461 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,  in mark_btf_func_reg_size()  argument
6464 	struct bpf_reg_state *reg = &cur_regs(env)[regno];  in mark_btf_func_reg_size()
6470 			DEF_NOT_SUBREG : env->insn_idx + 1;  in mark_btf_func_reg_size()
6474 			mark_insn_zext(env, reg);  in mark_btf_func_reg_size()
6475 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in mark_btf_func_reg_size()
6477 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);  in mark_btf_func_reg_size()
6482 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_kfunc_call()  argument
6485 	struct bpf_reg_state *regs = cur_regs(env);  in check_kfunc_call()
6496 	if (!env->ops->check_kfunc_call ||  in check_kfunc_call()
6497 	    !env->ops->check_kfunc_call(func_id)) {  in check_kfunc_call()
6498 		verbose(env, "calling kernel function %s is not allowed\n",  in check_kfunc_call()
6504 	err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);  in check_kfunc_call()
6509 		mark_reg_not_init(env, regs, caller_saved[i]);  in check_kfunc_call()
6514 		mark_reg_unknown(env, regs, BPF_REG_0);  in check_kfunc_call()
6515 		mark_btf_func_reg_size(env, BPF_REG_0, t->size);  in check_kfunc_call()
6522 			verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",  in check_kfunc_call()
6527 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
6531 		mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));  in check_kfunc_call()
6541 			mark_btf_func_reg_size(env, regno, sizeof(void *));  in check_kfunc_call()
6544 			mark_btf_func_reg_size(env, regno, t->size);  in check_kfunc_call()
6590 static bool check_reg_sane_offset(struct bpf_verifier_env *env,  in check_reg_sane_offset()  argument
6599 		verbose(env, "math between %s pointer and %lld is not allowed\n",  in check_reg_sane_offset()
6605 		verbose(env, "%s pointer offset %d is not allowed\n",  in check_reg_sane_offset()
6611 		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",  in check_reg_sane_offset()
6617 		verbose(env, "value %lld makes %s pointer be out of bounds\n",  in check_reg_sane_offset()
6625 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)  in cur_aux()  argument
6627 	return &env->insn_aux_data[env->insn_idx];  in cur_aux()
6669 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,  in can_skip_alu_sanitation()  argument
6672 	return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;  in can_skip_alu_sanitation()
6692 static int sanitize_val_alu(struct bpf_verifier_env *env,  in sanitize_val_alu()  argument
6695 	struct bpf_insn_aux_data *aux = cur_aux(env);  in sanitize_val_alu()
6697 	if (can_skip_alu_sanitation(env, insn))  in sanitize_val_alu()
6714 sanitize_speculative_path(struct bpf_verifier_env *env,  in sanitize_speculative_path()  argument
6721 	branch = push_stack(env, next_idx, curr_idx, true);  in sanitize_speculative_path()
6725 			mark_reg_unknown(env, regs, insn->dst_reg);  in sanitize_speculative_path()
6727 			mark_reg_unknown(env, regs, insn->dst_reg);  in sanitize_speculative_path()
6728 			mark_reg_unknown(env, regs, insn->src_reg);  in sanitize_speculative_path()
6734 static int sanitize_ptr_alu(struct bpf_verifier_env *env,  in sanitize_ptr_alu()  argument
6742 	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;  in sanitize_ptr_alu()
6743 	struct bpf_verifier_state *vstate = env->cur_state;  in sanitize_ptr_alu()
6753 	if (can_skip_alu_sanitation(env, insn))  in sanitize_ptr_alu()
6792 			env->explore_alu_limits = true;  in sanitize_ptr_alu()
6823 	ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,  in sanitize_ptr_alu()
6824 					env->insn_idx);  in sanitize_ptr_alu()
6830 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)  in sanitize_mark_insn_seen()  argument
6832 	struct bpf_verifier_state *vstate = env->cur_state;  in sanitize_mark_insn_seen()
6840 		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;  in sanitize_mark_insn_seen()
6843 static int sanitize_err(struct bpf_verifier_env *env,  in sanitize_err()  argument
6854 		verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",  in sanitize_err()
6858 		verbose(env, "R%d has pointer with unsupported alu operation, %s\n",  in sanitize_err()
6862 		verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",  in sanitize_err()
6866 		verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",  in sanitize_err()
6870 		verbose(env, "R%d could not be pushed for speculative verification, %s\n",  in sanitize_err()
6874 		verbose(env, "verifier internal error: unknown reason (%d)\n",  in sanitize_err()
6893 				struct bpf_verifier_env *env,  in check_stack_access_for_ptr_arithmetic()  argument
6902 		verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",  in check_stack_access_for_ptr_arithmetic()
6908 		verbose(env, "R%d stack pointer arithmetic goes out of range, "  in check_stack_access_for_ptr_arithmetic()
6916 static int sanitize_check_bounds(struct bpf_verifier_env *env,  in sanitize_check_bounds()  argument
6925 	if (env->bypass_spec_v1)  in sanitize_check_bounds()
6930 		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,  in sanitize_check_bounds()
6935 		if (check_map_access(env, dst, dst_reg->off, 1, false)) {  in sanitize_check_bounds()
6936 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "  in sanitize_check_bounds()
6953 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,  in adjust_ptr_min_max_vals()  argument
6958 	struct bpf_verifier_state *vstate = env->cur_state;  in adjust_ptr_min_max_vals()
6978 		__mark_reg_unknown(env, dst_reg);  in adjust_ptr_min_max_vals()
6984 		if (opcode == BPF_SUB && env->allow_ptr_leaks) {  in adjust_ptr_min_max_vals()
6985 			__mark_reg_unknown(env, dst_reg);  in adjust_ptr_min_max_vals()
6989 		verbose(env,  in adjust_ptr_min_max_vals()
6997 		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",  in adjust_ptr_min_max_vals()
7013 		verbose(env, "R%d pointer arithmetic on %s prohibited\n",  in adjust_ptr_min_max_vals()
7026 	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||  in adjust_ptr_min_max_vals()
7027 	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))  in adjust_ptr_min_max_vals()
7034 		ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,  in adjust_ptr_min_max_vals()
7037 			return sanitize_err(env, insn, ret, off_reg, dst_reg);  in adjust_ptr_min_max_vals()
7086 			dst_reg->id = ++env->id_gen;  in adjust_ptr_min_max_vals()
7094 			verbose(env, "R%d tried to subtract pointer from scalar\n",  in adjust_ptr_min_max_vals()
7103 			verbose(env, "R%d subtraction from stack pointer prohibited\n",  in adjust_ptr_min_max_vals()
7145 			dst_reg->id = ++env->id_gen;  in adjust_ptr_min_max_vals()
7155 		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",  in adjust_ptr_min_max_vals()
7160 		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",  in adjust_ptr_min_max_vals()
7165 	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))  in adjust_ptr_min_max_vals()
7172 	if (sanitize_check_bounds(env, insn, dst_reg) < 0)  in adjust_ptr_min_max_vals()
7175 		ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,  in adjust_ptr_min_max_vals()
7178 			return sanitize_err(env, insn, ret, off_reg, dst_reg);  in adjust_ptr_min_max_vals()
7755 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,  in adjust_scalar_min_max_vals()  argument
7760 	struct bpf_reg_state *regs = cur_regs(env);  in adjust_scalar_min_max_vals()
7789 			__mark_reg_unknown(env, dst_reg);  in adjust_scalar_min_max_vals()
7800 			__mark_reg_unknown(env, dst_reg);  in adjust_scalar_min_max_vals()
7807 		__mark_reg_unknown(env, dst_reg);  in adjust_scalar_min_max_vals()
7812 		ret = sanitize_val_alu(env, insn);  in adjust_scalar_min_max_vals()
7814 			return sanitize_err(env, insn, ret, NULL, NULL);  in adjust_scalar_min_max_vals()
7867 			mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
7880 			mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
7893 			mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
7902 		mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
7919 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,  in adjust_reg_min_max_vals()  argument
7922 	struct bpf_verifier_state *vstate = env->cur_state;  in adjust_reg_min_max_vals()
7946 				if (opcode == BPF_SUB && env->allow_ptr_leaks) {  in adjust_reg_min_max_vals()
7947 					mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_reg_min_max_vals()
7950 				verbose(env, "R%d pointer %s pointer prohibited\n",  in adjust_reg_min_max_vals()
7959 				err = mark_chain_precision(env, insn->dst_reg);  in adjust_reg_min_max_vals()
7962 				return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
7967 			err = mark_chain_precision(env, insn->src_reg);  in adjust_reg_min_max_vals()
7970 			return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
7981 			return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
7987 		print_verifier_state(env, state);  in adjust_reg_min_max_vals()
7988 		verbose(env, "verifier internal error: unexpected ptr_reg\n");  in adjust_reg_min_max_vals()
7992 		print_verifier_state(env, state);  in adjust_reg_min_max_vals()
7993 		verbose(env, "verifier internal error: no src_reg\n");  in adjust_reg_min_max_vals()
7996 	return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);  in adjust_reg_min_max_vals()
8000 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_alu_op()  argument
8002 	struct bpf_reg_state *regs = cur_regs(env);  in check_alu_op()
8011 				verbose(env, "BPF_NEG uses reserved fields\n");  in check_alu_op()
8018 				verbose(env, "BPF_END uses reserved fields\n");  in check_alu_op()
8024 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_alu_op()
8028 		if (is_pointer_value(env, insn->dst_reg)) {  in check_alu_op()
8029 			verbose(env, "R%d pointer arithmetic prohibited\n",  in check_alu_op()
8035 		err = check_reg_arg(env, insn->dst_reg, DST_OP);  in check_alu_op()
8043 				verbose(env, "BPF_MOV uses reserved fields\n");  in check_alu_op()
8048 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_alu_op()
8053 				verbose(env, "BPF_MOV uses reserved fields\n");  in check_alu_op()
8059 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in check_alu_op()
8076 					src_reg->id = ++env->id_gen;  in check_alu_op()
8082 				if (is_pointer_value(env, insn->src_reg)) {  in check_alu_op()
8083 					verbose(env,  in check_alu_op()
8095 					dst_reg->subreg_def = env->insn_idx + 1;  in check_alu_op()
8097 					mark_reg_unknown(env, regs,  in check_alu_op()
8107 			mark_reg_unknown(env, regs, insn->dst_reg);  in check_alu_op()
8119 		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);  in check_alu_op()
8126 				verbose(env, "BPF_ALU uses reserved fields\n");  in check_alu_op()
8130 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_alu_op()
8135 				verbose(env, "BPF_ALU uses reserved fields\n");  in check_alu_op()
8141 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_alu_op()
8147 			verbose(env, "div by zero\n");  in check_alu_op()
8156 				verbose(env, "invalid shift %d\n", insn->imm);  in check_alu_op()
8162 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in check_alu_op()
8166 		return adjust_reg_min_max_vals(env, insn);  in check_alu_op()
8939 static int check_cond_jmp_op(struct bpf_verifier_env *env,  in check_cond_jmp_op()  argument
8942 	struct bpf_verifier_state *this_branch = env->cur_state;  in check_cond_jmp_op()
8953 		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);  in check_cond_jmp_op()
8959 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");  in check_cond_jmp_op()
8964 		err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_cond_jmp_op()
8968 		if (is_pointer_value(env, insn->src_reg)) {  in check_cond_jmp_op()
8969 			verbose(env, "R%d pointer comparison prohibited\n",  in check_cond_jmp_op()
8976 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");  in check_cond_jmp_op()
8982 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_cond_jmp_op()
9014 			err = mark_chain_precision(env, insn->dst_reg);  in check_cond_jmp_op()
9017 			err = mark_chain_precision(env, insn->src_reg);  in check_cond_jmp_op()
9027 		if (!env->bypass_spec_v1 &&  in check_cond_jmp_op()
9028 		    !sanitize_speculative_path(env, insn, *insn_idx + 1,  in check_cond_jmp_op()
9038 		if (!env->bypass_spec_v1 &&  in check_cond_jmp_op()
9039 		    !sanitize_speculative_path(env, insn,  in check_cond_jmp_op()
9046 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,  in check_cond_jmp_op()
9121 		   is_pointer_value(env, insn->dst_reg)) {  in check_cond_jmp_op()
9122 		verbose(env, "R%d pointer comparison prohibited\n",  in check_cond_jmp_op()
9126 	if (env->log.level & BPF_LOG_LEVEL)  in check_cond_jmp_op()
9127 		print_verifier_state(env, this_branch->frame[this_branch->curframe]);  in check_cond_jmp_op()
9132 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_imm()  argument
9134 	struct bpf_insn_aux_data *aux = cur_aux(env);  in check_ld_imm()
9135 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_imm()
9141 		verbose(env, "invalid BPF_LD_IMM insn\n");  in check_ld_imm()
9145 		verbose(env, "BPF_LD_IMM64 uses reserved fields\n");  in check_ld_imm()
9149 	err = check_reg_arg(env, insn->dst_reg, DST_OP);  in check_ld_imm()
9163 		mark_reg_known_zero(env, regs, insn->dst_reg);  in check_ld_imm()
9176 			verbose(env, "bpf verifier is misconfigured\n");  in check_ld_imm()
9183 		struct bpf_prog_aux *aux = env->prog->aux;  in check_ld_imm()
9187 			verbose(env, "missing btf func_info\n");  in check_ld_imm()
9191 			verbose(env, "callback function not static\n");  in check_ld_imm()
9200 	map = env->used_maps[aux->map_index];  in check_ld_imm()
9201 	mark_reg_known_zero(env, regs, insn->dst_reg);  in check_ld_imm()
9209 			dst_reg->id = ++env->id_gen;  in check_ld_imm()
9214 		verbose(env, "bpf verifier is misconfigured\n");  in check_ld_imm()
9248 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_abs()  argument
9250 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_abs()
9255 	if (!may_access_skb(resolve_prog_type(env->prog))) {  in check_ld_abs()
9256 		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");  in check_ld_abs()
9260 	if (!env->ops->gen_ld_abs) {  in check_ld_abs()
9261 		verbose(env, "bpf verifier is misconfigured\n");  in check_ld_abs()
9268 		verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");  in check_ld_abs()
9273 	err = check_reg_arg(env, ctx_reg, SRC_OP);  in check_ld_abs()
9281 	err = check_reference_leak(env);  in check_ld_abs()
9283 		verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");  in check_ld_abs()
9287 	if (env->cur_state->active_spin_lock) {  in check_ld_abs()
9288 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");  in check_ld_abs()
9293 		verbose(env,  in check_ld_abs()
9300 		err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_ld_abs()
9305 	err = check_ctx_reg(env, &regs[ctx_reg], ctx_reg);  in check_ld_abs()
9311 		mark_reg_not_init(env, regs, caller_saved[i]);  in check_ld_abs()
9312 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);  in check_ld_abs()
9319 	mark_reg_unknown(env, regs, BPF_REG_0);  in check_ld_abs()
9321 	regs[BPF_REG_0].subreg_def = env->insn_idx + 1;  in check_ld_abs()
9325 static int check_return_code(struct bpf_verifier_env *env)  in check_return_code()  argument
9328 	const struct bpf_prog *prog = env->prog;  in check_return_code()
9331 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);  in check_return_code()
9333 	struct bpf_func_state *frame = env->cur_state->frame[0];  in check_return_code()
9349 	err = check_reg_arg(env, BPF_REG_0, SRC_OP);  in check_return_code()
9353 	if (is_pointer_value(env, BPF_REG_0)) {  in check_return_code()
9354 		verbose(env, "R0 leaks addr as return value\n");  in check_return_code()
9358 	reg = cur_regs(env) + BPF_REG_0;  in check_return_code()
9363 			verbose(env, "In async callback the register R0 is not a known value (%s)\n",  in check_return_code()
9369 			verbose_invalid_scalar(env, reg, &range, "async callback", "R0");  in check_return_code()
9377 			verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",  in check_return_code()
9386 		if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||  in check_return_code()
9387 		    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||  in check_return_code()
9388 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||  in check_return_code()
9389 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||  in check_return_code()
9390 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||  in check_return_code()
9391 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)  in check_return_code()
9393 		if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||  in check_return_code()
9394 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)  in check_return_code()
9398 		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {  in check_return_code()
9410 		if (!env->prog->aux->attach_btf_id)  in check_return_code()
9415 		switch (env->prog->expected_attach_type) {  in check_return_code()
9441 		verbose(env, "At program exit the register R0 is not a known value (%s)\n",  in check_return_code()
9447 		verbose_invalid_scalar(env, reg, &range, "program exit", "R0");  in check_return_code()
9453 		env->prog->enforce_expected_attach_type = 1;  in check_return_code()
9497 static u32 state_htab_size(struct bpf_verifier_env *env)  in state_htab_size()  argument
9499 	return env->prog->len;  in state_htab_size()
9503 					struct bpf_verifier_env *env,  in explored_state()  argument
9506 	struct bpf_verifier_state *cur = env->cur_state;  in explored_state()
9509 	return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];  in explored_state()
9512 static void init_explored_state(struct bpf_verifier_env *env, int idx)  in init_explored_state()  argument
9514 	env->insn_aux_data[idx].prune_point = true;  in init_explored_state()
9527 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env,  in push_insn()  argument
9530 	int *insn_stack = env->cfg.insn_stack;  in push_insn()
9531 	int *insn_state = env->cfg.insn_state;  in push_insn()
9539 	if (w < 0 || w >= env->prog->len) {  in push_insn()
9540 		verbose_linfo(env, t, "%d: ", t);  in push_insn()
9541 		verbose(env, "jump out of range from insn %d to %d\n", t, w);  in push_insn()
9547 		init_explored_state(env, w);  in push_insn()
9553 		if (env->cfg.cur_stack >= env->prog->len)  in push_insn()
9555 		insn_stack[env->cfg.cur_stack++] = w;  in push_insn()
9558 		if (loop_ok && env->bpf_capable)  in push_insn()
9560 		verbose_linfo(env, t, "%d: ", t);  in push_insn()
9561 		verbose_linfo(env, w, "%d: ", w);  in push_insn()
9562 		verbose(env, "back-edge from insn %d to %d\n", t, w);  in push_insn()
9568 		verbose(env, "insn state internal bug\n");  in push_insn()
9576 				struct bpf_verifier_env *env,  in visit_func_call_insn()  argument
9581 	ret = push_insn(t, t + 1, FALLTHROUGH, env, false);  in visit_func_call_insn()
9586 		init_explored_state(env, t + 1);  in visit_func_call_insn()
9588 		init_explored_state(env, t);  in visit_func_call_insn()
9589 		ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env,  in visit_func_call_insn()
9604 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env)  in visit_insn()  argument
9606 	struct bpf_insn *insns = env->prog->insnsi;  in visit_insn()
9610 		return visit_func_call_insn(t, insn_cnt, insns, env, true);  in visit_insn()
9615 		return push_insn(t, t + 1, FALLTHROUGH, env, false);  in visit_insn()
9628 			init_explored_state(env, t);  in visit_insn()
9629 		return visit_func_call_insn(t, insn_cnt, insns, env,  in visit_insn()
9637 		ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env,  in visit_insn()
9646 		init_explored_state(env, t + insns[t].off + 1);  in visit_insn()
9651 			init_explored_state(env, t + 1);  in visit_insn()
9657 		init_explored_state(env, t);  in visit_insn()
9658 		ret = push_insn(t, t + 1, FALLTHROUGH, env, true);  in visit_insn()
9662 		return push_insn(t, t + insns[t].off + 1, BRANCH, env, true);  in visit_insn()
9669 static int check_cfg(struct bpf_verifier_env *env)  in check_cfg()  argument
9671 	int insn_cnt = env->prog->len;  in check_cfg()
9676 	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);  in check_cfg()
9680 	insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);  in check_cfg()
9688 	env->cfg.cur_stack = 1;  in check_cfg()
9690 	while (env->cfg.cur_stack > 0) {  in check_cfg()
9691 		int t = insn_stack[env->cfg.cur_stack - 1];  in check_cfg()
9693 		ret = visit_insn(t, insn_cnt, env);  in check_cfg()
9697 			env->cfg.cur_stack--;  in check_cfg()
9703 				verbose(env, "visit_insn internal bug\n");  in check_cfg()
9710 	if (env->cfg.cur_stack < 0) {  in check_cfg()
9711 		verbose(env, "pop stack internal bug\n");  in check_cfg()
9718 			verbose(env, "unreachable insn %d\n", i);  in check_cfg()
9728 	env->cfg.insn_state = env->cfg.insn_stack = NULL;  in check_cfg()
9732 static int check_abnormal_return(struct bpf_verifier_env *env)  in check_abnormal_return()  argument
9736 	for (i = 1; i < env->subprog_cnt; i++) {  in check_abnormal_return()
9737 		if (env->subprog_info[i].has_ld_abs) {  in check_abnormal_return()
9738 			verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");  in check_abnormal_return()
9741 		if (env->subprog_info[i].has_tail_call) {  in check_abnormal_return()
9742 			verbose(env, "tail_call is not allowed in subprogs without BTF\n");  in check_abnormal_return()
9753 static int check_btf_func(struct bpf_verifier_env *env,  in check_btf_func()  argument
9771 		if (check_abnormal_return(env))  in check_btf_func()
9776 	if (nfuncs != env->subprog_cnt) {  in check_btf_func()
9777 		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");  in check_btf_func()
9785 		verbose(env, "invalid func info rec size %u\n", urec_size);  in check_btf_func()
9789 	prog = env->prog;  in check_btf_func()
9806 				verbose(env, "nonzero tailing record in func info");  in check_btf_func()
9827 				verbose(env,  in check_btf_func()
9833 			verbose(env,  in check_btf_func()
9839 		if (env->subprog_info[i].start != krecord[i].insn_off) {  in check_btf_func()
9840 			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");  in check_btf_func()
9847 			verbose(env, "invalid type id %d in func info",  in check_btf_func()
9860 		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {  in check_btf_func()
9861 			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");  in check_btf_func()
9864 		if (i && !scalar_return && env->subprog_info[i].has_tail_call) {  in check_btf_func()
9865 			verbose(env, "tail_call is only allowed in functions that return 'int'.\n");  in check_btf_func()
9884 static void adjust_btf_func(struct bpf_verifier_env *env)  in adjust_btf_func()  argument
9886 	struct bpf_prog_aux *aux = env->prog->aux;  in adjust_btf_func()
9892 	for (i = 0; i < env->subprog_cnt; i++)  in adjust_btf_func()
9893 		aux->func_info[i].insn_off = env->subprog_info[i].start;  in adjust_btf_func()
9900 static int check_btf_line(struct bpf_verifier_env *env,  in check_btf_line()  argument
9932 	prog = env->prog;  in check_btf_line()
9936 	sub = env->subprog_info;  in check_btf_line()
9944 				verbose(env, "nonzero tailing record in line_info");  in check_btf_line()
9971 			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",  in check_btf_line()
9979 			verbose(env,  in check_btf_line()
9988 			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);  in check_btf_line()
9993 		if (s != env->subprog_cnt) {  in check_btf_line()
9998 				verbose(env, "missing bpf_line_info for func#%u\n", s);  in check_btf_line()
10008 	if (s != env->subprog_cnt) {  in check_btf_line()
10009 		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",  in check_btf_line()
10010 			env->subprog_cnt - s, s);  in check_btf_line()
10025 static int check_btf_info(struct bpf_verifier_env *env,  in check_btf_info()  argument
10033 		if (check_abnormal_return(env))  in check_btf_info()
10045 	env->prog->aux->btf = btf;  in check_btf_info()
10047 	err = check_btf_func(env, attr, uattr);  in check_btf_info()
10051 	err = check_btf_line(env, attr, uattr);  in check_btf_info()
10101 static void clean_func_state(struct bpf_verifier_env *env,  in clean_func_state()  argument
10115 			__mark_reg_not_init(env, &st->regs[i]);  in clean_func_state()
10123 			__mark_reg_not_init(env, &st->stack[i].spilled_ptr);  in clean_func_state()
10130 static void clean_verifier_state(struct bpf_verifier_env *env,  in clean_verifier_state()  argument
10140 		clean_func_state(env, st->frame[i]);  in clean_verifier_state()
10175 static void clean_live_states(struct bpf_verifier_env *env, int insn,  in clean_live_states()  argument
10181 	sl = *explored_state(env, insn);  in clean_live_states()
10191 		clean_verifier_state(env, &sl->state);  in clean_live_states()
10198 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,  in regsafe()  argument
10225 		if (env->explore_alu_limits)  in regsafe()
10317 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,  in stacksafe()  argument
10363 		if (!regsafe(env, &old->stack[spi].spilled_ptr,  in stacksafe()
10414 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,  in func_states_equal()  argument
10419 	memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));  in func_states_equal()
10421 		if (!regsafe(env, &old->regs[i], &cur->regs[i],  in func_states_equal()
10422 			     env->idmap_scratch))  in func_states_equal()
10425 	if (!stacksafe(env, old, cur, env->idmap_scratch))  in func_states_equal()
10434 static bool states_equal(struct bpf_verifier_env *env,  in states_equal()  argument
10458 		if (!func_states_equal(env, old->frame[i], cur->frame[i]))  in states_equal()
10467 static int propagate_liveness_reg(struct bpf_verifier_env *env,  in propagate_liveness_reg()  argument
10486 	err = mark_reg_read(env, reg, parent_reg, flag);  in propagate_liveness_reg()
10500 static int propagate_liveness(struct bpf_verifier_env *env,  in propagate_liveness()  argument
10522 			err = propagate_liveness_reg(env, &state_reg[i],  in propagate_liveness()
10527 				mark_insn_zext(env, &parent_reg[i]);  in propagate_liveness()
10535 			err = propagate_liveness_reg(env, state_reg,  in propagate_liveness()
10547 static int propagate_precision(struct bpf_verifier_env *env,  in propagate_precision()  argument
10560 		if (env->log.level & BPF_LOG_LEVEL2)  in propagate_precision()
10561 			verbose(env, "propagating r%d\n", i);  in propagate_precision()
10562 		err = mark_chain_precision(env, i);  in propagate_precision()
10574 		if (env->log.level & BPF_LOG_LEVEL2)  in propagate_precision()
10575 			verbose(env, "propagating fp%d\n",  in propagate_precision()
10577 		err = mark_chain_precision_stack(env, i);  in propagate_precision()
10603 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)  in is_state_visited()  argument
10607 	struct bpf_verifier_state *cur = env->cur_state, *new;  in is_state_visited()
10609 	bool add_new_state = env->test_state_freq ? true : false;  in is_state_visited()
10611 	cur->last_insn_idx = env->prev_insn_idx;  in is_state_visited()
10612 	if (!env->insn_aux_data[insn_idx].prune_point)  in is_state_visited()
10626 	if (env->jmps_processed - env->prev_jmps_processed >= 2 &&  in is_state_visited()
10627 	    env->insn_processed - env->prev_insn_processed >= 8)  in is_state_visited()
10630 	pprev = explored_state(env, insn_idx);  in is_state_visited()
10633 	clean_live_states(env, insn_idx, cur);  in is_state_visited()
10657 				   states_equal(env, &sl->state, cur)) {  in is_state_visited()
10658 				verbose_linfo(env, insn_idx, "; ");  in is_state_visited()
10659 				verbose(env, "infinite loop detected at insn %d\n", insn_idx);  in is_state_visited()
10674 			if (env->jmps_processed - env->prev_jmps_processed < 20 &&  in is_state_visited()
10675 			    env->insn_processed - env->prev_insn_processed < 100)  in is_state_visited()
10679 		if (states_equal(env, &sl->state, cur)) {  in is_state_visited()
10691 			err = propagate_liveness(env, &sl->state, cur);  in is_state_visited()
10698 			err = err ? : push_jmp_history(env, cur);  in is_state_visited()
10699 			err = err ? : propagate_precision(env, &sl->state);  in is_state_visited()
10731 				env->peak_states--;  in is_state_visited()
10737 				sl->next = env->free_list;  in is_state_visited()
10738 				env->free_list = sl;  in is_state_visited()
10748 	if (env->max_states_per_insn < states_cnt)  in is_state_visited()
10749 		env->max_states_per_insn = states_cnt;  in is_state_visited()
10751 	if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)  in is_state_visited()
10752 		return push_jmp_history(env, cur);  in is_state_visited()
10755 		return push_jmp_history(env, cur);  in is_state_visited()
10769 	env->total_states++;  in is_state_visited()
10770 	env->peak_states++;  in is_state_visited()
10771 	env->prev_jmps_processed = env->jmps_processed;  in is_state_visited()
10772 	env->prev_insn_processed = env->insn_processed;  in is_state_visited()
10789 	new_sl->next = *explored_state(env, insn_idx);  in is_state_visited()
10790 	*explored_state(env, insn_idx) = new_sl;  in is_state_visited()
10863 static int do_check(struct bpf_verifier_env *env)  in do_check()  argument
10865 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);  in do_check()
10866 	struct bpf_verifier_state *state = env->cur_state;  in do_check()
10867 	struct bpf_insn *insns = env->prog->insnsi;  in do_check()
10869 	int insn_cnt = env->prog->len;  in do_check()
10878 		env->prev_insn_idx = prev_insn_idx;  in do_check()
10879 		if (env->insn_idx >= insn_cnt) {  in do_check()
10880 			verbose(env, "invalid insn idx %d insn_cnt %d\n",  in do_check()
10881 				env->insn_idx, insn_cnt);  in do_check()
10885 		insn = &insns[env->insn_idx];  in do_check()
10888 		if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {  in do_check()
10889 			verbose(env,  in do_check()
10891 				env->insn_processed);  in do_check()
10895 		err = is_state_visited(env, env->insn_idx);  in do_check()
10900 			if (env->log.level & BPF_LOG_LEVEL) {  in do_check()
10902 					verbose(env, "\nfrom %d to %d%s: safe\n",  in do_check()
10903 						env->prev_insn_idx, env->insn_idx,  in do_check()
10904 						env->cur_state->speculative ?  in do_check()
10907 					verbose(env, "%d: safe\n", env->insn_idx);  in do_check()
10918 		if (env->log.level & BPF_LOG_LEVEL2 ||  in do_check()
10919 		    (env->log.level & BPF_LOG_LEVEL && do_print_state)) {  in do_check()
10920 			if (env->log.level & BPF_LOG_LEVEL2)  in do_check()
10921 				verbose(env, "%d:", env->insn_idx);  in do_check()
10923 				verbose(env, "\nfrom %d to %d%s:",  in do_check()
10924 					env->prev_insn_idx, env->insn_idx,  in do_check()
10925 					env->cur_state->speculative ?  in do_check()
10927 			print_verifier_state(env, state->frame[state->curframe]);  in do_check()
10931 		if (env->log.level & BPF_LOG_LEVEL) {  in do_check()
10935 				.private_data	= env,  in do_check()
10938 			verbose_linfo(env, env->insn_idx, "; ");  in do_check()
10939 			verbose(env, "%d: ", env->insn_idx);  in do_check()
10940 			print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);  in do_check()
10943 		if (bpf_prog_is_dev_bound(env->prog->aux)) {  in do_check()
10944 			err = bpf_prog_offload_verify_insn(env, env->insn_idx,  in do_check()
10945 							   env->prev_insn_idx);  in do_check()
10950 		regs = cur_regs(env);  in do_check()
10951 		sanitize_mark_insn_seen(env);  in do_check()
10952 		prev_insn_idx = env->insn_idx;  in do_check()
10955 			err = check_alu_op(env, insn);  in do_check()
10965 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in do_check()
10969 			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in do_check()
10978 			err = check_mem_access(env, env->insn_idx, insn->src_reg,  in do_check()
10984 			prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;  in do_check()
11001 				verbose(env, "same insn cannot be used with different pointers\n");  in do_check()
11009 				err = check_atomic(env, env->insn_idx, insn);  in do_check()
11012 				env->insn_idx++;  in do_check()
11017 				verbose(env, "BPF_STX uses reserved fields\n");  in do_check()
11022 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in do_check()
11026 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in do_check()
11033 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in do_check()
11039 			prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;  in do_check()
11044 				verbose(env, "same insn cannot be used with different pointers\n");  in do_check()
11051 				verbose(env, "BPF_ST uses reserved fields\n");  in do_check()
11055 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in do_check()
11059 			if (is_ctx_reg(env, insn->dst_reg)) {  in do_check()
11060 				verbose(env, "BPF_ST stores into R%d %s is not allowed\n",  in do_check()
11062 					reg_type_str[reg_state(env, insn->dst_reg)->type]);  in do_check()
11067 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in do_check()
11076 			env->jmps_processed++;  in do_check()
11085 					verbose(env, "BPF_CALL uses reserved fields\n");  in do_check()
11089 				if (env->cur_state->active_spin_lock &&  in do_check()
11092 					verbose(env, "function calls are not allowed while holding a lock\n");  in do_check()
11096 					err = check_func_call(env, insn, &env->insn_idx);  in do_check()
11098 					err = check_kfunc_call(env, insn);  in do_check()
11100 					err = check_helper_call(env, insn, &env->insn_idx);  in do_check()
11109 					verbose(env, "BPF_JA uses reserved fields\n");  in do_check()
11113 				env->insn_idx += insn->off + 1;  in do_check()
11122 					verbose(env, "BPF_EXIT uses reserved fields\n");  in do_check()
11126 				if (env->cur_state->active_spin_lock) {  in do_check()
11127 					verbose(env, "bpf_spin_unlock is missing\n");  in do_check()
11133 					err = prepare_func_exit(env, &env->insn_idx);  in do_check()
11140 				err = check_reference_leak(env);  in do_check()
11144 				err = check_return_code(env);  in do_check()
11148 				update_branch_counts(env, env->cur_state);  in do_check()
11149 				err = pop_stack(env, &prev_insn_idx,  in do_check()
11150 						&env->insn_idx, pop_log);  in do_check()
11160 				err = check_cond_jmp_op(env, insn, &env->insn_idx);  in do_check()
11168 				err = check_ld_abs(env, insn);  in do_check()
11173 				err = check_ld_imm(env, insn);  in do_check()
11177 				env->insn_idx++;  in do_check()
11178 				sanitize_mark_insn_seen(env);  in do_check()
11180 				verbose(env, "invalid BPF_LD mode\n");  in do_check()
11184 			verbose(env, "unknown insn class %d\n", class);  in do_check()
11188 		env->insn_idx++;  in do_check()
11225 static int check_pseudo_btf_id(struct bpf_verifier_env *env,  in check_pseudo_btf_id()  argument
11245 			verbose(env, "invalid module BTF object FD specified.\n");  in check_pseudo_btf_id()
11250 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"…  in check_pseudo_btf_id()
11259 		verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);  in check_pseudo_btf_id()
11265 		verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id);  in check_pseudo_btf_id()
11273 		verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",  in check_pseudo_btf_id()
11308 			verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",  in check_pseudo_btf_id()
11322 	for (i = 0; i < env->used_btf_cnt; i++) {  in check_pseudo_btf_id()
11323 		if (env->used_btfs[i].btf == btf) {  in check_pseudo_btf_id()
11329 	if (env->used_btf_cnt >= MAX_USED_BTFS) {  in check_pseudo_btf_id()
11334 	btf_mod = &env->used_btfs[env->used_btf_cnt];  in check_pseudo_btf_id()
11347 	env->used_btf_cnt++;  in check_pseudo_btf_id()
11385 static int check_map_prog_compatibility(struct bpf_verifier_env *env,  in check_map_prog_compatibility()  argument
11410 			verbose(env, "perf_event programs can only use preallocated hash map\n");  in check_map_prog_compatibility()
11414 			verbose(env, "trace type programs can only use preallocated hash map\n");  in check_map_prog_compatibility()
11418 …verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to prealloc…  in check_map_prog_compatibility()
11423 			verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");  in check_map_prog_compatibility()
11428 			verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");  in check_map_prog_compatibility()
11433 			verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n");  in check_map_prog_compatibility()
11440 		verbose(env, "offload device mismatch between prog and map\n");  in check_map_prog_compatibility()
11445 		verbose(env, "bpf_struct_ops map cannot be used in prog\n");  in check_map_prog_compatibility()
11460 				verbose(env,  in check_map_prog_compatibility()
11468 			verbose(env,  in check_map_prog_compatibility()
11489 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)  in resolve_pseudo_ldimm64()  argument
11491 	struct bpf_insn *insn = env->prog->insnsi;  in resolve_pseudo_ldimm64()
11492 	int insn_cnt = env->prog->len;  in resolve_pseudo_ldimm64()
11495 	err = bpf_prog_calc_tag(env->prog);  in resolve_pseudo_ldimm64()
11502 			verbose(env, "BPF_LDX uses reserved fields\n");  in resolve_pseudo_ldimm64()
11516 				verbose(env, "invalid bpf_ld_imm64 insn\n");  in resolve_pseudo_ldimm64()
11525 				aux = &env->insn_aux_data[i];  in resolve_pseudo_ldimm64()
11526 				err = check_pseudo_btf_id(env, insn, aux);  in resolve_pseudo_ldimm64()
11533 				aux = &env->insn_aux_data[i];  in resolve_pseudo_ldimm64()
11551 				verbose(env, "unrecognized bpf_ld_imm64 insn\n");  in resolve_pseudo_ldimm64()
11558 				if (bpfptr_is_null(env->fd_array)) {  in resolve_pseudo_ldimm64()
11559 					verbose(env, "fd_idx without fd_array is invalid\n");  in resolve_pseudo_ldimm64()
11562 				if (copy_from_bpfptr_offset(&fd, env->fd_array,  in resolve_pseudo_ldimm64()
11575 				verbose(env, "fd %d is not pointing to valid bpf_map\n",  in resolve_pseudo_ldimm64()
11580 			err = check_map_prog_compatibility(env, map, env->prog);  in resolve_pseudo_ldimm64()
11586 			aux = &env->insn_aux_data[i];  in resolve_pseudo_ldimm64()
11594 					verbose(env, "direct value offset of %u is not allowed\n", off);  in resolve_pseudo_ldimm64()
11600 					verbose(env, "no direct value access support for this map type\n");  in resolve_pseudo_ldimm64()
11607 					verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",  in resolve_pseudo_ldimm64()
11621 			for (j = 0; j < env->used_map_cnt; j++) {  in resolve_pseudo_ldimm64()
11622 				if (env->used_maps[j] == map) {  in resolve_pseudo_ldimm64()
11629 			if (env->used_map_cnt >= MAX_USED_MAPS) {  in resolve_pseudo_ldimm64()
11641 			aux->map_index = env->used_map_cnt;  in resolve_pseudo_ldimm64()
11642 			env->used_maps[env->used_map_cnt++] = map;  in resolve_pseudo_ldimm64()
11645 			    bpf_cgroup_storage_assign(env->prog->aux, map)) {  in resolve_pseudo_ldimm64()
11646 				verbose(env, "only one cgroup storage of each type is allowed\n");  in resolve_pseudo_ldimm64()
11660 			verbose(env, "unknown opcode %02x\n", insn->code);  in resolve_pseudo_ldimm64()
11673 static void release_maps(struct bpf_verifier_env *env)  in release_maps()  argument
11675 	__bpf_free_used_maps(env->prog->aux, env->used_maps,  in release_maps()
11676 			     env->used_map_cnt);  in release_maps()
11680 static void release_btfs(struct bpf_verifier_env *env)  in release_btfs()  argument
11682 	__bpf_free_used_btfs(env->prog->aux, env->used_btfs,  in release_btfs()
11683 			     env->used_btf_cnt);  in release_btfs()
11687 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)  in convert_pseudo_ld_imm64()  argument
11689 	struct bpf_insn *insn = env->prog->insnsi;  in convert_pseudo_ld_imm64()
11690 	int insn_cnt = env->prog->len;  in convert_pseudo_ld_imm64()
11706 static void adjust_insn_aux_data(struct bpf_verifier_env *env,  in adjust_insn_aux_data()  argument
11710 	struct bpf_insn_aux_data *old_data = env->insn_aux_data;  in adjust_insn_aux_data()
11720 	old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);  in adjust_insn_aux_data()
11732 		new_data[i].zext_dst = insn_has_def32(env, insn + i);  in adjust_insn_aux_data()
11734 	env->insn_aux_data = new_data;  in adjust_insn_aux_data()
11738 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)  in adjust_subprog_starts()  argument
11745 	for (i = 0; i <= env->subprog_cnt; i++) {  in adjust_subprog_starts()
11746 		if (env->subprog_info[i].start <= off)  in adjust_subprog_starts()
11748 		env->subprog_info[i].start += len - 1;  in adjust_subprog_starts()
11766 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,  in bpf_patch_insn_data()  argument
11773 		new_data = vzalloc(array_size(env->prog->len + len - 1,  in bpf_patch_insn_data()
11779 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);  in bpf_patch_insn_data()
11782 			verbose(env,  in bpf_patch_insn_data()
11784 				env->insn_aux_data[off].orig_idx);  in bpf_patch_insn_data()
11788 	adjust_insn_aux_data(env, new_data, new_prog, off, len);  in bpf_patch_insn_data()
11789 	adjust_subprog_starts(env, off, len);  in bpf_patch_insn_data()
11794 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,  in adjust_subprog_starts_after_remove()  argument
11800 	for (i = 0; i < env->subprog_cnt; i++)  in adjust_subprog_starts_after_remove()
11801 		if (env->subprog_info[i].start >= off)  in adjust_subprog_starts_after_remove()
11804 	for (j = i; j < env->subprog_cnt; j++)  in adjust_subprog_starts_after_remove()
11805 		if (env->subprog_info[j].start >= off + cnt)  in adjust_subprog_starts_after_remove()
11810 	if (env->subprog_info[j].start != off + cnt)  in adjust_subprog_starts_after_remove()
11814 		struct bpf_prog_aux *aux = env->prog->aux;  in adjust_subprog_starts_after_remove()
11818 		move = env->subprog_cnt + 1 - j;  in adjust_subprog_starts_after_remove()
11820 		memmove(env->subprog_info + i,  in adjust_subprog_starts_after_remove()
11821 			env->subprog_info + j,  in adjust_subprog_starts_after_remove()
11822 			sizeof(*env->subprog_info) * move);  in adjust_subprog_starts_after_remove()
11823 		env->subprog_cnt -= j - i;  in adjust_subprog_starts_after_remove()
11839 		if (env->subprog_info[i].start == off)  in adjust_subprog_starts_after_remove()
11844 	for (; i <= env->subprog_cnt; i++)  in adjust_subprog_starts_after_remove()
11845 		env->subprog_info[i].start -= cnt;  in adjust_subprog_starts_after_remove()
11850 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,  in bpf_adj_linfo_after_remove()  argument
11853 	struct bpf_prog *prog = env->prog;  in bpf_adj_linfo_after_remove()
11900 	for (i = 0; i <= env->subprog_cnt; i++)  in bpf_adj_linfo_after_remove()
11901 		if (env->subprog_info[i].linfo_idx > l_off) {  in bpf_adj_linfo_after_remove()
11905 			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)  in bpf_adj_linfo_after_remove()
11906 				env->subprog_info[i].linfo_idx -= l_cnt;  in bpf_adj_linfo_after_remove()
11908 				env->subprog_info[i].linfo_idx = l_off;  in bpf_adj_linfo_after_remove()
11914 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)  in verifier_remove_insns()  argument
11916 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in verifier_remove_insns()
11917 	unsigned int orig_prog_len = env->prog->len;  in verifier_remove_insns()
11920 	if (bpf_prog_is_dev_bound(env->prog->aux))  in verifier_remove_insns()
11921 		bpf_prog_offload_remove_insns(env, off, cnt);  in verifier_remove_insns()
11923 	err = bpf_remove_insns(env->prog, off, cnt);  in verifier_remove_insns()
11927 	err = adjust_subprog_starts_after_remove(env, off, cnt);  in verifier_remove_insns()
11931 	err = bpf_adj_linfo_after_remove(env, off, cnt);  in verifier_remove_insns()
11952 static void sanitize_dead_code(struct bpf_verifier_env *env)  in sanitize_dead_code()  argument
11954 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in sanitize_dead_code()
11956 	struct bpf_insn *insn = env->prog->insnsi;  in sanitize_dead_code()
11957 	const int insn_cnt = env->prog->len;  in sanitize_dead_code()
11982 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)  in opt_hard_wire_dead_code_branches()  argument
11984 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in opt_hard_wire_dead_code_branches()
11986 	struct bpf_insn *insn = env->prog->insnsi;  in opt_hard_wire_dead_code_branches()
11987 	const int insn_cnt = env->prog->len;  in opt_hard_wire_dead_code_branches()
12001 		if (bpf_prog_is_dev_bound(env->prog->aux))  in opt_hard_wire_dead_code_branches()
12002 			bpf_prog_offload_replace_insn(env, i, &ja);  in opt_hard_wire_dead_code_branches()
12008 static int opt_remove_dead_code(struct bpf_verifier_env *env)  in opt_remove_dead_code()  argument
12010 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in opt_remove_dead_code()
12011 	int insn_cnt = env->prog->len;  in opt_remove_dead_code()
12023 		err = verifier_remove_insns(env, i, j);  in opt_remove_dead_code()
12026 		insn_cnt = env->prog->len;  in opt_remove_dead_code()
12032 static int opt_remove_nops(struct bpf_verifier_env *env)  in opt_remove_nops()  argument
12035 	struct bpf_insn *insn = env->prog->insnsi;  in opt_remove_nops()
12036 	int insn_cnt = env->prog->len;  in opt_remove_nops()
12043 		err = verifier_remove_insns(env, i, 1);  in opt_remove_nops()
12053 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,  in opt_subreg_zext_lo32_rnd_hi32()  argument
12057 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in opt_subreg_zext_lo32_rnd_hi32()
12058 	int i, patch_len, delta = 0, len = env->prog->len;  in opt_subreg_zext_lo32_rnd_hi32()
12059 	struct bpf_insn *insns = env->prog->insnsi;  in opt_subreg_zext_lo32_rnd_hi32()
12091 			if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {  in opt_subreg_zext_lo32_rnd_hi32()
12126 			verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");  in opt_subreg_zext_lo32_rnd_hi32()
12136 		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);  in opt_subreg_zext_lo32_rnd_hi32()
12139 		env->prog = new_prog;  in opt_subreg_zext_lo32_rnd_hi32()
12141 		aux = env->insn_aux_data;  in opt_subreg_zext_lo32_rnd_hi32()
12153 static int convert_ctx_accesses(struct bpf_verifier_env *env)  in convert_ctx_accesses()  argument
12155 	const struct bpf_verifier_ops *ops = env->ops;  in convert_ctx_accesses()
12157 	const int insn_cnt = env->prog->len;  in convert_ctx_accesses()
12164 	if (ops->gen_prologue || env->seen_direct_write) {  in convert_ctx_accesses()
12166 			verbose(env, "bpf verifier is misconfigured\n");  in convert_ctx_accesses()
12169 		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,  in convert_ctx_accesses()
12170 					env->prog);  in convert_ctx_accesses()
12172 			verbose(env, "bpf verifier is misconfigured\n");  in convert_ctx_accesses()
12175 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);  in convert_ctx_accesses()
12179 			env->prog = new_prog;  in convert_ctx_accesses()
12184 	if (bpf_prog_is_dev_bound(env->prog->aux))  in convert_ctx_accesses()
12187 	insn = env->prog->insnsi + delta;  in convert_ctx_accesses()
12214 		    env->insn_aux_data[i + delta].sanitize_stack_spill) {  in convert_ctx_accesses()
12221 			new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);  in convert_ctx_accesses()
12226 			env->prog = new_prog;  in convert_ctx_accesses()
12234 		switch (env->insn_aux_data[i + delta].ptr_type) {  in convert_ctx_accesses()
12254 				env->prog->aux->num_exentries++;  in convert_ctx_accesses()
12255 			} else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {  in convert_ctx_accesses()
12256 				verbose(env, "Writes through BTF pointers are not allowed\n");  in convert_ctx_accesses()
12264 		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;  in convert_ctx_accesses()
12279 				verbose(env, "bpf verifier narrow ctx access misconfigured\n");  in convert_ctx_accesses()
12294 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,  in convert_ctx_accesses()
12298 			verbose(env, "bpf verifier is misconfigured\n");  in convert_ctx_accesses()
12306 				verbose(env, "bpf verifier narrow ctx load misconfigured\n");  in convert_ctx_accesses()
12326 		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in convert_ctx_accesses()
12333 		env->prog = new_prog;  in convert_ctx_accesses()
12340 static int jit_subprogs(struct bpf_verifier_env *env)  in jit_subprogs()  argument
12342 	struct bpf_prog *prog = env->prog, **func, *tmp;  in jit_subprogs()
12349 	if (env->subprog_cnt <= 1)  in jit_subprogs()
12354 			env->insn_aux_data[i].call_imm = insn->imm;  in jit_subprogs()
12365 		subprog = find_subprog(env, i + insn->imm + 1);  in jit_subprogs()
12378 		env->insn_aux_data[i].call_imm = insn->imm;  in jit_subprogs()
12388 	func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);  in jit_subprogs()
12392 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
12394 		subprog_end = env->subprog_info[i + 1].start;  in jit_subprogs()
12432 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;  in jit_subprogs()
12438 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;  in jit_subprogs()
12447 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;  in jit_subprogs()
12460 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
12488 		func[i]->aux->func_cnt = env->subprog_cnt;  in jit_subprogs()
12490 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
12494 			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");  in jit_subprogs()
12504 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
12515 			insn[0].imm = env->insn_aux_data[i].call_imm;  in jit_subprogs()
12516 			insn[1].imm = find_subprog(env, i + insn[0].imm + 1);  in jit_subprogs()
12521 		insn->off = env->insn_aux_data[i].call_imm;  in jit_subprogs()
12522 		subprog = find_subprog(env, i + insn->off + 1);  in jit_subprogs()
12529 	prog->aux->func_cnt = env->subprog_cnt;  in jit_subprogs()
12545 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
12559 		insn->imm = env->insn_aux_data[i].call_imm;  in jit_subprogs()
12565 static int fixup_call_args(struct bpf_verifier_env *env)  in fixup_call_args()  argument
12568 	struct bpf_prog *prog = env->prog;  in fixup_call_args()
12575 	if (env->prog->jit_requested &&  in fixup_call_args()
12576 	    !bpf_prog_is_dev_bound(env->prog->aux)) {  in fixup_call_args()
12577 		err = jit_subprogs(env);  in fixup_call_args()
12585 		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");  in fixup_call_args()
12588 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {  in fixup_call_args()
12592 		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");  in fixup_call_args()
12600 			verbose(env, "callbacks are not allowed in non-JITed programs\n");  in fixup_call_args()
12606 		depth = get_callee_stack_depth(env, insn, i);  in fixup_call_args()
12616 static int fixup_kfunc_call(struct bpf_verifier_env *env,  in fixup_kfunc_call()  argument
12624 	desc = find_kfunc_desc(env->prog, insn->imm);  in fixup_kfunc_call()
12626 		verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",  in fixup_kfunc_call()
12639 static int do_misc_fixups(struct bpf_verifier_env *env)  in do_misc_fixups()  argument
12641 	struct bpf_prog *prog = env->prog;  in do_misc_fixups()
12686 			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);  in do_misc_fixups()
12691 			env->prog = prog = new_prog;  in do_misc_fixups()
12700 			cnt = env->ops->gen_ld_abs(insn, insn_buf);  in do_misc_fixups()
12702 				verbose(env, "bpf verifier is misconfigured\n");  in do_misc_fixups()
12706 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
12711 			env->prog = prog = new_prog;  in do_misc_fixups()
12725 			aux = &env->insn_aux_data[i + delta];  in do_misc_fixups()
12759 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
12764 			env->prog = prog = new_prog;  in do_misc_fixups()
12774 			ret = fixup_kfunc_call(env, insn);  in do_misc_fixups()
12793 			if (!allow_tail_call_in_subprogs(env))  in do_misc_fixups()
12805 			aux = &env->insn_aux_data[i + delta];  in do_misc_fixups()
12806 			if (env->bpf_capable && !expect_blinding &&  in do_misc_fixups()
12820 					verbose(env, "adding tail call poke descriptor failed\n");  in do_misc_fixups()
12838 				verbose(env, "tail_call abusing map_ptr\n");  in do_misc_fixups()
12851 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
12856 			env->prog = prog = new_prog;  in do_misc_fixups()
12884 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
12889 			env->prog = prog = new_prog;  in do_misc_fixups()
12906 			aux = &env->insn_aux_data[i + delta];  in do_misc_fixups()
12918 					verbose(env, "bpf verifier is misconfigured\n");  in do_misc_fixups()
12922 				new_prog = bpf_patch_insn_data(env, i + delta,  in do_misc_fixups()
12928 				env->prog = prog = new_prog;  in do_misc_fixups()
12999 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,  in do_misc_fixups()
13005 			env->prog = prog = new_prog;  in do_misc_fixups()
13016 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);  in do_misc_fixups()
13020 			env->prog = prog = new_prog;  in do_misc_fixups()
13026 		fn = env->ops->get_func_proto(insn->imm, env->prog);  in do_misc_fixups()
13031 			verbose(env,  in do_misc_fixups()
13045 			verbose(env, "bpf verifier is misconfigured\n");  in do_misc_fixups()
13051 			verbose(env, "tracking tail call prog failed\n");  in do_misc_fixups()
13056 	sort_kfunc_descs_by_imm(env->prog);  in do_misc_fixups()
13061 static void free_states(struct bpf_verifier_env *env)  in free_states()  argument
13066 	sl = env->free_list;  in free_states()
13073 	env->free_list = NULL;  in free_states()
13075 	if (!env->explored_states)  in free_states()
13078 	for (i = 0; i < state_htab_size(env); i++) {  in free_states()
13079 		sl = env->explored_states[i];  in free_states()
13087 		env->explored_states[i] = NULL;  in free_states()
13091 static int do_check_common(struct bpf_verifier_env *env, int subprog)  in do_check_common()  argument
13093 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);  in do_check_common()
13098 	env->prev_linfo = NULL;  in do_check_common()
13099 	env->pass_cnt++;  in do_check_common()
13112 	env->cur_state = state;  in do_check_common()
13113 	init_func_state(env, state->frame[0],  in do_check_common()
13119 	if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {  in do_check_common()
13120 		ret = btf_prepare_func_args(env, subprog, regs);  in do_check_common()
13125 				mark_reg_known_zero(env, regs, i);  in do_check_common()
13127 				mark_reg_unknown(env, regs, i);  in do_check_common()
13131 				mark_reg_known_zero(env, regs, i);  in do_check_common()
13133 				regs[i].id = ++env->id_gen;  in do_check_common()
13139 		mark_reg_known_zero(env, regs, BPF_REG_1);  in do_check_common()
13140 		ret = btf_check_subprog_arg_match(env, subprog, regs);  in do_check_common()
13154 	ret = do_check(env);  in do_check_common()
13159 	if (env->cur_state) {  in do_check_common()
13160 		free_verifier_state(env->cur_state, true);  in do_check_common()
13161 		env->cur_state = NULL;  in do_check_common()
13163 	while (!pop_stack(env, NULL, NULL, false));  in do_check_common()
13165 		bpf_vlog_reset(&env->log, 0);  in do_check_common()
13166 	free_states(env);  in do_check_common()
13187 static int do_check_subprogs(struct bpf_verifier_env *env)  in do_check_subprogs()  argument
13189 	struct bpf_prog_aux *aux = env->prog->aux;  in do_check_subprogs()
13195 	for (i = 1; i < env->subprog_cnt; i++) {  in do_check_subprogs()
13198 		env->insn_idx = env->subprog_info[i].start;  in do_check_subprogs()
13199 		WARN_ON_ONCE(env->insn_idx == 0);  in do_check_subprogs()
13200 		ret = do_check_common(env, i);  in do_check_subprogs()
13203 		} else if (env->log.level & BPF_LOG_LEVEL) {  in do_check_subprogs()
13204 			verbose(env,  in do_check_subprogs()
13212 static int do_check_main(struct bpf_verifier_env *env)  in do_check_main()  argument
13216 	env->insn_idx = 0;  in do_check_main()
13217 	ret = do_check_common(env, 0);  in do_check_main()
13219 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;  in do_check_main()
13224 static void print_verification_stats(struct bpf_verifier_env *env)  in print_verification_stats()  argument
13228 	if (env->log.level & BPF_LOG_STATS) {  in print_verification_stats()
13229 		verbose(env, "verification time %lld usec\n",  in print_verification_stats()
13230 			div_u64(env->verification_time, 1000));  in print_verification_stats()
13231 		verbose(env, "stack depth ");  in print_verification_stats()
13232 		for (i = 0; i < env->subprog_cnt; i++) {  in print_verification_stats()
13233 			u32 depth = env->subprog_info[i].stack_depth;  in print_verification_stats()
13235 			verbose(env, "%d", depth);  in print_verification_stats()
13236 			if (i + 1 < env->subprog_cnt)  in print_verification_stats()
13237 				verbose(env, "+");  in print_verification_stats()
13239 		verbose(env, "\n");  in print_verification_stats()
13241 	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "  in print_verification_stats()
13243 		env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,  in print_verification_stats()
13244 		env->max_states_per_insn, env->total_states,  in print_verification_stats()
13245 		env->peak_states, env->longest_mark_read_walk);  in print_verification_stats()
13248 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)  in check_struct_ops_btf_id()  argument
13253 	struct bpf_prog *prog = env->prog;  in check_struct_ops_btf_id()
13258 		verbose(env, "struct ops programs must have a GPL compatible license\n");  in check_struct_ops_btf_id()
13265 		verbose(env, "attach_btf_id %u is not a supported struct\n",  in check_struct_ops_btf_id()
13273 		verbose(env, "attach to invalid member idx %u of struct %s\n",  in check_struct_ops_btf_id()
13283 		verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",  in check_struct_ops_btf_id()
13292 			verbose(env, "attach to unsupported member %s of struct %s\n",  in check_struct_ops_btf_id()
13300 	env->ops = st_ops->verifier_ops;  in check_struct_ops_btf_id()
13577 static int check_attach_btf_id(struct bpf_verifier_env *env)  in BTF_SET_START()
13579 	struct bpf_prog *prog = env->prog;  in BTF_SET_START()
13591 		verbose(env, "Syscall programs can only be sleepable\n");  in BTF_SET_START()
13597 		verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");  in BTF_SET_START()
13602 		return check_struct_ops_btf_id(env);  in BTF_SET_START()
13609 	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);  in BTF_SET_START()
13618 		env->ops = bpf_verifier_ops[tgt_prog->type];  in BTF_SET_START()
13641 		ret = bpf_lsm_verify_prog(&env->log, prog);  in BTF_SET_START()
13672 	struct bpf_verifier_env *env;  in bpf_check()  local
13684 	env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);  in bpf_check()
13685 	if (!env)  in bpf_check()
13687 	log = &env->log;  in bpf_check()
13690 	env->insn_aux_data =  in bpf_check()
13693 	if (!env->insn_aux_data)  in bpf_check()
13696 		env->insn_aux_data[i].orig_idx = i;  in bpf_check()
13697 	env->prog = *prog;  in bpf_check()
13698 	env->ops = bpf_verifier_ops[env->prog->type];  in bpf_check()
13699 	env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);  in bpf_check()
13725 		verbose(env, "in-kernel BTF is malformed\n");  in bpf_check()
13730 	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);  in bpf_check()
13732 		env->strict_alignment = true;  in bpf_check()
13734 		env->strict_alignment = false;  in bpf_check()
13736 	env->allow_ptr_leaks = bpf_allow_ptr_leaks();  in bpf_check()
13737 	env->allow_uninit_stack = bpf_allow_uninit_stack();  in bpf_check()
13738 	env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();  in bpf_check()
13739 	env->bypass_spec_v1 = bpf_bypass_spec_v1();  in bpf_check()
13740 	env->bypass_spec_v4 = bpf_bypass_spec_v4();  in bpf_check()
13741 	env->bpf_capable = bpf_capable();  in bpf_check()
13744 		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;  in bpf_check()
13746 	env->explored_states = kvcalloc(state_htab_size(env),  in bpf_check()
13750 	if (!env->explored_states)  in bpf_check()
13753 	ret = add_subprog_and_kfunc(env);  in bpf_check()
13757 	ret = check_subprogs(env);  in bpf_check()
13761 	ret = check_btf_info(env, attr, uattr);  in bpf_check()
13765 	ret = check_attach_btf_id(env);  in bpf_check()
13769 	ret = resolve_pseudo_ldimm64(env);  in bpf_check()
13773 	if (bpf_prog_is_dev_bound(env->prog->aux)) {  in bpf_check()
13774 		ret = bpf_prog_offload_verifier_prep(env->prog);  in bpf_check()
13779 	ret = check_cfg(env);  in bpf_check()
13783 	ret = do_check_subprogs(env);  in bpf_check()
13784 	ret = ret ?: do_check_main(env);  in bpf_check()
13786 	if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))  in bpf_check()
13787 		ret = bpf_prog_offload_finalize(env);  in bpf_check()
13790 	kvfree(env->explored_states);  in bpf_check()
13793 		ret = check_max_stack_depth(env);  in bpf_check()
13798 			opt_hard_wire_dead_code_branches(env);  in bpf_check()
13800 			ret = opt_remove_dead_code(env);  in bpf_check()
13802 			ret = opt_remove_nops(env);  in bpf_check()
13805 			sanitize_dead_code(env);  in bpf_check()
13810 		ret = convert_ctx_accesses(env);  in bpf_check()
13813 		ret = do_misc_fixups(env);  in bpf_check()
13818 	if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {  in bpf_check()
13819 		ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);  in bpf_check()
13820 		env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret  in bpf_check()
13825 		ret = fixup_call_args(env);  in bpf_check()
13827 	env->verification_time = ktime_get_ns() - start_time;  in bpf_check()
13828 	print_verification_stats(env);  in bpf_check()
13840 	if (env->used_map_cnt) {  in bpf_check()
13842 		env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,  in bpf_check()
13843 							  sizeof(env->used_maps[0]),  in bpf_check()
13846 		if (!env->prog->aux->used_maps) {  in bpf_check()
13851 		memcpy(env->prog->aux->used_maps, env->used_maps,  in bpf_check()
13852 		       sizeof(env->used_maps[0]) * env->used_map_cnt);  in bpf_check()
13853 		env->prog->aux->used_map_cnt = env->used_map_cnt;  in bpf_check()
13855 	if (env->used_btf_cnt) {  in bpf_check()
13857 		env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,  in bpf_check()
13858 							  sizeof(env->used_btfs[0]),  in bpf_check()
13860 		if (!env->prog->aux->used_btfs) {  in bpf_check()
13865 		memcpy(env->prog->aux->used_btfs, env->used_btfs,  in bpf_check()
13866 		       sizeof(env->used_btfs[0]) * env->used_btf_cnt);  in bpf_check()
13867 		env->prog->aux->used_btf_cnt = env->used_btf_cnt;  in bpf_check()
13869 	if (env->used_map_cnt || env->used_btf_cnt) {  in bpf_check()
13873 		convert_pseudo_ld_imm64(env);  in bpf_check()
13876 	adjust_btf_func(env);  in bpf_check()
13879 	if (!env->prog->aux->used_maps)  in bpf_check()
13883 		release_maps(env);  in bpf_check()
13884 	if (!env->prog->aux->used_btfs)  in bpf_check()
13885 		release_btfs(env);  in bpf_check()
13890 	if (env->prog->type == BPF_PROG_TYPE_EXT)  in bpf_check()
13891 		env->prog->expected_attach_type = 0;  in bpf_check()
13893 	*prog = env->prog;  in bpf_check()
13897 	vfree(env->insn_aux_data);  in bpf_check()
13899 	kfree(env);  in bpf_check()