191 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
192 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
274 find_linfo(const struct bpf_verifier_env *env, u32 insn_off)  in find_linfo()
335 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,  in bpf_verifier_log_write()
351 	struct bpf_verifier_env *env = private_data;  in verbose()
384 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,  in verbose_linfo()
412 static void verbose_invalid_scalar(struct bpf_verifier_env *env,  in verbose_invalid_scalar()
541 static const char *reg_type_str(struct bpf_verifier_env *env,  in reg_type_str()
600 static void print_liveness(struct bpf_verifier_env *env,  in print_liveness()
632 static struct bpf_func_state *func(struct bpf_verifier_env *env,  in func()
645 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)  in mark_reg_scratched()
650 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)  in mark_stack_slot_scratched()
655 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)  in reg_scratched()
660 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)  in stack_slot_scratched()
665 static bool verifier_state_scratched(const struct bpf_verifier_env *env)  in verifier_state_scratched()
670 static void mark_verifier_state_clean(struct bpf_verifier_env *env)  in mark_verifier_state_clean()
677 static void mark_verifier_state_scratched(struct bpf_verifier_env *env)  in mark_verifier_state_scratched()
700 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in mark_stack_slots_dynptr()
738 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in unmark_stack_slots_dynptr()
767 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in is_dynptr_reg_valid_uninit()
785 bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env,  in is_dynptr_reg_valid_init()
805 bool is_dynptr_type_expected(struct bpf_verifier_env *env,  in is_dynptr_type_expected()
836 static void print_verifier_state(struct bpf_verifier_env *env,  in print_verifier_state()
981 static void print_insn_state(struct bpf_verifier_env *env,  in print_insn_state()
1104 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)  in acquire_reference_state()
1228 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)  in update_branch_counts()
1245 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,  in pop_stack()
1274 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,  in push_stack()
1327 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1379 static void mark_reg_known_zero(struct bpf_verifier_env *env,  in mark_reg_known_zero()
1681 static void __mark_reg_unknown(const struct bpf_verifier_env *env,  in __mark_reg_unknown()
1696 static void mark_reg_unknown(struct bpf_verifier_env *env,  in mark_reg_unknown()
1709 static void __mark_reg_not_init(const struct bpf_verifier_env *env,  in __mark_reg_not_init()
1716 static void mark_reg_not_init(struct bpf_verifier_env *env,  in mark_reg_not_init()
1729 static void mark_btf_ld_reg(struct bpf_verifier_env *env,  in mark_btf_ld_reg()
1746 static void init_reg_state(struct bpf_verifier_env *env,  in init_reg_state()
1766 static void init_func_state(struct bpf_verifier_env *env,  in init_func_state()
1779 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,  in push_async_cb()
1838 static int find_subprog(struct bpf_verifier_env *env, int off)  in find_subprog()
1850 static int add_subprog(struct bpf_verifier_env *env, int off)  in add_subprog()
1930 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,  in __find_kfunc_desc_btf()
2000 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)  in find_kfunc_desc_btf()
2016 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)  in add_kfunc_call()
2177 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)  in add_subprog_and_kfunc()
2219 static int check_subprogs(struct bpf_verifier_env *env)  in check_subprogs()
2271 static int mark_reg_read(struct bpf_verifier_env *env,  in mark_reg_read()
2323 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,  in is_reg64()
2431 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)  in insn_has_def32()
2441 static void mark_insn_zext(struct bpf_verifier_env *env,  in mark_insn_zext()
2454 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,  in check_reg_arg()
2502 static int push_jmp_history(struct bpf_verifier_env *env,  in push_jmp_history()
2556 static int backtrack_insn(struct bpf_verifier_env *env, int idx,  in backtrack_insn()
2741 static void mark_all_scalars_precise(struct bpf_verifier_env *env,  in mark_all_scalars_precise()
2771 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,  in __mark_chain_precision()
2933 int mark_chain_precision(struct bpf_verifier_env *env, int regno)  in mark_chain_precision()
2938 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)  in mark_chain_precision_stack()
3024 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,  in check_stack_write_fixed_off()
3149 static int check_stack_write_var_off(struct bpf_verifier_env *env,  in check_stack_write_var_off()
3244 static void mark_reg_stack_read(struct bpf_verifier_env *env,  in mark_reg_stack_read()
3295 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,  in check_stack_read_fixed_off()
3394 static int check_stack_range_initialized(struct bpf_verifier_env *env,
3400 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)  in reg_state()
3418 static int check_stack_read_var_off(struct bpf_verifier_env *env,  in check_stack_read_var_off()
3449 static int check_stack_read(struct bpf_verifier_env *env,  in check_stack_read()
3510 static int check_stack_write(struct bpf_verifier_env *env,  in check_stack_write()
3533 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,  in check_map_access_type()
3556 static int __check_mem_access(struct bpf_verifier_env *env, int regno,  in __check_mem_access()
3592 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,  in check_mem_region_access()
3647 static int __check_ptr_off_reg(struct bpf_verifier_env *env,  in __check_ptr_off_reg()
3679 int check_ptr_off_reg(struct bpf_verifier_env *env,  in check_ptr_off_reg()
3685 static int map_kptr_match_type(struct bpf_verifier_env *env,  in map_kptr_match_type()
3757 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,  in check_map_kptr_access()
3813 static int check_map_access(struct bpf_verifier_env *env, u32 regno,  in check_map_access()
3886 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,  in may_access_direct_pkt_data()
3928 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,  in check_packet_access()
3971 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,  in check_ctx_access()
4007 static int check_flow_keys_access(struct bpf_verifier_env *env, int off,  in check_flow_keys_access()
4019 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,  in check_sock_access()
4064 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)  in is_pointer_value()
4069 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)  in is_ctx_reg()
4076 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)  in is_sk_reg()
4083 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)  in is_pkt_reg()
4090 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)  in is_flow_key_reg()
4098 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,  in check_pkt_ptr_alignment()
4133 static int check_generic_ptr_alignment(struct bpf_verifier_env *env,  in check_generic_ptr_alignment()
4157 static int check_ptr_alignment(struct bpf_verifier_env *env,  in check_ptr_alignment()
4210 static int update_stack_depth(struct bpf_verifier_env *env,  in update_stack_depth()
4230 static int check_max_stack_depth(struct bpf_verifier_env *env)  in check_max_stack_depth()
4339 static int get_callee_stack_depth(struct bpf_verifier_env *env,  in get_callee_stack_depth()
4354 static int __check_buffer_access(struct bpf_verifier_env *env,  in __check_buffer_access()
4378 static int check_tp_buffer_access(struct bpf_verifier_env *env,  in check_tp_buffer_access()
4394 static int check_buffer_access(struct bpf_verifier_env *env,  in check_buffer_access()
4501 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,  in check_ptr_to_btf_access()
4572 static int check_ptr_to_map_access(struct bpf_verifier_env *env,  in check_ptr_to_map_access()
4656 		struct bpf_verifier_env *env,  in check_stack_access_within_bounds()
4720 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,  in check_mem_access()
4960 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)  in check_atomic()
5074 		struct bpf_verifier_env *env, int regno, int off,  in check_stack_range_initialized()
5200 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,  in check_helper_mem_access()
5290 static int check_mem_size_reg(struct bpf_verifier_env *env,  in check_mem_size_reg()
5345 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_mem_reg()
5377 int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_kfunc_mem_size_reg()
5424 static int process_spin_lock(struct bpf_verifier_env *env, int regno,  in process_spin_lock()
5486 static int process_timer_func(struct bpf_verifier_env *env, int regno,  in process_timer_func()
5534 static int process_kptr_func(struct bpf_verifier_env *env, int regno,  in process_kptr_func()
5607 static int resolve_map_arg_type(struct bpf_verifier_env *env,  in resolve_map_arg_type()
5745 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,  in check_reg_type()
5833 int check_func_arg_reg_off(struct bpf_verifier_env *env,  in check_func_arg_reg_off()
5889 static u32 stack_slot_get_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in stack_slot_get_id()
5897 static int check_func_arg(struct bpf_verifier_env *env, u32 arg,  in check_func_arg()
6214 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)  in may_update_sockmap()
6246 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)  in allow_tail_call_in_subprogs()
6252 static int check_map_func_compatibility(struct bpf_verifier_env *env,  in check_map_func_compatibility()
6574 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)  in clear_all_pkt_pointers()
6614 static int release_reference(struct bpf_verifier_env *env,  in release_reference()
6637 static void clear_caller_saved_regs(struct bpf_verifier_env *env,  in clear_caller_saved_regs()
6649 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
6654 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in __check_func_call()
6776 int map_set_for_each_callback_args(struct bpf_verifier_env *env,  in map_set_for_each_callback_args()
6803 static int set_callee_state(struct bpf_verifier_env *env,  in set_callee_state()
6817 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_func_call()
6833 static int set_map_elem_callback_state(struct bpf_verifier_env *env,  in set_map_elem_callback_state()
6863 static int set_loop_callback_state(struct bpf_verifier_env *env,  in set_loop_callback_state()
6885 static int set_timer_callback_state(struct bpf_verifier_env *env,  in set_timer_callback_state()
6915 static int set_find_vma_callback_state(struct bpf_verifier_env *env,  in set_find_vma_callback_state()
6943 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,  in set_user_ringbuf_callback_state()
6967 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)  in prepare_func_exit()
7052 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,  in record_func_map()
7098 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,  in record_func_key()
7134 static int check_reference_leak(struct bpf_verifier_env *env)  in check_reference_leak()
7153 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,  in check_bpf_snprintf_call()
7190 static int check_get_func_ip(struct bpf_verifier_env *env)  in check_get_func_ip()
7211 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)  in cur_aux()
7216 static bool loop_flag_is_zero(struct bpf_verifier_env *env)  in loop_flag_is_zero()
7228 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)  in update_loop_inline_state()
7246 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_helper_call()
7649 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,  in mark_btf_func_reg_size()
7670 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_kfunc_call()
7845 static bool check_reg_sane_offset(struct bpf_verifier_env *env,  in check_reg_sane_offset()
7919 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,  in can_skip_alu_sanitation()
7942 static int sanitize_val_alu(struct bpf_verifier_env *env,  in sanitize_val_alu()
7964 sanitize_speculative_path(struct bpf_verifier_env *env,  in sanitize_speculative_path()
7984 static int sanitize_ptr_alu(struct bpf_verifier_env *env,  in sanitize_ptr_alu()
8080 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)  in sanitize_mark_insn_seen()
8093 static int sanitize_err(struct bpf_verifier_env *env,  in sanitize_err()
8143 				struct bpf_verifier_env *env,  in check_stack_access_for_ptr_arithmetic()
8166 static int sanitize_check_bounds(struct bpf_verifier_env *env,  in sanitize_check_bounds()
8203 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,  in adjust_ptr_min_max_vals()
9000 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,  in adjust_scalar_min_max_vals()
9161 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,  in adjust_reg_min_max_vals()
9242 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_alu_op()
10130 static int check_cond_jmp_op(struct bpf_verifier_env *env,  in check_cond_jmp_op()
10323 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_imm()
10442 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_abs()
10519 static int check_return_code(struct bpf_verifier_env *env)  in check_return_code()
10721 static u32 state_htab_size(struct bpf_verifier_env *env)  in state_htab_size()
10727 					struct bpf_verifier_env *env,  in explored_state()
10736 static void init_explored_state(struct bpf_verifier_env *env, int idx)  in init_explored_state()
10751 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env,  in push_insn()
10800 				struct bpf_verifier_env *env,  in visit_func_call_insn()
10828 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env)  in visit_insn()
10893 static int check_cfg(struct bpf_verifier_env *env)  in check_cfg()
10956 static int check_abnormal_return(struct bpf_verifier_env *env)  in check_abnormal_return()
10977 static int check_btf_func(struct bpf_verifier_env *env,  in check_btf_func()
11108 static void adjust_btf_func(struct bpf_verifier_env *env)  in adjust_btf_func()
11123 static int check_btf_line(struct bpf_verifier_env *env,  in check_btf_line()
11251 static int check_core_relo(struct bpf_verifier_env *env,  in check_core_relo()
11320 static int check_btf_info(struct bpf_verifier_env *env,  in check_btf_info()
11400 static void clean_func_state(struct bpf_verifier_env *env,  in clean_func_state()
11429 static void clean_verifier_state(struct bpf_verifier_env *env,  in clean_verifier_state()
11474 static void clean_live_states(struct bpf_verifier_env *env, int insn,  in clean_live_states()
11497 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,  in regsafe()
11615 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,  in stacksafe()
11712 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,  in func_states_equal()
11732 static bool states_equal(struct bpf_verifier_env *env,  in states_equal()
11765 static int propagate_liveness_reg(struct bpf_verifier_env *env,  in propagate_liveness_reg()
11798 static int propagate_liveness(struct bpf_verifier_env *env,  in propagate_liveness()
11845 static int propagate_precision(struct bpf_verifier_env *env,  in propagate_precision()
11901 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)  in is_state_visited()
12157 static int do_check(struct bpf_verifier_env *env)  in do_check()
12529 static int check_pseudo_btf_id(struct bpf_verifier_env *env,  in check_pseudo_btf_id()
12673 static int check_map_prog_compatibility(struct bpf_verifier_env *env,  in check_map_prog_compatibility()
12753 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)  in resolve_pseudo_ldimm64()
12937 static void release_maps(struct bpf_verifier_env *env)  in release_maps()
12944 static void release_btfs(struct bpf_verifier_env *env)  in release_btfs()
12951 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)  in convert_pseudo_ld_imm64()
12970 static void adjust_insn_aux_data(struct bpf_verifier_env *env,  in adjust_insn_aux_data()
13002 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)  in adjust_subprog_starts()
13030 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,  in bpf_patch_insn_data()
13058 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,  in adjust_subprog_starts_after_remove()
13114 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,  in bpf_adj_linfo_after_remove()
13178 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)  in verifier_remove_insns()
13216 static void sanitize_dead_code(struct bpf_verifier_env *env)  in sanitize_dead_code()
13246 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)  in opt_hard_wire_dead_code_branches()
13272 static int opt_remove_dead_code(struct bpf_verifier_env *env)  in opt_remove_dead_code()
13296 static int opt_remove_nops(struct bpf_verifier_env *env)  in opt_remove_nops()
13317 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,  in opt_subreg_zext_lo32_rnd_hi32()
13417 static int convert_ctx_accesses(struct bpf_verifier_env *env)  in convert_ctx_accesses()
13602 static int jit_subprogs(struct bpf_verifier_env *env)  in jit_subprogs()
13830 static int fixup_call_args(struct bpf_verifier_env *env)  in fixup_call_args()
13881 static int fixup_kfunc_call(struct bpf_verifier_env *env,  in fixup_kfunc_call()
13909 static int do_misc_fixups(struct bpf_verifier_env *env)  in do_misc_fixups()
14427 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,  in inline_bpf_loop()
14518 static int optimize_bpf_loop(struct bpf_verifier_env *env)  in optimize_bpf_loop()
14563 static void free_states(struct bpf_verifier_env *env)  in free_states()
14593 static int do_check_common(struct bpf_verifier_env *env, int subprog)  in do_check_common()
14689 static int do_check_subprogs(struct bpf_verifier_env *env)  in do_check_subprogs()
14714 static int do_check_main(struct bpf_verifier_env *env)  in do_check_main()
14726 static void print_verification_stats(struct bpf_verifier_env *env)  in print_verification_stats()
14750 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)  in check_struct_ops_btf_id()
15080 static int check_attach_btf_id(struct bpf_verifier_env *env)  in BTF_SET_START()
15175 	struct bpf_verifier_env *env;  in bpf_check()
15187 	env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);  in bpf_check()