419 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)  in reg_may_point_to_spin_lock()
552 				   const struct bpf_reg_state *reg)  in func()
568 	const struct bpf_reg_state *reg;  in print_verifier_state()
1011 				struct bpf_reg_state *reg);
1014 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)  in ___mark_reg_known()
1031 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)  in __mark_reg_known()
1035 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));  in __mark_reg_known()
1039 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)  in __mark_reg32_known()
1051 static void __mark_reg_known_zero(struct bpf_reg_state *reg)  in __mark_reg_known_zero()
1056 static void __mark_reg_const_zero(struct bpf_reg_state *reg)  in __mark_reg_const_zero()
1063 				struct bpf_reg_state *regs, u32 regno)  in mark_reg_known_zero()
1075 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)  in reg_is_pkt_pointer()
1080 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)  in reg_is_pkt_pointer_any()
1087 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,  in reg_is_init_pkt_pointer()
1101 static void __mark_reg_unbounded(struct bpf_reg_state *reg)  in __mark_reg_unbounded()
1114 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)  in __mark_reg64_unbounded()
1122 static void __mark_reg32_unbounded(struct bpf_reg_state *reg)  in __mark_reg32_unbounded()
1130 static void __update_reg32_bounds(struct bpf_reg_state *reg)  in __update_reg32_bounds()
1145 static void __update_reg64_bounds(struct bpf_reg_state *reg)  in __update_reg64_bounds()
1158 static void __update_reg_bounds(struct bpf_reg_state *reg)  in __update_reg_bounds()
1165 static void __reg32_deduce_bounds(struct bpf_reg_state *reg)  in __reg32_deduce_bounds()
1199 static void __reg64_deduce_bounds(struct bpf_reg_state *reg)  in __reg64_deduce_bounds()
1233 static void __reg_deduce_bounds(struct bpf_reg_state *reg)  in __reg_deduce_bounds()
1240 static void __reg_bound_offset(struct bpf_reg_state *reg)  in __reg_bound_offset()
1252 static void __reg_assign_32_into_64(struct bpf_reg_state *reg)  in __reg_assign_32_into_64()
1270 static void __reg_combine_32_into_64(struct bpf_reg_state *reg)  in __reg_combine_32_into_64()
1311 static void __reg_combine_64_into_32(struct bpf_reg_state *reg)  in __reg_combine_64_into_32()
1335 			       struct bpf_reg_state *reg)  in __mark_reg_unknown()
1341 	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));  in __mark_reg_unknown()
1350 			     struct bpf_reg_state *regs, u32 regno)  in mark_reg_unknown()
1363 				struct bpf_reg_state *reg)  in __mark_reg_not_init()
1370 			      struct bpf_reg_state *regs, u32 regno)  in mark_reg_not_init()
1383 			    struct bpf_reg_state *regs, u32 regno,  in mark_btf_ld_reg()
1399 	struct bpf_reg_state *regs = state->regs;  in init_reg_state()
1555 			 const struct bpf_reg_state *state,  in mark_reg_read()
1556 			 struct bpf_reg_state *parent, u8 flag)  in mark_reg_read()
1607 		     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)  in is_reg64()
1705 			   struct bpf_reg_state *reg)  in mark_insn_zext()
1723 	struct bpf_reg_state *reg, *regs = state->regs;  in check_reg_arg()
1993 	struct bpf_reg_state *reg;  in mark_all_scalars_precise()
2026 	struct bpf_reg_state *reg;  in __mark_chain_precision()
2224 static bool register_is_null(struct bpf_reg_state *reg)  in register_is_null()
2229 static bool register_is_const(struct bpf_reg_state *reg)  in register_is_const()
2234 static bool __is_scalar_unbounded(struct bpf_reg_state *reg)  in __is_scalar_unbounded()
2243 static bool register_is_bounded(struct bpf_reg_state *reg)  in register_is_bounded()
2249 			       const struct bpf_reg_state *reg)  in __is_pointer_value()
2258 				int spi, struct bpf_reg_state *reg)  in save_register_state()
2279 	struct bpf_reg_state *reg = NULL;  in check_stack_write()
2407 	struct bpf_reg_state *reg;  in check_stack_read()
2502 			      const struct bpf_reg_state *reg,  in check_stack_access()
2529 	struct bpf_reg_state *regs = cur_regs(env);  in check_map_access_type()
2554 	struct bpf_reg_state *reg;  in __check_mem_access()
2587 	struct bpf_reg_state *reg = &state->regs[regno];  in check_mem_region_access()
2645 	struct bpf_reg_state *reg = &state->regs[regno];  in check_map_access()
2723 	struct bpf_reg_state *regs = cur_regs(env);  in check_packet_access()
2724 	struct bpf_reg_state *reg = &regs[regno];  in check_packet_access()
2811 	struct bpf_reg_state *regs = cur_regs(env);  in check_sock_access()
2812 	struct bpf_reg_state *reg = &regs[regno];  in check_sock_access()
2852 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)  in reg_state()
2864 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_ctx_reg()
2871 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_sk_reg()
2878 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_pkt_reg()
2885 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_flow_key_reg()
2892 				   const struct bpf_reg_state *reg,  in check_pkt_ptr_alignment()
2927 				       const struct bpf_reg_state *reg,  in check_generic_ptr_alignment()
2951 			       const struct bpf_reg_state *reg, int off,  in check_ptr_alignment()
3134 		  const struct bpf_reg_state *reg, int regno)  in check_ctx_reg()
3159 				 const struct bpf_reg_state *reg,  in __check_buffer_access()
3182 				  const struct bpf_reg_state *reg,  in check_tp_buffer_access()
3198 			       const struct bpf_reg_state *reg,  in check_buffer_access()
3217 static void zext_32_to_64(struct bpf_reg_state *reg)  in zext_32_to_64()
3226 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)  in coerce_reg_to_size()
3290 				   struct bpf_reg_state *regs,  in check_ptr_to_btf_access()
3295 	struct bpf_reg_state *reg = regs + regno;  in check_ptr_to_btf_access()
3340 				   struct bpf_reg_state *regs,  in check_ptr_to_map_access()
3345 	struct bpf_reg_state *reg = regs + regno;  in check_ptr_to_map_access()
3405 	struct bpf_reg_state *regs = cur_regs(env);  in check_mem_access()
3406 	struct bpf_reg_state *reg = regs + regno;  in check_mem_access()
3647 	struct bpf_reg_state *reg = reg_state(env, regno);  in __check_stack_boundary()
3676 	struct bpf_reg_state *reg = reg_state(env, regno);  in check_stack_boundary()
3794 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_helper_mem_access()
3862 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_spin_lock()
4081 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_reg_type()
4140 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_func_arg()
4664 	struct bpf_reg_state *regs = state->regs, *reg;  in __clear_all_pkt_pointers()
4692 	struct bpf_reg_state *regs = state->regs, *reg;  in release_reg_references()
4728 				    struct bpf_reg_state *regs)  in clear_caller_saved_regs()
4842 	struct bpf_reg_state *r0;  in prepare_func_exit()
4881 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,  in do_refine_retval_range()
4885 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];  in do_refine_retval_range()
4951 	struct bpf_reg_state *regs = cur_regs(env), *reg;  in record_func_key()
5000 	struct bpf_reg_state *regs;  in check_helper_call()
5289 				  const struct bpf_reg_state *reg,  in check_reg_sane_offset()
5328 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,  in retrieve_ptr_limit()
5395 			    const struct bpf_reg_state *ptr_reg,  in sanitize_ptr_alu()
5396 			    struct bpf_reg_state *dst_reg,  in sanitize_ptr_alu()
5404 	struct bpf_reg_state tmp;  in sanitize_ptr_alu()
5452 				   const struct bpf_reg_state *ptr_reg,  in adjust_ptr_min_max_vals()
5453 				   const struct bpf_reg_state *off_reg)  in adjust_ptr_min_max_vals()
5457 	struct bpf_reg_state *regs = state->regs, *dst_reg;  in adjust_ptr_min_max_vals()
5699 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,  in scalar32_min_max_add()
5700 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_add()
5725 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,  in scalar_min_max_add()
5726 			       struct bpf_reg_state *src_reg)  in scalar_min_max_add()
5751 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,  in scalar32_min_max_sub()
5752 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_sub()
5779 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,  in scalar_min_max_sub()
5780 			       struct bpf_reg_state *src_reg)  in scalar_min_max_sub()
5807 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,  in scalar32_min_max_mul()
5808 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_mul()
5839 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,  in scalar_min_max_mul()
5840 			       struct bpf_reg_state *src_reg)  in scalar_min_max_mul()
5871 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,  in scalar32_min_max_and()
5872 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_and()
5907 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,  in scalar_min_max_and()
5908 			       struct bpf_reg_state *src_reg)  in scalar_min_max_and()
5942 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,  in scalar32_min_max_or()
5943 				struct bpf_reg_state *src_reg)  in scalar32_min_max_or()
5977 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,  in scalar_min_max_or()
5978 			      struct bpf_reg_state *src_reg)  in scalar_min_max_or()
6012 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,  in scalar32_min_max_xor()
6013 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_xor()
6042 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,  in scalar_min_max_xor()
6043 			       struct bpf_reg_state *src_reg)  in scalar_min_max_xor()
6073 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,  in __scalar32_min_max_lsh()
6091 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_lsh()
6092 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_lsh()
6109 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,  in __scalar64_min_max_lsh()
6139 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_lsh()
6140 			       struct bpf_reg_state *src_reg)  in scalar_min_max_lsh()
6154 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_rsh()
6155 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_rsh()
6186 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_rsh()
6187 			       struct bpf_reg_state *src_reg)  in scalar_min_max_rsh()
6220 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_arsh()
6221 				  struct bpf_reg_state *src_reg)  in scalar32_min_max_arsh()
6243 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_arsh()
6244 				struct bpf_reg_state *src_reg)  in scalar_min_max_arsh()
6276 				      struct bpf_reg_state *dst_reg,  in adjust_scalar_min_max_vals()
6277 				      struct bpf_reg_state src_reg)  in adjust_scalar_min_max_vals()
6279 	struct bpf_reg_state *regs = cur_regs(env);  in adjust_scalar_min_max_vals()
6448 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;  in adjust_reg_min_max_vals()
6449 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};  in adjust_reg_min_max_vals()
6526 	struct bpf_reg_state *regs = cur_regs(env);  in check_alu_op()
6588 			struct bpf_reg_state *src_reg = regs + insn->src_reg;  in check_alu_op()
6589 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;  in check_alu_op()
6697 				     struct bpf_reg_state *dst_reg,  in __find_good_pkt_pointers()
6700 	struct bpf_reg_state *reg;  in __find_good_pkt_pointers()
6719 				   struct bpf_reg_state *dst_reg,  in find_good_pkt_pointers()
6794 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)  in is_branch32_taken()
6868 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)  in is_branch64_taken()
6947 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,  in is_branch_taken()
6980 static void reg_set_min_max(struct bpf_reg_state *true_reg,  in reg_set_min_max()
6981 			    struct bpf_reg_state *false_reg,  in reg_set_min_max()
7005 		struct bpf_reg_state *reg =  in reg_set_min_max()
7133 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,  in reg_set_min_max_inv()
7134 				struct bpf_reg_state *false_reg,  in reg_set_min_max_inv()
7163 static void __reg_combine_min_max(struct bpf_reg_state *src_reg,  in __reg_combine_min_max()
7164 				  struct bpf_reg_state *dst_reg)  in __reg_combine_min_max()
7193 static void reg_combine_min_max(struct bpf_reg_state *true_src,  in reg_combine_min_max()
7194 				struct bpf_reg_state *true_dst,  in reg_combine_min_max()
7195 				struct bpf_reg_state *false_src,  in reg_combine_min_max()
7196 				struct bpf_reg_state *false_dst,  in reg_combine_min_max()
7210 				 struct bpf_reg_state *reg, u32 id,  in mark_ptr_or_null_reg()
7278 	struct bpf_reg_state *reg;  in __mark_ptr_or_null_regs()
7298 	struct bpf_reg_state *regs = state->regs;  in mark_ptr_or_null_regs()
7315 				   struct bpf_reg_state *dst_reg,  in try_match_pkt_pointers()
7316 				   struct bpf_reg_state *src_reg,  in try_match_pkt_pointers()
7412 			       struct bpf_reg_state *known_reg)  in find_equal_scalars()
7415 	struct bpf_reg_state *reg;  in find_equal_scalars()
7440 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;  in check_cond_jmp_op()
7441 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;  in check_cond_jmp_op()
7537 		struct bpf_reg_state *src_reg = &regs[insn->src_reg];  in check_cond_jmp_op()
7612 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_imm()
7613 	struct bpf_reg_state *dst_reg;  in check_ld_imm()
7706 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_abs()
7785 	struct bpf_reg_state *reg;  in check_return_code()
8460 static bool range_within(struct bpf_reg_state *old,  in range_within()
8461 			 struct bpf_reg_state *cur)  in range_within()
8602 static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,  in regsafe()
8611 	equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;  in regsafe()
8654 		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&  in regsafe()
8667 		if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))  in regsafe()
8879 				  struct bpf_reg_state *reg,  in propagate_liveness_reg()
8880 				  struct bpf_reg_state *parent_reg)  in propagate_liveness_reg()
8915 	struct bpf_reg_state *state_reg, *parent_reg;  in propagate_liveness()
8961 	struct bpf_reg_state *state_reg;  in propagate_precision()
9008 			   offsetof(struct bpf_reg_state, parent)))  in states_maybe_looping()
9263 	struct bpf_reg_state *regs;  in do_check()
11258 	struct bpf_reg_state *regs;  in do_check_common()