408 				   struct bpf_reg_state *reg,  in verbose_invalid_scalar()
460 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)  in reg_may_point_to_spin_lock()
603 				   const struct bpf_reg_state *reg)  in func()
618 	const struct bpf_reg_state *reg;  in print_verifier_state()
1070 				struct bpf_reg_state *reg);
1073 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)  in ___mark_reg_known()
1090 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)  in __mark_reg_known()
1094 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));  in __mark_reg_known()
1098 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)  in __mark_reg32_known()
1110 static void __mark_reg_known_zero(struct bpf_reg_state *reg)  in __mark_reg_known_zero()
1115 static void __mark_reg_const_zero(struct bpf_reg_state *reg)  in __mark_reg_const_zero()
1122 				struct bpf_reg_state *regs, u32 regno)  in mark_reg_known_zero()
1134 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)  in mark_ptr_not_null_reg()
1183 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)  in reg_is_pkt_pointer()
1188 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)  in reg_is_pkt_pointer_any()
1195 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,  in reg_is_init_pkt_pointer()
1209 static void __mark_reg_unbounded(struct bpf_reg_state *reg)  in __mark_reg_unbounded()
1222 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)  in __mark_reg64_unbounded()
1230 static void __mark_reg32_unbounded(struct bpf_reg_state *reg)  in __mark_reg32_unbounded()
1238 static void __update_reg32_bounds(struct bpf_reg_state *reg)  in __update_reg32_bounds()
1253 static void __update_reg64_bounds(struct bpf_reg_state *reg)  in __update_reg64_bounds()
1266 static void __update_reg_bounds(struct bpf_reg_state *reg)  in __update_reg_bounds()
1273 static void __reg32_deduce_bounds(struct bpf_reg_state *reg)  in __reg32_deduce_bounds()
1307 static void __reg64_deduce_bounds(struct bpf_reg_state *reg)  in __reg64_deduce_bounds()
1341 static void __reg_deduce_bounds(struct bpf_reg_state *reg)  in __reg_deduce_bounds()
1348 static void __reg_bound_offset(struct bpf_reg_state *reg)  in __reg_bound_offset()
1360 static void __reg_assign_32_into_64(struct bpf_reg_state *reg)  in __reg_assign_32_into_64()
1378 static void __reg_combine_32_into_64(struct bpf_reg_state *reg)  in __reg_combine_32_into_64()
1417 static void __reg_combine_64_into_32(struct bpf_reg_state *reg)  in __reg_combine_64_into_32()
1441 			       struct bpf_reg_state *reg)  in __mark_reg_unknown()
1447 	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));  in __mark_reg_unknown()
1456 			     struct bpf_reg_state *regs, u32 regno)  in mark_reg_unknown()
1469 				struct bpf_reg_state *reg)  in __mark_reg_not_init()
1476 			      struct bpf_reg_state *regs, u32 regno)  in mark_reg_not_init()
1489 			    struct bpf_reg_state *regs, u32 regno,  in mark_btf_ld_reg()
1507 	struct bpf_reg_state *regs = state->regs;  in init_reg_state()
1890 			 const struct bpf_reg_state *state,  in mark_reg_read()
1891 			 struct bpf_reg_state *parent, u8 flag)  in mark_reg_read()
1942 		     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)  in is_reg64()
2060 			   struct bpf_reg_state *reg)  in mark_insn_zext()
2078 	struct bpf_reg_state *reg, *regs = state->regs;  in check_reg_arg()
2360 	struct bpf_reg_state *reg;  in mark_all_scalars_precise()
2393 	struct bpf_reg_state *reg;  in __mark_chain_precision()
2595 static bool register_is_null(struct bpf_reg_state *reg)  in register_is_null()
2600 static bool register_is_const(struct bpf_reg_state *reg)  in register_is_const()
2605 static bool __is_scalar_unbounded(struct bpf_reg_state *reg)  in __is_scalar_unbounded()
2614 static bool register_is_bounded(struct bpf_reg_state *reg)  in register_is_bounded()
2620 			       const struct bpf_reg_state *reg)  in __is_pointer_value()
2629 				int spi, struct bpf_reg_state *reg)  in save_register_state()
2652 	struct bpf_reg_state *reg = NULL;  in check_stack_write_fixed_off()
2776 	struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;  in check_stack_write_var_off()
2920 	struct bpf_reg_state *reg;  in check_stack_read_fixed_off()
2998 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)  in reg_state()
3020 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read_var_off()
3051 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read()
3112 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_write()
3134 	struct bpf_reg_state *regs = cur_regs(env);  in check_map_access_type()
3159 	struct bpf_reg_state *reg;  in __check_mem_access()
3196 	struct bpf_reg_state *reg = &state->regs[regno];  in check_mem_region_access()
3254 	struct bpf_reg_state *reg = &state->regs[regno];  in check_map_access()
3341 	struct bpf_reg_state *regs = cur_regs(env);  in check_packet_access()
3342 	struct bpf_reg_state *reg = &regs[regno];  in check_packet_access()
3433 	struct bpf_reg_state *regs = cur_regs(env);  in check_sock_access()
3434 	struct bpf_reg_state *reg = &regs[regno];  in check_sock_access()
3481 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_ctx_reg()
3488 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_sk_reg()
3495 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_pkt_reg()
3502 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_flow_key_reg()
3509 				   const struct bpf_reg_state *reg,  in check_pkt_ptr_alignment()
3544 				       const struct bpf_reg_state *reg,  in check_generic_ptr_alignment()
3568 			       const struct bpf_reg_state *reg, int off,  in check_ptr_alignment()
3765 		  const struct bpf_reg_state *reg, int regno)  in check_ctx_reg()
3790 				 const struct bpf_reg_state *reg,  in __check_buffer_access()
3813 				  const struct bpf_reg_state *reg,  in check_tp_buffer_access()
3829 			       const struct bpf_reg_state *reg,  in check_buffer_access()
3848 static void zext_32_to_64(struct bpf_reg_state *reg)  in zext_32_to_64()
3857 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)  in coerce_reg_to_size()
3921 				   struct bpf_reg_state *regs,  in check_ptr_to_btf_access()
3926 	struct bpf_reg_state *reg = regs + regno;  in check_ptr_to_btf_access()
3971 				   struct bpf_reg_state *regs,  in check_ptr_to_map_access()
3976 	struct bpf_reg_state *reg = regs + regno;  in check_ptr_to_map_access()
4057 	struct bpf_reg_state *regs = cur_regs(env);  in check_stack_access_within_bounds()
4058 	struct bpf_reg_state *reg = regs + regno;  in check_stack_access_within_bounds()
4121 	struct bpf_reg_state *regs = cur_regs(env);  in check_mem_access()
4122 	struct bpf_reg_state *reg = regs + regno;  in check_mem_access()
4433 	struct bpf_reg_state *reg = reg_state(env, regno);  in check_stack_range_initialized()
4559 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_helper_mem_access()
4610 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_mem_reg()
4621 		const struct bpf_reg_state saved_reg = *reg;  in check_mem_reg()
4655 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_spin_lock()
4717 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_timer_func()
4933 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_reg_type()
4992 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_func_arg()
5596 	struct bpf_reg_state *regs = state->regs, *reg;  in __clear_all_pkt_pointers()
5628 	struct bpf_reg_state *reg = &state->regs[regn];  in mark_pkt_end()
5650 	struct bpf_reg_state *regs = state->regs, *reg;  in release_reg_references()
5686 				    struct bpf_reg_state *regs)  in clear_caller_saved_regs()
5937 	struct bpf_reg_state *r0;  in prepare_func_exit()
5990 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,  in do_refine_retval_range()
5994 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];  in do_refine_retval_range()
6063 	struct bpf_reg_state *regs = cur_regs(env), *reg;  in record_func_key()
6110 				   struct bpf_reg_state *regs)  in check_bpf_snprintf_call()
6112 	struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];  in check_bpf_snprintf_call()
6113 	struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];  in check_bpf_snprintf_call()
6173 	struct bpf_reg_state *regs;  in check_helper_call()
6464 	struct bpf_reg_state *reg = &cur_regs(env)[regno];  in mark_btf_func_reg_size()
6485 	struct bpf_reg_state *regs = cur_regs(env);  in check_kfunc_call()
6591 				  const struct bpf_reg_state *reg,  in check_reg_sane_offset()
6638 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,  in retrieve_ptr_limit()
6719 	struct bpf_reg_state *regs;  in sanitize_speculative_path()
6736 			    const struct bpf_reg_state *ptr_reg,  in sanitize_ptr_alu()
6737 			    const struct bpf_reg_state *off_reg,  in sanitize_ptr_alu()
6738 			    struct bpf_reg_state *dst_reg,  in sanitize_ptr_alu()
6749 	struct bpf_reg_state tmp;  in sanitize_ptr_alu()
6845 			const struct bpf_reg_state *off_reg,  in sanitize_err()
6846 			const struct bpf_reg_state *dst_reg)  in sanitize_err()
6895 				const struct bpf_reg_state *reg,  in check_stack_access_for_ptr_arithmetic()
6918 				 const struct bpf_reg_state *dst_reg)  in sanitize_check_bounds()
6955 				   const struct bpf_reg_state *ptr_reg,  in adjust_ptr_min_max_vals()
6956 				   const struct bpf_reg_state *off_reg)  in adjust_ptr_min_max_vals()
6960 	struct bpf_reg_state *regs = state->regs, *dst_reg;  in adjust_ptr_min_max_vals()
7184 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,  in scalar32_min_max_add()
7185 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_add()
7210 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,  in scalar_min_max_add()
7211 			       struct bpf_reg_state *src_reg)  in scalar_min_max_add()
7236 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,  in scalar32_min_max_sub()
7237 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_sub()
7264 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,  in scalar_min_max_sub()
7265 			       struct bpf_reg_state *src_reg)  in scalar_min_max_sub()
7292 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,  in scalar32_min_max_mul()
7293 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_mul()
7324 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,  in scalar_min_max_mul()
7325 			       struct bpf_reg_state *src_reg)  in scalar_min_max_mul()
7356 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,  in scalar32_min_max_and()
7357 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_and()
7390 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,  in scalar_min_max_and()
7391 			       struct bpf_reg_state *src_reg)  in scalar_min_max_and()
7425 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,  in scalar32_min_max_or()
7426 				struct bpf_reg_state *src_reg)  in scalar32_min_max_or()
7459 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,  in scalar_min_max_or()
7460 			      struct bpf_reg_state *src_reg)  in scalar_min_max_or()
7494 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,  in scalar32_min_max_xor()
7495 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_xor()
7523 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,  in scalar_min_max_xor()
7524 			       struct bpf_reg_state *src_reg)  in scalar_min_max_xor()
7554 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,  in __scalar32_min_max_lsh()
7572 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_lsh()
7573 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_lsh()
7590 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,  in __scalar64_min_max_lsh()
7620 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_lsh()
7621 			       struct bpf_reg_state *src_reg)  in scalar_min_max_lsh()
7635 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_rsh()
7636 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_rsh()
7667 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_rsh()
7668 			       struct bpf_reg_state *src_reg)  in scalar_min_max_rsh()
7701 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_arsh()
7702 				  struct bpf_reg_state *src_reg)  in scalar32_min_max_arsh()
7724 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_arsh()
7725 				struct bpf_reg_state *src_reg)  in scalar_min_max_arsh()
7757 				      struct bpf_reg_state *dst_reg,  in adjust_scalar_min_max_vals()
7758 				      struct bpf_reg_state src_reg)  in adjust_scalar_min_max_vals()
7760 	struct bpf_reg_state *regs = cur_regs(env);  in adjust_scalar_min_max_vals()
7924 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;  in adjust_reg_min_max_vals()
7925 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};  in adjust_reg_min_max_vals()
8002 	struct bpf_reg_state *regs = cur_regs(env);  in check_alu_op()
8064 			struct bpf_reg_state *src_reg = regs + insn->src_reg;  in check_alu_op()
8065 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;  in check_alu_op()
8173 				     struct bpf_reg_state *dst_reg,  in __find_good_pkt_pointers()
8176 	struct bpf_reg_state *reg;  in __find_good_pkt_pointers()
8195 				   struct bpf_reg_state *dst_reg,  in find_good_pkt_pointers()
8269 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)  in is_branch32_taken()
8343 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)  in is_branch64_taken()
8422 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,  in is_branch_taken()
8471 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,  in is_pkt_ptr_branch_taken()
8472 				   struct bpf_reg_state *src_reg,  in is_pkt_ptr_branch_taken()
8475 	struct bpf_reg_state *pkt;  in is_pkt_ptr_branch_taken()
8516 static void reg_set_min_max(struct bpf_reg_state *true_reg,  in reg_set_min_max()
8517 			    struct bpf_reg_state *false_reg,  in reg_set_min_max()
8541 		struct bpf_reg_state *reg =  in reg_set_min_max()
8669 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,  in reg_set_min_max_inv()
8670 				struct bpf_reg_state *false_reg,  in reg_set_min_max_inv()
8683 static void __reg_combine_min_max(struct bpf_reg_state *src_reg,  in __reg_combine_min_max()
8684 				  struct bpf_reg_state *dst_reg)  in __reg_combine_min_max()
8713 static void reg_combine_min_max(struct bpf_reg_state *true_src,  in reg_combine_min_max()
8714 				struct bpf_reg_state *true_dst,  in reg_combine_min_max()
8715 				struct bpf_reg_state *false_src,  in reg_combine_min_max()
8716 				struct bpf_reg_state *false_dst,  in reg_combine_min_max()
8730 				 struct bpf_reg_state *reg, u32 id,  in mark_ptr_or_null_reg()
8774 	struct bpf_reg_state *reg;  in __mark_ptr_or_null_regs()
8794 	struct bpf_reg_state *regs = state->regs;  in mark_ptr_or_null_regs()
8811 				   struct bpf_reg_state *dst_reg,  in try_match_pkt_pointers()
8812 				   struct bpf_reg_state *src_reg,  in try_match_pkt_pointers()
8916 			       struct bpf_reg_state *known_reg)  in find_equal_scalars()
8919 	struct bpf_reg_state *reg;  in find_equal_scalars()
8944 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;  in check_cond_jmp_op()
8945 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;  in check_cond_jmp_op()
9060 		struct bpf_reg_state *src_reg = &regs[insn->src_reg];  in check_cond_jmp_op()
9135 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_imm()
9136 	struct bpf_reg_state *dst_reg;  in check_ld_imm()
9250 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_abs()
9329 	struct bpf_reg_state *reg;  in check_return_code()
10059 static bool range_within(struct bpf_reg_state *old,  in range_within()
10060 			 struct bpf_reg_state *cur)  in range_within()
10198 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,  in regsafe()
10199 		    struct bpf_reg_state *rcur, struct bpf_id_pair *idmap)  in regsafe()
10207 	equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;  in regsafe()
10253 		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&  in regsafe()
10266 		if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))  in regsafe()
10468 				  struct bpf_reg_state *reg,  in propagate_liveness_reg()
10469 				  struct bpf_reg_state *parent_reg)  in propagate_liveness_reg()
10504 	struct bpf_reg_state *state_reg, *parent_reg;  in propagate_liveness()
10550 	struct bpf_reg_state *state_reg;  in propagate_precision()
10597 			   offsetof(struct bpf_reg_state, parent)))  in states_maybe_looping()
10868 	struct bpf_reg_state *regs;  in do_check()
13095 	struct bpf_reg_state *regs;  in do_check_common()