413 				   struct bpf_reg_state *reg,  in verbose_invalid_scalar()
454 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)  in reg_may_point_to_spin_lock()
633 				   const struct bpf_reg_state *reg)  in func()
700 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in mark_stack_slots_dynptr()
738 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in unmark_stack_slots_dynptr()
767 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in is_dynptr_reg_valid_uninit()
786 			      struct bpf_reg_state *reg)  in is_dynptr_reg_valid_init()
806 			     struct bpf_reg_state *reg,  in is_dynptr_type_expected()
840 	const struct bpf_reg_state *reg;  in print_verifier_state()
1328 				struct bpf_reg_state *reg);
1331 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)  in ___mark_reg_known()
1348 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)  in __mark_reg_known()
1352 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));  in __mark_reg_known()
1356 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)  in __mark_reg32_known()
1368 static void __mark_reg_known_zero(struct bpf_reg_state *reg)  in __mark_reg_known_zero()
1373 static void __mark_reg_const_zero(struct bpf_reg_state *reg)  in __mark_reg_const_zero()
1380 				struct bpf_reg_state *regs, u32 regno)  in mark_reg_known_zero()
1392 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)  in mark_ptr_not_null_reg()
1419 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)  in reg_is_pkt_pointer()
1424 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)  in reg_is_pkt_pointer_any()
1431 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,  in reg_is_init_pkt_pointer()
1445 static void __mark_reg_unbounded(struct bpf_reg_state *reg)  in __mark_reg_unbounded()
1458 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)  in __mark_reg64_unbounded()
1466 static void __mark_reg32_unbounded(struct bpf_reg_state *reg)  in __mark_reg32_unbounded()
1474 static void __update_reg32_bounds(struct bpf_reg_state *reg)  in __update_reg32_bounds()
1489 static void __update_reg64_bounds(struct bpf_reg_state *reg)  in __update_reg64_bounds()
1502 static void __update_reg_bounds(struct bpf_reg_state *reg)  in __update_reg_bounds()
1509 static void __reg32_deduce_bounds(struct bpf_reg_state *reg)  in __reg32_deduce_bounds()
1543 static void __reg64_deduce_bounds(struct bpf_reg_state *reg)  in __reg64_deduce_bounds()
1577 static void __reg_deduce_bounds(struct bpf_reg_state *reg)  in __reg_deduce_bounds()
1584 static void __reg_bound_offset(struct bpf_reg_state *reg)  in __reg_bound_offset()
1596 static void reg_bounds_sync(struct bpf_reg_state *reg)  in reg_bounds_sync()
1616 static void __reg_assign_32_into_64(struct bpf_reg_state *reg)  in __reg_assign_32_into_64()
1635 static void __reg_combine_32_into_64(struct bpf_reg_state *reg)  in __reg_combine_32_into_64()
1666 static void __reg_combine_64_into_32(struct bpf_reg_state *reg)  in __reg_combine_64_into_32()
1682 			       struct bpf_reg_state *reg)  in __mark_reg_unknown()
1688 	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));  in __mark_reg_unknown()
1697 			     struct bpf_reg_state *regs, u32 regno)  in mark_reg_unknown()
1710 				struct bpf_reg_state *reg)  in __mark_reg_not_init()
1717 			      struct bpf_reg_state *regs, u32 regno)  in mark_reg_not_init()
1730 			    struct bpf_reg_state *regs, u32 regno,  in mark_btf_ld_reg()
1749 	struct bpf_reg_state *regs = state->regs;  in init_reg_state()
2272 			 const struct bpf_reg_state *state,  in mark_reg_read()
2273 			 struct bpf_reg_state *parent, u8 flag)  in mark_reg_read()
2324 		     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)  in is_reg64()
2442 			   struct bpf_reg_state *reg)  in mark_insn_zext()
2460 	struct bpf_reg_state *reg, *regs = state->regs;  in check_reg_arg()
2745 	struct bpf_reg_state *reg;  in mark_all_scalars_precise()
2778 	struct bpf_reg_state *reg;  in __mark_chain_precision()
2970 static bool register_is_null(struct bpf_reg_state *reg)  in register_is_null()
2975 static bool register_is_const(struct bpf_reg_state *reg)  in register_is_const()
2980 static bool __is_scalar_unbounded(struct bpf_reg_state *reg)  in __is_scalar_unbounded()
2989 static bool register_is_bounded(struct bpf_reg_state *reg)  in register_is_bounded()
2995 			       const struct bpf_reg_state *reg)  in __is_pointer_value()
3004 				int spi, struct bpf_reg_state *reg,  in save_register_state()
3033 	struct bpf_reg_state *reg = NULL;  in check_stack_write_fixed_off()
3158 	struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;  in check_stack_write_var_off()
3303 	struct bpf_reg_state *reg;  in check_stack_read_fixed_off()
3400 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)  in reg_state()
3422 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read_var_off()
3453 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read()
3514 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_write()
3536 	struct bpf_reg_state *regs = cur_regs(env);  in check_map_access_type()
3561 	struct bpf_reg_state *reg;  in __check_mem_access()
3598 	struct bpf_reg_state *reg = &state->regs[regno];  in check_mem_region_access()
3648 			       const struct bpf_reg_state *reg, int regno,  in __check_ptr_off_reg()
3680 		      const struct bpf_reg_state *reg, int regno)  in check_ptr_off_reg()
3687 			       struct bpf_reg_state *reg, u32 regno)  in map_kptr_match_type()
3763 	struct bpf_reg_state *val_reg;  in check_map_kptr_access()
3819 	struct bpf_reg_state *reg = &state->regs[regno];  in check_map_access()
3931 	struct bpf_reg_state *regs = cur_regs(env);  in check_packet_access()
3932 	struct bpf_reg_state *reg = &regs[regno];  in check_packet_access()
4023 	struct bpf_reg_state *regs = cur_regs(env);  in check_sock_access()
4024 	struct bpf_reg_state *reg = &regs[regno];  in check_sock_access()
4071 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_ctx_reg()
4078 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_sk_reg()
4085 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_pkt_reg()
4092 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_flow_key_reg()
4099 				   const struct bpf_reg_state *reg,  in check_pkt_ptr_alignment()
4134 				       const struct bpf_reg_state *reg,  in check_generic_ptr_alignment()
4158 			       const struct bpf_reg_state *reg, int off,  in check_ptr_alignment()
4356 				 const struct bpf_reg_state *reg,  in __check_buffer_access()
4379 				  const struct bpf_reg_state *reg,  in check_tp_buffer_access()
4395 			       const struct bpf_reg_state *reg,  in check_buffer_access()
4414 static void zext_32_to_64(struct bpf_reg_state *reg)  in zext_32_to_64()
4423 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)  in coerce_reg_to_size()
4502 				   struct bpf_reg_state *regs,  in check_ptr_to_btf_access()
4507 	struct bpf_reg_state *reg = regs + regno;  in check_ptr_to_btf_access()
4573 				   struct bpf_reg_state *regs,  in check_ptr_to_map_access()
4578 	struct bpf_reg_state *reg = regs + regno;  in check_ptr_to_map_access()
4660 	struct bpf_reg_state *regs = cur_regs(env);  in check_stack_access_within_bounds()
4661 	struct bpf_reg_state *reg = regs + regno;  in check_stack_access_within_bounds()
4724 	struct bpf_reg_state *regs = cur_regs(env);  in check_mem_access()
4725 	struct bpf_reg_state *reg = regs + regno;  in check_mem_access()
5078 	struct bpf_reg_state *reg = reg_state(env, regno);  in check_stack_range_initialized()
5204 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_helper_mem_access()
5291 			      struct bpf_reg_state *reg, u32 regno,  in check_mem_size_reg()
5345 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_mem_reg()
5349 	struct bpf_reg_state saved_reg;  in check_mem_reg()
5377 int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_kfunc_mem_size_reg()
5380 	struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];  in check_kfunc_mem_size_reg()
5382 	struct bpf_reg_state saved_reg;  in check_kfunc_mem_size_reg()
5427 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_spin_lock()
5489 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_timer_func()
5537 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_kptr_func()
5750 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_reg_type()
5834 			   const struct bpf_reg_state *reg, int regno,  in check_func_arg_reg_off()
5889 static u32 stack_slot_get_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in stack_slot_get_id()
5902 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_func_arg()
6577 	struct bpf_reg_state *reg;  in clear_all_pkt_pointers()
6593 	struct bpf_reg_state *reg = &state->regs[regn];  in mark_pkt_end()
6618 	struct bpf_reg_state *reg;  in release_reference()
6638 				    struct bpf_reg_state *regs)  in clear_caller_saved_regs()
6971 	struct bpf_reg_state *r0;  in prepare_func_exit()
7030 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,  in do_refine_retval_range()
7034 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];  in do_refine_retval_range()
7102 	struct bpf_reg_state *regs = cur_regs(env), *reg;  in record_func_key()
7154 				   struct bpf_reg_state *regs)  in check_bpf_snprintf_call()
7156 	struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];  in check_bpf_snprintf_call()
7157 	struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];  in check_bpf_snprintf_call()
7218 	struct bpf_reg_state *regs = cur_regs(env);  in loop_flag_is_zero()
7219 	struct bpf_reg_state *reg = &regs[BPF_REG_4];  in loop_flag_is_zero()
7253 	struct bpf_reg_state *regs;  in check_helper_call()
7425 				struct bpf_reg_state *reg = &regs[BPF_REG_1 + i];  in check_helper_call()
7652 	struct bpf_reg_state *reg = &cur_regs(env)[regno];  in mark_btf_func_reg_size()
7674 	struct bpf_reg_state *regs = cur_regs(env);  in check_kfunc_call()
7846 				  const struct bpf_reg_state *reg,  in check_reg_sane_offset()
7888 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,  in retrieve_ptr_limit()
7969 	struct bpf_reg_state *regs;  in sanitize_speculative_path()
7986 			    const struct bpf_reg_state *ptr_reg,  in sanitize_ptr_alu()
7987 			    const struct bpf_reg_state *off_reg,  in sanitize_ptr_alu()
7988 			    struct bpf_reg_state *dst_reg,  in sanitize_ptr_alu()
7999 	struct bpf_reg_state tmp;  in sanitize_ptr_alu()
8095 			const struct bpf_reg_state *off_reg,  in sanitize_err()
8096 			const struct bpf_reg_state *dst_reg)  in sanitize_err()
8145 				const struct bpf_reg_state *reg,  in check_stack_access_for_ptr_arithmetic()
8168 				 const struct bpf_reg_state *dst_reg)  in sanitize_check_bounds()
8205 				   const struct bpf_reg_state *ptr_reg,  in adjust_ptr_min_max_vals()
8206 				   const struct bpf_reg_state *off_reg)  in adjust_ptr_min_max_vals()
8210 	struct bpf_reg_state *regs = state->regs, *dst_reg;  in adjust_ptr_min_max_vals()
8429 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,  in scalar32_min_max_add()
8430 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_add()
8455 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,  in scalar_min_max_add()
8456 			       struct bpf_reg_state *src_reg)  in scalar_min_max_add()
8481 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,  in scalar32_min_max_sub()
8482 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_sub()
8509 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,  in scalar_min_max_sub()
8510 			       struct bpf_reg_state *src_reg)  in scalar_min_max_sub()
8537 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,  in scalar32_min_max_mul()
8538 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_mul()
8569 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,  in scalar_min_max_mul()
8570 			       struct bpf_reg_state *src_reg)  in scalar_min_max_mul()
8601 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,  in scalar32_min_max_and()
8602 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_and()
8635 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,  in scalar_min_max_and()
8636 			       struct bpf_reg_state *src_reg)  in scalar_min_max_and()
8670 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,  in scalar32_min_max_or()
8671 				struct bpf_reg_state *src_reg)  in scalar32_min_max_or()
8704 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,  in scalar_min_max_or()
8705 			      struct bpf_reg_state *src_reg)  in scalar_min_max_or()
8739 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,  in scalar32_min_max_xor()
8740 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_xor()
8768 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,  in scalar_min_max_xor()
8769 			       struct bpf_reg_state *src_reg)  in scalar_min_max_xor()
8799 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,  in __scalar32_min_max_lsh()
8817 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_lsh()
8818 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_lsh()
8835 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,  in __scalar64_min_max_lsh()
8865 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_lsh()
8866 			       struct bpf_reg_state *src_reg)  in scalar_min_max_lsh()
8880 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_rsh()
8881 				 struct bpf_reg_state *src_reg)  in scalar32_min_max_rsh()
8912 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_rsh()
8913 			       struct bpf_reg_state *src_reg)  in scalar_min_max_rsh()
8946 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,  in scalar32_min_max_arsh()
8947 				  struct bpf_reg_state *src_reg)  in scalar32_min_max_arsh()
8969 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,  in scalar_min_max_arsh()
8970 				struct bpf_reg_state *src_reg)  in scalar_min_max_arsh()
9002 				      struct bpf_reg_state *dst_reg,  in adjust_scalar_min_max_vals()
9003 				      struct bpf_reg_state src_reg)  in adjust_scalar_min_max_vals()
9005 	struct bpf_reg_state *regs = cur_regs(env);  in adjust_scalar_min_max_vals()
9166 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;  in adjust_reg_min_max_vals()
9167 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};  in adjust_reg_min_max_vals()
9244 	struct bpf_reg_state *regs = cur_regs(env);  in check_alu_op()
9306 			struct bpf_reg_state *src_reg = regs + insn->src_reg;  in check_alu_op()
9307 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;  in check_alu_op()
9416 				   struct bpf_reg_state *dst_reg,  in find_good_pkt_pointers()
9421 	struct bpf_reg_state *reg;  in find_good_pkt_pointers()
9494 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)  in is_branch32_taken()
9568 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)  in is_branch64_taken()
9647 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,  in is_branch_taken()
9696 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,  in is_pkt_ptr_branch_taken()
9697 				   struct bpf_reg_state *src_reg,  in is_pkt_ptr_branch_taken()
9700 	struct bpf_reg_state *pkt;  in is_pkt_ptr_branch_taken()
9741 static void reg_set_min_max(struct bpf_reg_state *true_reg,  in reg_set_min_max()
9742 			    struct bpf_reg_state *false_reg,  in reg_set_min_max()
9901 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,  in reg_set_min_max_inv()
9902 				struct bpf_reg_state *false_reg,  in reg_set_min_max_inv()
9915 static void __reg_combine_min_max(struct bpf_reg_state *src_reg,  in __reg_combine_min_max()
9916 				  struct bpf_reg_state *dst_reg)  in __reg_combine_min_max()
9932 static void reg_combine_min_max(struct bpf_reg_state *true_src,  in reg_combine_min_max()
9933 				struct bpf_reg_state *true_dst,  in reg_combine_min_max()
9934 				struct bpf_reg_state *false_src,  in reg_combine_min_max()
9935 				struct bpf_reg_state *false_dst,  in reg_combine_min_max()
9949 				 struct bpf_reg_state *reg, u32 id,  in mark_ptr_or_null_reg()
9997 	struct bpf_reg_state *regs = state->regs, *reg;  in mark_ptr_or_null_regs()
10014 				   struct bpf_reg_state *dst_reg,  in try_match_pkt_pointers()
10015 				   struct bpf_reg_state *src_reg,  in try_match_pkt_pointers()
10119 			       struct bpf_reg_state *known_reg)  in find_equal_scalars()
10122 	struct bpf_reg_state *reg;  in find_equal_scalars()
10135 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;  in check_cond_jmp_op()
10136 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;  in check_cond_jmp_op()
10251 		struct bpf_reg_state *src_reg = &regs[insn->src_reg];  in check_cond_jmp_op()
10326 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_imm()
10327 	struct bpf_reg_state *dst_reg;  in check_ld_imm()
10444 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_abs()
10523 	struct bpf_reg_state *reg;  in check_return_code()
11358 static bool range_within(struct bpf_reg_state *old,  in range_within()
11359 			 struct bpf_reg_state *cur)  in range_within()
11497 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,  in regsafe()
11498 		    struct bpf_reg_state *rcur, struct bpf_id_pair *idmap)  in regsafe()
11506 	equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;  in regsafe()
11554 			if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))  in regsafe()
11568 		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&  in regsafe()
11766 				  struct bpf_reg_state *reg,  in propagate_liveness_reg()
11767 				  struct bpf_reg_state *parent_reg)  in propagate_liveness_reg()
11802 	struct bpf_reg_state *state_reg, *parent_reg;  in propagate_liveness()
11848 	struct bpf_reg_state *state_reg;  in propagate_precision()
11895 			   offsetof(struct bpf_reg_state, parent)))  in states_maybe_looping()
12162 	struct bpf_reg_state *regs;  in do_check()
14597 	struct bpf_reg_state *regs;  in do_check_common()