Lines Matching refs:dst_reg
3093 return insn->dst_reg; in insn_def_regno()
3100 int dst_reg = insn_def_regno(insn); in insn_has_def32() local
3102 if (dst_reg == -1) in insn_has_def32()
3105 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
3410 u32 dreg = insn->dst_reg; in backtrack_insn()
3490 if (insn->dst_reg != BPF_REG_FP) in backtrack_insn()
4301 u32 dst_reg = insn->dst_reg; in check_stack_write_fixed_off() local
4342 if (dst_reg != BPF_REG_FP) { in check_stack_write_fixed_off()
6712 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
6735 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
6736 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
6737 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
6738 is_sk_reg(env, insn->dst_reg)) { in check_atomic()
6740 insn->dst_reg, in check_atomic()
6741 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
6765 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
6768 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
6775 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
11798 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
11800 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
11811 struct bpf_reg_state *dst_reg, in sanitize_ptr_alu() argument
11819 bool ptr_is_dst_reg = ptr_reg == dst_reg; in sanitize_ptr_alu()
11893 tmp = *dst_reg; in sanitize_ptr_alu()
11894 copy_register_state(dst_reg, ptr_reg); in sanitize_ptr_alu()
11899 *dst_reg = tmp; in sanitize_ptr_alu()
11919 const struct bpf_reg_state *dst_reg) in sanitize_err() argument
11923 u32 dst = insn->dst_reg, src = insn->src_reg; in sanitize_err()
11928 off_reg == dst_reg ? dst : src, err); in sanitize_err()
11932 off_reg == dst_reg ? src : dst, err); in sanitize_err()
11991 const struct bpf_reg_state *dst_reg) in sanitize_check_bounds() argument
11993 u32 dst = insn->dst_reg; in sanitize_check_bounds()
12001 switch (dst_reg->type) { in sanitize_check_bounds()
12003 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
12004 dst_reg->off + dst_reg->var_off.value)) in sanitize_check_bounds()
12008 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
12033 struct bpf_reg_state *regs = state->regs, *dst_reg; in adjust_ptr_min_max_vals() local
12041 u32 dst = insn->dst_reg; in adjust_ptr_min_max_vals()
12044 dst_reg = ®s[dst]; in adjust_ptr_min_max_vals()
12051 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
12058 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
12095 dst_reg->type = ptr_reg->type; in adjust_ptr_min_max_vals()
12096 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
12103 __mark_reg32_unbounded(dst_reg); in adjust_ptr_min_max_vals()
12106 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
12109 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
12120 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
12121 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
12122 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
12123 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
12124 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
12125 dst_reg->off = ptr_reg->off + smin_val; in adjust_ptr_min_max_vals()
12126 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
12140 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
12141 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
12143 dst_reg->smin_value = smin_ptr + smin_val; in adjust_ptr_min_max_vals()
12144 dst_reg->smax_value = smax_ptr + smax_val; in adjust_ptr_min_max_vals()
12148 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
12149 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
12151 dst_reg->umin_value = umin_ptr + umin_val; in adjust_ptr_min_max_vals()
12152 dst_reg->umax_value = umax_ptr + umax_val; in adjust_ptr_min_max_vals()
12154 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
12155 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
12156 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
12158 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
12160 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
12164 if (dst_reg == off_reg) { in adjust_ptr_min_max_vals()
12182 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
12183 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
12184 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
12185 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
12186 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
12187 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
12188 dst_reg->off = ptr_reg->off - smin_val; in adjust_ptr_min_max_vals()
12189 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
12198 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
12199 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
12201 dst_reg->smin_value = smin_ptr - smax_val; in adjust_ptr_min_max_vals()
12202 dst_reg->smax_value = smax_ptr - smin_val; in adjust_ptr_min_max_vals()
12206 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
12207 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
12210 dst_reg->umin_value = umin_ptr - umax_val; in adjust_ptr_min_max_vals()
12211 dst_reg->umax_value = umax_ptr - umin_val; in adjust_ptr_min_max_vals()
12213 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
12214 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
12215 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
12217 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
12220 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
12237 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
12239 reg_bounds_sync(dst_reg); in adjust_ptr_min_max_vals()
12240 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
12243 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
12246 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
12252 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, in scalar32_min_max_add() argument
12260 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || in scalar32_min_max_add()
12261 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { in scalar32_min_max_add()
12262 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_add()
12263 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_add()
12265 dst_reg->s32_min_value += smin_val; in scalar32_min_max_add()
12266 dst_reg->s32_max_value += smax_val; in scalar32_min_max_add()
12268 if (dst_reg->u32_min_value + umin_val < umin_val || in scalar32_min_max_add()
12269 dst_reg->u32_max_value + umax_val < umax_val) { in scalar32_min_max_add()
12270 dst_reg->u32_min_value = 0; in scalar32_min_max_add()
12271 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_add()
12273 dst_reg->u32_min_value += umin_val; in scalar32_min_max_add()
12274 dst_reg->u32_max_value += umax_val; in scalar32_min_max_add()
12278 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, in scalar_min_max_add() argument
12286 if (signed_add_overflows(dst_reg->smin_value, smin_val) || in scalar_min_max_add()
12287 signed_add_overflows(dst_reg->smax_value, smax_val)) { in scalar_min_max_add()
12288 dst_reg->smin_value = S64_MIN; in scalar_min_max_add()
12289 dst_reg->smax_value = S64_MAX; in scalar_min_max_add()
12291 dst_reg->smin_value += smin_val; in scalar_min_max_add()
12292 dst_reg->smax_value += smax_val; in scalar_min_max_add()
12294 if (dst_reg->umin_value + umin_val < umin_val || in scalar_min_max_add()
12295 dst_reg->umax_value + umax_val < umax_val) { in scalar_min_max_add()
12296 dst_reg->umin_value = 0; in scalar_min_max_add()
12297 dst_reg->umax_value = U64_MAX; in scalar_min_max_add()
12299 dst_reg->umin_value += umin_val; in scalar_min_max_add()
12300 dst_reg->umax_value += umax_val; in scalar_min_max_add()
12304 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, in scalar32_min_max_sub() argument
12312 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || in scalar32_min_max_sub()
12313 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { in scalar32_min_max_sub()
12315 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_sub()
12316 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_sub()
12318 dst_reg->s32_min_value -= smax_val; in scalar32_min_max_sub()
12319 dst_reg->s32_max_value -= smin_val; in scalar32_min_max_sub()
12321 if (dst_reg->u32_min_value < umax_val) { in scalar32_min_max_sub()
12323 dst_reg->u32_min_value = 0; in scalar32_min_max_sub()
12324 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_sub()
12327 dst_reg->u32_min_value -= umax_val; in scalar32_min_max_sub()
12328 dst_reg->u32_max_value -= umin_val; in scalar32_min_max_sub()
12332 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, in scalar_min_max_sub() argument
12340 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || in scalar_min_max_sub()
12341 signed_sub_overflows(dst_reg->smax_value, smin_val)) { in scalar_min_max_sub()
12343 dst_reg->smin_value = S64_MIN; in scalar_min_max_sub()
12344 dst_reg->smax_value = S64_MAX; in scalar_min_max_sub()
12346 dst_reg->smin_value -= smax_val; in scalar_min_max_sub()
12347 dst_reg->smax_value -= smin_val; in scalar_min_max_sub()
12349 if (dst_reg->umin_value < umax_val) { in scalar_min_max_sub()
12351 dst_reg->umin_value = 0; in scalar_min_max_sub()
12352 dst_reg->umax_value = U64_MAX; in scalar_min_max_sub()
12355 dst_reg->umin_value -= umax_val; in scalar_min_max_sub()
12356 dst_reg->umax_value -= umin_val; in scalar_min_max_sub()
12360 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, in scalar32_min_max_mul() argument
12367 if (smin_val < 0 || dst_reg->s32_min_value < 0) { in scalar32_min_max_mul()
12369 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
12375 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { in scalar32_min_max_mul()
12377 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
12380 dst_reg->u32_min_value *= umin_val; in scalar32_min_max_mul()
12381 dst_reg->u32_max_value *= umax_val; in scalar32_min_max_mul()
12382 if (dst_reg->u32_max_value > S32_MAX) { in scalar32_min_max_mul()
12384 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_mul()
12385 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_mul()
12387 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_mul()
12388 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_mul()
12392 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, in scalar_min_max_mul() argument
12399 if (smin_val < 0 || dst_reg->smin_value < 0) { in scalar_min_max_mul()
12401 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
12407 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { in scalar_min_max_mul()
12409 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
12412 dst_reg->umin_value *= umin_val; in scalar_min_max_mul()
12413 dst_reg->umax_value *= umax_val; in scalar_min_max_mul()
12414 if (dst_reg->umax_value > S64_MAX) { in scalar_min_max_mul()
12416 dst_reg->smin_value = S64_MIN; in scalar_min_max_mul()
12417 dst_reg->smax_value = S64_MAX; in scalar_min_max_mul()
12419 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_mul()
12420 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_mul()
12424 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, in scalar32_min_max_and() argument
12428 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_and()
12429 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_and()
12434 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_and()
12441 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_and()
12442 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); in scalar32_min_max_and()
12443 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_and()
12447 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_and()
12448 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_and()
12453 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_and()
12454 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_and()
12458 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, in scalar_min_max_and() argument
12462 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_and()
12467 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_and()
12474 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_and()
12475 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); in scalar_min_max_and()
12476 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_and()
12480 dst_reg->smin_value = S64_MIN; in scalar_min_max_and()
12481 dst_reg->smax_value = S64_MAX; in scalar_min_max_and()
12486 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_and()
12487 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_and()
12490 __update_reg_bounds(dst_reg); in scalar_min_max_and()
12493 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, in scalar32_min_max_or() argument
12497 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_or()
12498 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_or()
12503 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_or()
12510 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); in scalar32_min_max_or()
12511 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_or()
12512 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_or()
12516 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_or()
12517 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_or()
12522 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_or()
12523 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_or()
12527 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, in scalar_min_max_or() argument
12531 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_or()
12536 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_or()
12543 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); in scalar_min_max_or()
12544 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_or()
12545 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_or()
12549 dst_reg->smin_value = S64_MIN; in scalar_min_max_or()
12550 dst_reg->smax_value = S64_MAX; in scalar_min_max_or()
12555 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_or()
12556 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_or()
12559 __update_reg_bounds(dst_reg); in scalar_min_max_or()
12562 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, in scalar32_min_max_xor() argument
12566 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_xor()
12567 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_xor()
12571 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_xor()
12576 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_xor()
12577 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_xor()
12579 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { in scalar32_min_max_xor()
12583 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_xor()
12584 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_xor()
12586 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_xor()
12587 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_xor()
12591 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, in scalar_min_max_xor() argument
12595 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_xor()
12600 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_xor()
12605 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_xor()
12606 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_xor()
12608 if (dst_reg->smin_value >= 0 && smin_val >= 0) { in scalar_min_max_xor()
12612 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_xor()
12613 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_xor()
12615 dst_reg->smin_value = S64_MIN; in scalar_min_max_xor()
12616 dst_reg->smax_value = S64_MAX; in scalar_min_max_xor()
12619 __update_reg_bounds(dst_reg); in scalar_min_max_xor()
12622 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar32_min_max_lsh() argument
12628 dst_reg->s32_min_value = S32_MIN; in __scalar32_min_max_lsh()
12629 dst_reg->s32_max_value = S32_MAX; in __scalar32_min_max_lsh()
12631 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { in __scalar32_min_max_lsh()
12632 dst_reg->u32_min_value = 0; in __scalar32_min_max_lsh()
12633 dst_reg->u32_max_value = U32_MAX; in __scalar32_min_max_lsh()
12635 dst_reg->u32_min_value <<= umin_val; in __scalar32_min_max_lsh()
12636 dst_reg->u32_max_value <<= umax_val; in __scalar32_min_max_lsh()
12640 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_lsh() argument
12646 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_lsh()
12648 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar32_min_max_lsh()
12649 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); in scalar32_min_max_lsh()
12654 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_lsh()
12655 __update_reg32_bounds(dst_reg); in scalar32_min_max_lsh()
12658 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar64_min_max_lsh() argument
12668 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) in __scalar64_min_max_lsh()
12669 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; in __scalar64_min_max_lsh()
12671 dst_reg->smax_value = S64_MAX; in __scalar64_min_max_lsh()
12673 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) in __scalar64_min_max_lsh()
12674 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; in __scalar64_min_max_lsh()
12676 dst_reg->smin_value = S64_MIN; in __scalar64_min_max_lsh()
12679 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { in __scalar64_min_max_lsh()
12680 dst_reg->umin_value = 0; in __scalar64_min_max_lsh()
12681 dst_reg->umax_value = U64_MAX; in __scalar64_min_max_lsh()
12683 dst_reg->umin_value <<= umin_val; in __scalar64_min_max_lsh()
12684 dst_reg->umax_value <<= umax_val; in __scalar64_min_max_lsh()
12688 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar_min_max_lsh() argument
12695 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
12696 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
12698 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); in scalar_min_max_lsh()
12700 __update_reg_bounds(dst_reg); in scalar_min_max_lsh()
12703 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_rsh() argument
12706 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_rsh()
12724 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_rsh()
12725 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_rsh()
12727 dst_reg->var_off = tnum_rshift(subreg, umin_val); in scalar32_min_max_rsh()
12728 dst_reg->u32_min_value >>= umax_val; in scalar32_min_max_rsh()
12729 dst_reg->u32_max_value >>= umin_val; in scalar32_min_max_rsh()
12731 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_rsh()
12732 __update_reg32_bounds(dst_reg); in scalar32_min_max_rsh()
12735 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar_min_max_rsh() argument
12755 dst_reg->smin_value = S64_MIN; in scalar_min_max_rsh()
12756 dst_reg->smax_value = S64_MAX; in scalar_min_max_rsh()
12757 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); in scalar_min_max_rsh()
12758 dst_reg->umin_value >>= umax_val; in scalar_min_max_rsh()
12759 dst_reg->umax_value >>= umin_val; in scalar_min_max_rsh()
12765 __mark_reg32_unbounded(dst_reg); in scalar_min_max_rsh()
12766 __update_reg_bounds(dst_reg); in scalar_min_max_rsh()
12769 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_arsh() argument
12777 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); in scalar32_min_max_arsh()
12778 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); in scalar32_min_max_arsh()
12780 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); in scalar32_min_max_arsh()
12785 dst_reg->u32_min_value = 0; in scalar32_min_max_arsh()
12786 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_arsh()
12788 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_arsh()
12789 __update_reg32_bounds(dst_reg); in scalar32_min_max_arsh()
12792 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar_min_max_arsh() argument
12800 dst_reg->smin_value >>= umin_val; in scalar_min_max_arsh()
12801 dst_reg->smax_value >>= umin_val; in scalar_min_max_arsh()
12803 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); in scalar_min_max_arsh()
12808 dst_reg->umin_value = 0; in scalar_min_max_arsh()
12809 dst_reg->umax_value = U64_MAX; in scalar_min_max_arsh()
12815 __mark_reg32_unbounded(dst_reg); in scalar_min_max_arsh()
12816 __update_reg_bounds(dst_reg); in scalar_min_max_arsh()
12825 struct bpf_reg_state *dst_reg, in adjust_scalar_min_max_vals() argument
12857 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
12868 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
12875 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
12901 scalar32_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12902 scalar_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12903 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
12906 scalar32_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12907 scalar_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12908 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
12911 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
12912 scalar32_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12913 scalar_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12916 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
12917 scalar32_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12918 scalar_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12921 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
12922 scalar32_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12923 scalar_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12926 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
12927 scalar32_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12928 scalar_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12935 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12939 scalar32_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12941 scalar_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12948 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12952 scalar32_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12954 scalar_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12961 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12965 scalar32_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12967 scalar_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
12970 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
12976 zext_32_to_64(dst_reg); in adjust_scalar_min_max_vals()
12977 reg_bounds_sync(dst_reg); in adjust_scalar_min_max_vals()
12989 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; in adjust_reg_min_max_vals() local
12994 dst_reg = ®s[insn->dst_reg]; in adjust_reg_min_max_vals()
12996 if (dst_reg->type != SCALAR_VALUE) in adjust_reg_min_max_vals()
12997 ptr_reg = dst_reg; in adjust_reg_min_max_vals()
13002 dst_reg->id = 0; in adjust_reg_min_max_vals()
13006 if (dst_reg->type != SCALAR_VALUE) { in adjust_reg_min_max_vals()
13012 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
13016 insn->dst_reg, in adjust_reg_min_max_vals()
13024 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
13028 src_reg, dst_reg); in adjust_reg_min_max_vals()
13036 dst_reg, src_reg); in adjust_reg_min_max_vals()
13037 } else if (dst_reg->precise) { in adjust_reg_min_max_vals()
13066 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
13095 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
13099 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
13101 insn->dst_reg); in check_alu_op()
13106 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
13143 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
13149 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; in check_alu_op() local
13164 copy_register_state(dst_reg, src_reg); in check_alu_op()
13165 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
13166 dst_reg->subreg_def = DEF_NOT_SUBREG; in check_alu_op()
13180 copy_register_state(dst_reg, src_reg); in check_alu_op()
13182 dst_reg->id = 0; in check_alu_op()
13183 coerce_reg_to_size_sx(dst_reg, insn->off >> 3); in check_alu_op()
13184 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
13185 dst_reg->subreg_def = DEF_NOT_SUBREG; in check_alu_op()
13187 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
13203 copy_register_state(dst_reg, src_reg); in check_alu_op()
13209 dst_reg->id = 0; in check_alu_op()
13210 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
13211 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
13218 copy_register_state(dst_reg, src_reg); in check_alu_op()
13220 dst_reg->id = 0; in check_alu_op()
13221 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
13222 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
13223 coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); in check_alu_op()
13227 insn->dst_reg); in check_alu_op()
13229 zext_32_to_64(dst_reg); in check_alu_op()
13230 reg_bounds_sync(dst_reg); in check_alu_op()
13237 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
13238 regs[insn->dst_reg].type = SCALAR_VALUE; in check_alu_op()
13240 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
13243 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
13273 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
13294 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
13305 struct bpf_reg_state *dst_reg, in find_good_pkt_pointers() argument
13313 if (dst_reg->off < 0 || in find_good_pkt_pointers()
13314 (dst_reg->off == 0 && range_right_open)) in find_good_pkt_pointers()
13318 if (dst_reg->umax_value > MAX_PACKET_OFF || in find_good_pkt_pointers()
13319 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) in find_good_pkt_pointers()
13325 new_range = dst_reg->off; in find_good_pkt_pointers()
13377 if (reg->type == type && reg->id == dst_reg->id) in find_good_pkt_pointers()
13593 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, in is_pkt_ptr_branch_taken() argument
13600 pkt = dst_reg; in is_pkt_ptr_branch_taken()
13601 } else if (dst_reg->type == PTR_TO_PACKET_END) { in is_pkt_ptr_branch_taken()
13813 struct bpf_reg_state *dst_reg) in __reg_combine_min_max() argument
13815 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, in __reg_combine_min_max()
13816 dst_reg->umin_value); in __reg_combine_min_max()
13817 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, in __reg_combine_min_max()
13818 dst_reg->umax_value); in __reg_combine_min_max()
13819 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, in __reg_combine_min_max()
13820 dst_reg->smin_value); in __reg_combine_min_max()
13821 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, in __reg_combine_min_max()
13822 dst_reg->smax_value); in __reg_combine_min_max()
13823 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, in __reg_combine_min_max()
13824 dst_reg->var_off); in __reg_combine_min_max()
13826 reg_bounds_sync(dst_reg); in __reg_combine_min_max()
13916 struct bpf_reg_state *dst_reg, in try_match_pkt_pointers() argument
13930 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
13932 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
13935 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
13936 dst_reg->type, false); in try_match_pkt_pointers()
13937 mark_pkt_end(other_branch, insn->dst_reg, true); in try_match_pkt_pointers()
13938 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
13940 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
13951 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
13953 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
13956 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
13957 dst_reg->type, true); in try_match_pkt_pointers()
13958 mark_pkt_end(this_branch, insn->dst_reg, false); in try_match_pkt_pointers()
13959 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
13961 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
13972 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
13974 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
13977 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
13978 dst_reg->type, true); in try_match_pkt_pointers()
13979 mark_pkt_end(other_branch, insn->dst_reg, false); in try_match_pkt_pointers()
13980 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
13982 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
13993 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
13995 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
13998 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
13999 dst_reg->type, false); in try_match_pkt_pointers()
14000 mark_pkt_end(this_branch, insn->dst_reg, true); in try_match_pkt_pointers()
14001 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
14003 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
14038 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; in check_cond_jmp_op() local
14052 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
14056 dst_reg = ®s[insn->dst_reg]; in check_cond_jmp_op()
14069 if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) && in check_cond_jmp_op()
14085 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); in check_cond_jmp_op()
14088 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
14094 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
14098 } else if (dst_reg->type == SCALAR_VALUE && in check_cond_jmp_op()
14099 is_jmp32 && tnum_is_const(tnum_subreg(dst_reg->var_off))) { in check_cond_jmp_op()
14101 tnum_subreg(dst_reg->var_off).value, in check_cond_jmp_op()
14104 } else if (dst_reg->type == SCALAR_VALUE && in check_cond_jmp_op()
14105 !is_jmp32 && tnum_is_const(dst_reg->var_off)) { in check_cond_jmp_op()
14107 dst_reg->var_off.value, in check_cond_jmp_op()
14110 } else if (reg_is_pkt_pointer_any(dst_reg) && in check_cond_jmp_op()
14113 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); in check_cond_jmp_op()
14120 if (!__is_pointer_value(false, dst_reg)) in check_cond_jmp_op()
14121 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
14169 if (dst_reg->type == SCALAR_VALUE && in check_cond_jmp_op()
14174 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
14175 dst_reg, in check_cond_jmp_op()
14179 else if (tnum_is_const(dst_reg->var_off) || in check_cond_jmp_op()
14181 tnum_is_const(tnum_subreg(dst_reg->var_off)))) in check_cond_jmp_op()
14184 dst_reg->var_off.value, in check_cond_jmp_op()
14185 tnum_subreg(dst_reg->var_off).value, in check_cond_jmp_op()
14191 &other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
14192 src_reg, dst_reg, opcode); in check_cond_jmp_op()
14200 } else if (dst_reg->type == SCALAR_VALUE) { in check_cond_jmp_op()
14201 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
14202 dst_reg, insn->imm, (u32)insn->imm, in check_cond_jmp_op()
14206 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && in check_cond_jmp_op()
14207 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { in check_cond_jmp_op()
14208 find_equal_scalars(this_branch, dst_reg); in check_cond_jmp_op()
14209 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); in check_cond_jmp_op()
14225 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && in check_cond_jmp_op()
14226 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && in check_cond_jmp_op()
14228 base_type(dst_reg->type) != PTR_TO_BTF_ID) { in check_cond_jmp_op()
14245 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); in check_cond_jmp_op()
14255 type_may_be_null(dst_reg->type)) { in check_cond_jmp_op()
14259 mark_ptr_or_null_regs(this_branch, insn->dst_reg, in check_cond_jmp_op()
14261 mark_ptr_or_null_regs(other_branch, insn->dst_reg, in check_cond_jmp_op()
14263 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], in check_cond_jmp_op()
14265 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
14267 insn->dst_reg); in check_cond_jmp_op()
14280 struct bpf_reg_state *dst_reg; in check_ld_imm() local
14293 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
14297 dst_reg = ®s[insn->dst_reg]; in check_ld_imm()
14301 dst_reg->type = SCALAR_VALUE; in check_ld_imm()
14302 __mark_reg_known(®s[insn->dst_reg], imm); in check_ld_imm()
14310 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
14313 dst_reg->type = aux->btf_var.reg_type; in check_ld_imm()
14314 switch (base_type(dst_reg->type)) { in check_ld_imm()
14316 dst_reg->mem_size = aux->btf_var.mem_size; in check_ld_imm()
14319 dst_reg->btf = aux->btf_var.btf; in check_ld_imm()
14320 dst_reg->btf_id = aux->btf_var.btf_id; in check_ld_imm()
14343 dst_reg->type = PTR_TO_FUNC; in check_ld_imm()
14344 dst_reg->subprogno = subprogno; in check_ld_imm()
14349 dst_reg->map_ptr = map; in check_ld_imm()
14353 dst_reg->type = PTR_TO_MAP_VALUE; in check_ld_imm()
14354 dst_reg->off = aux->map_off; in check_ld_imm()
14359 dst_reg->type = CONST_PTR_TO_MAP; in check_ld_imm()
14412 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || in check_ld_abs()
16548 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
16559 BPF_READ, insn->dst_reg, false, in do_check()
16588 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
16592 dst_reg_type = regs[insn->dst_reg].type; in do_check()
16595 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
16613 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
16617 dst_reg_type = regs[insn->dst_reg].type; in do_check()
16620 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
16640 insn->dst_reg != BPF_REG_0 || in do_check()
16668 insn->dst_reg != BPF_REG_0 || in do_check()
16685 insn->dst_reg != BPF_REG_0 || in do_check()
17063 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || in resolve_pseudo_ldimm64()
17655 rnd_hi32_patch[3].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
17684 zext_patch[1].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
17873 insn->dst_reg, in convert_ctx_accesses()
17875 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
17880 insn->dst_reg, in convert_ctx_accesses()
17882 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
17888 insn->dst_reg, insn->dst_reg, in convert_ctx_accesses()
18354 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), in do_misc_fixups()
18365 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), in do_misc_fixups()
18421 off_reg = issrc ? insn->src_reg : insn->dst_reg; in do_misc_fixups()
18435 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); in do_misc_fixups()