Lines Matching refs:dst_reg
2426 return insn->dst_reg; in insn_def_regno()
2433 int dst_reg = insn_def_regno(insn); in insn_has_def32() local
2435 if (dst_reg == -1) in insn_has_def32()
2438 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
2568 u32 dreg = 1u << insn->dst_reg; in backtrack_insn()
2644 if (insn->dst_reg != BPF_REG_FP) in backtrack_insn()
3032 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; in check_stack_write_fixed_off() local
3068 if (dst_reg != BPF_REG_FP) { in check_stack_write_fixed_off()
4993 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
5016 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
5017 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
5018 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
5019 is_sk_reg(env, insn->dst_reg)) { in check_atomic()
5021 insn->dst_reg, in check_atomic()
5022 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
5046 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5049 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5056 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
7975 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
7977 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
7988 struct bpf_reg_state *dst_reg, in sanitize_ptr_alu() argument
7996 bool ptr_is_dst_reg = ptr_reg == dst_reg; in sanitize_ptr_alu()
8070 tmp = *dst_reg; in sanitize_ptr_alu()
8071 *dst_reg = *ptr_reg; in sanitize_ptr_alu()
8076 *dst_reg = tmp; in sanitize_ptr_alu()
8096 const struct bpf_reg_state *dst_reg) in sanitize_err() argument
8100 u32 dst = insn->dst_reg, src = insn->src_reg; in sanitize_err()
8105 off_reg == dst_reg ? dst : src, err); in sanitize_err()
8109 off_reg == dst_reg ? src : dst, err); in sanitize_err()
8168 const struct bpf_reg_state *dst_reg) in sanitize_check_bounds() argument
8170 u32 dst = insn->dst_reg; in sanitize_check_bounds()
8178 switch (dst_reg->type) { in sanitize_check_bounds()
8180 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
8181 dst_reg->off + dst_reg->var_off.value)) in sanitize_check_bounds()
8185 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
8210 struct bpf_reg_state *regs = state->regs, *dst_reg; in adjust_ptr_min_max_vals() local
8218 u32 dst = insn->dst_reg; in adjust_ptr_min_max_vals()
8221 dst_reg = ®s[dst]; in adjust_ptr_min_max_vals()
8228 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8235 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8272 dst_reg->type = ptr_reg->type; in adjust_ptr_min_max_vals()
8273 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
8280 __mark_reg32_unbounded(dst_reg); in adjust_ptr_min_max_vals()
8283 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8286 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
8297 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
8298 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
8299 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
8300 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
8301 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
8302 dst_reg->off = ptr_reg->off + smin_val; in adjust_ptr_min_max_vals()
8303 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8317 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
8318 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
8320 dst_reg->smin_value = smin_ptr + smin_val; in adjust_ptr_min_max_vals()
8321 dst_reg->smax_value = smax_ptr + smax_val; in adjust_ptr_min_max_vals()
8325 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
8326 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
8328 dst_reg->umin_value = umin_ptr + umin_val; in adjust_ptr_min_max_vals()
8329 dst_reg->umax_value = umax_ptr + umax_val; in adjust_ptr_min_max_vals()
8331 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
8332 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
8333 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8335 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8337 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
8341 if (dst_reg == off_reg) { in adjust_ptr_min_max_vals()
8359 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
8360 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
8361 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
8362 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
8363 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
8364 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
8365 dst_reg->off = ptr_reg->off - smin_val; in adjust_ptr_min_max_vals()
8366 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8375 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
8376 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
8378 dst_reg->smin_value = smin_ptr - smax_val; in adjust_ptr_min_max_vals()
8379 dst_reg->smax_value = smax_ptr - smin_val; in adjust_ptr_min_max_vals()
8383 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
8384 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
8387 dst_reg->umin_value = umin_ptr - umax_val; in adjust_ptr_min_max_vals()
8388 dst_reg->umax_value = umax_ptr - umin_val; in adjust_ptr_min_max_vals()
8390 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
8391 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
8392 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8394 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8397 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
8414 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
8416 reg_bounds_sync(dst_reg); in adjust_ptr_min_max_vals()
8417 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
8420 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8423 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
8429 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, in scalar32_min_max_add() argument
8437 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || in scalar32_min_max_add()
8438 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { in scalar32_min_max_add()
8439 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_add()
8440 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_add()
8442 dst_reg->s32_min_value += smin_val; in scalar32_min_max_add()
8443 dst_reg->s32_max_value += smax_val; in scalar32_min_max_add()
8445 if (dst_reg->u32_min_value + umin_val < umin_val || in scalar32_min_max_add()
8446 dst_reg->u32_max_value + umax_val < umax_val) { in scalar32_min_max_add()
8447 dst_reg->u32_min_value = 0; in scalar32_min_max_add()
8448 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_add()
8450 dst_reg->u32_min_value += umin_val; in scalar32_min_max_add()
8451 dst_reg->u32_max_value += umax_val; in scalar32_min_max_add()
8455 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, in scalar_min_max_add() argument
8463 if (signed_add_overflows(dst_reg->smin_value, smin_val) || in scalar_min_max_add()
8464 signed_add_overflows(dst_reg->smax_value, smax_val)) { in scalar_min_max_add()
8465 dst_reg->smin_value = S64_MIN; in scalar_min_max_add()
8466 dst_reg->smax_value = S64_MAX; in scalar_min_max_add()
8468 dst_reg->smin_value += smin_val; in scalar_min_max_add()
8469 dst_reg->smax_value += smax_val; in scalar_min_max_add()
8471 if (dst_reg->umin_value + umin_val < umin_val || in scalar_min_max_add()
8472 dst_reg->umax_value + umax_val < umax_val) { in scalar_min_max_add()
8473 dst_reg->umin_value = 0; in scalar_min_max_add()
8474 dst_reg->umax_value = U64_MAX; in scalar_min_max_add()
8476 dst_reg->umin_value += umin_val; in scalar_min_max_add()
8477 dst_reg->umax_value += umax_val; in scalar_min_max_add()
8481 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, in scalar32_min_max_sub() argument
8489 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || in scalar32_min_max_sub()
8490 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { in scalar32_min_max_sub()
8492 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_sub()
8493 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_sub()
8495 dst_reg->s32_min_value -= smax_val; in scalar32_min_max_sub()
8496 dst_reg->s32_max_value -= smin_val; in scalar32_min_max_sub()
8498 if (dst_reg->u32_min_value < umax_val) { in scalar32_min_max_sub()
8500 dst_reg->u32_min_value = 0; in scalar32_min_max_sub()
8501 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_sub()
8504 dst_reg->u32_min_value -= umax_val; in scalar32_min_max_sub()
8505 dst_reg->u32_max_value -= umin_val; in scalar32_min_max_sub()
8509 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, in scalar_min_max_sub() argument
8517 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || in scalar_min_max_sub()
8518 signed_sub_overflows(dst_reg->smax_value, smin_val)) { in scalar_min_max_sub()
8520 dst_reg->smin_value = S64_MIN; in scalar_min_max_sub()
8521 dst_reg->smax_value = S64_MAX; in scalar_min_max_sub()
8523 dst_reg->smin_value -= smax_val; in scalar_min_max_sub()
8524 dst_reg->smax_value -= smin_val; in scalar_min_max_sub()
8526 if (dst_reg->umin_value < umax_val) { in scalar_min_max_sub()
8528 dst_reg->umin_value = 0; in scalar_min_max_sub()
8529 dst_reg->umax_value = U64_MAX; in scalar_min_max_sub()
8532 dst_reg->umin_value -= umax_val; in scalar_min_max_sub()
8533 dst_reg->umax_value -= umin_val; in scalar_min_max_sub()
8537 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, in scalar32_min_max_mul() argument
8544 if (smin_val < 0 || dst_reg->s32_min_value < 0) { in scalar32_min_max_mul()
8546 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
8552 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { in scalar32_min_max_mul()
8554 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
8557 dst_reg->u32_min_value *= umin_val; in scalar32_min_max_mul()
8558 dst_reg->u32_max_value *= umax_val; in scalar32_min_max_mul()
8559 if (dst_reg->u32_max_value > S32_MAX) { in scalar32_min_max_mul()
8561 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_mul()
8562 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_mul()
8564 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_mul()
8565 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_mul()
8569 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, in scalar_min_max_mul() argument
8576 if (smin_val < 0 || dst_reg->smin_value < 0) { in scalar_min_max_mul()
8578 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
8584 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { in scalar_min_max_mul()
8586 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
8589 dst_reg->umin_value *= umin_val; in scalar_min_max_mul()
8590 dst_reg->umax_value *= umax_val; in scalar_min_max_mul()
8591 if (dst_reg->umax_value > S64_MAX) { in scalar_min_max_mul()
8593 dst_reg->smin_value = S64_MIN; in scalar_min_max_mul()
8594 dst_reg->smax_value = S64_MAX; in scalar_min_max_mul()
8596 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_mul()
8597 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_mul()
8601 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, in scalar32_min_max_and() argument
8605 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_and()
8606 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_and()
8611 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_and()
8618 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_and()
8619 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); in scalar32_min_max_and()
8620 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_and()
8624 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_and()
8625 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_and()
8630 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_and()
8631 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_and()
8635 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, in scalar_min_max_and() argument
8639 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_and()
8644 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_and()
8651 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_and()
8652 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); in scalar_min_max_and()
8653 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_and()
8657 dst_reg->smin_value = S64_MIN; in scalar_min_max_and()
8658 dst_reg->smax_value = S64_MAX; in scalar_min_max_and()
8663 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_and()
8664 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_and()
8667 __update_reg_bounds(dst_reg); in scalar_min_max_and()
8670 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, in scalar32_min_max_or() argument
8674 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_or()
8675 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_or()
8680 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_or()
8687 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); in scalar32_min_max_or()
8688 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_or()
8689 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_or()
8693 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_or()
8694 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_or()
8699 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_or()
8700 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_or()
8704 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, in scalar_min_max_or() argument
8708 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_or()
8713 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_or()
8720 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); in scalar_min_max_or()
8721 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_or()
8722 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_or()
8726 dst_reg->smin_value = S64_MIN; in scalar_min_max_or()
8727 dst_reg->smax_value = S64_MAX; in scalar_min_max_or()
8732 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_or()
8733 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_or()
8736 __update_reg_bounds(dst_reg); in scalar_min_max_or()
8739 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, in scalar32_min_max_xor() argument
8743 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_xor()
8744 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_xor()
8748 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_xor()
8753 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_xor()
8754 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_xor()
8756 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { in scalar32_min_max_xor()
8760 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_xor()
8761 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_xor()
8763 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_xor()
8764 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_xor()
8768 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, in scalar_min_max_xor() argument
8772 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_xor()
8777 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_xor()
8782 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_xor()
8783 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_xor()
8785 if (dst_reg->smin_value >= 0 && smin_val >= 0) { in scalar_min_max_xor()
8789 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_xor()
8790 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_xor()
8792 dst_reg->smin_value = S64_MIN; in scalar_min_max_xor()
8793 dst_reg->smax_value = S64_MAX; in scalar_min_max_xor()
8796 __update_reg_bounds(dst_reg); in scalar_min_max_xor()
8799 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar32_min_max_lsh() argument
8805 dst_reg->s32_min_value = S32_MIN; in __scalar32_min_max_lsh()
8806 dst_reg->s32_max_value = S32_MAX; in __scalar32_min_max_lsh()
8808 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { in __scalar32_min_max_lsh()
8809 dst_reg->u32_min_value = 0; in __scalar32_min_max_lsh()
8810 dst_reg->u32_max_value = U32_MAX; in __scalar32_min_max_lsh()
8812 dst_reg->u32_min_value <<= umin_val; in __scalar32_min_max_lsh()
8813 dst_reg->u32_max_value <<= umax_val; in __scalar32_min_max_lsh()
8817 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_lsh() argument
8823 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_lsh()
8825 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar32_min_max_lsh()
8826 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); in scalar32_min_max_lsh()
8831 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_lsh()
8832 __update_reg32_bounds(dst_reg); in scalar32_min_max_lsh()
8835 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar64_min_max_lsh() argument
8845 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) in __scalar64_min_max_lsh()
8846 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; in __scalar64_min_max_lsh()
8848 dst_reg->smax_value = S64_MAX; in __scalar64_min_max_lsh()
8850 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) in __scalar64_min_max_lsh()
8851 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; in __scalar64_min_max_lsh()
8853 dst_reg->smin_value = S64_MIN; in __scalar64_min_max_lsh()
8856 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { in __scalar64_min_max_lsh()
8857 dst_reg->umin_value = 0; in __scalar64_min_max_lsh()
8858 dst_reg->umax_value = U64_MAX; in __scalar64_min_max_lsh()
8860 dst_reg->umin_value <<= umin_val; in __scalar64_min_max_lsh()
8861 dst_reg->umax_value <<= umax_val; in __scalar64_min_max_lsh()
8865 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar_min_max_lsh() argument
8872 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
8873 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
8875 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); in scalar_min_max_lsh()
8877 __update_reg_bounds(dst_reg); in scalar_min_max_lsh()
8880 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_rsh() argument
8883 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_rsh()
8901 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_rsh()
8902 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_rsh()
8904 dst_reg->var_off = tnum_rshift(subreg, umin_val); in scalar32_min_max_rsh()
8905 dst_reg->u32_min_value >>= umax_val; in scalar32_min_max_rsh()
8906 dst_reg->u32_max_value >>= umin_val; in scalar32_min_max_rsh()
8908 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_rsh()
8909 __update_reg32_bounds(dst_reg); in scalar32_min_max_rsh()
8912 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar_min_max_rsh() argument
8932 dst_reg->smin_value = S64_MIN; in scalar_min_max_rsh()
8933 dst_reg->smax_value = S64_MAX; in scalar_min_max_rsh()
8934 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); in scalar_min_max_rsh()
8935 dst_reg->umin_value >>= umax_val; in scalar_min_max_rsh()
8936 dst_reg->umax_value >>= umin_val; in scalar_min_max_rsh()
8942 __mark_reg32_unbounded(dst_reg); in scalar_min_max_rsh()
8943 __update_reg_bounds(dst_reg); in scalar_min_max_rsh()
8946 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_arsh() argument
8954 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); in scalar32_min_max_arsh()
8955 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); in scalar32_min_max_arsh()
8957 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); in scalar32_min_max_arsh()
8962 dst_reg->u32_min_value = 0; in scalar32_min_max_arsh()
8963 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_arsh()
8965 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_arsh()
8966 __update_reg32_bounds(dst_reg); in scalar32_min_max_arsh()
8969 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar_min_max_arsh() argument
8977 dst_reg->smin_value >>= umin_val; in scalar_min_max_arsh()
8978 dst_reg->smax_value >>= umin_val; in scalar_min_max_arsh()
8980 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); in scalar_min_max_arsh()
8985 dst_reg->umin_value = 0; in scalar_min_max_arsh()
8986 dst_reg->umax_value = U64_MAX; in scalar_min_max_arsh()
8992 __mark_reg32_unbounded(dst_reg); in scalar_min_max_arsh()
8993 __update_reg_bounds(dst_reg); in scalar_min_max_arsh()
9002 struct bpf_reg_state *dst_reg, in adjust_scalar_min_max_vals() argument
9034 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9045 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9052 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9078 scalar32_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9079 scalar_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9080 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9083 scalar32_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9084 scalar_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9085 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9088 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9089 scalar32_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9090 scalar_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9093 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9094 scalar32_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9095 scalar_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9098 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9099 scalar32_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9100 scalar_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9103 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9104 scalar32_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9105 scalar_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9112 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9116 scalar32_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9118 scalar_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9125 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9129 scalar32_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9131 scalar_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9138 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9142 scalar32_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9144 scalar_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9147 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9153 zext_32_to_64(dst_reg); in adjust_scalar_min_max_vals()
9154 reg_bounds_sync(dst_reg); in adjust_scalar_min_max_vals()
9166 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; in adjust_reg_min_max_vals() local
9171 dst_reg = ®s[insn->dst_reg]; in adjust_reg_min_max_vals()
9173 if (dst_reg->type != SCALAR_VALUE) in adjust_reg_min_max_vals()
9174 ptr_reg = dst_reg; in adjust_reg_min_max_vals()
9179 dst_reg->id = 0; in adjust_reg_min_max_vals()
9183 if (dst_reg->type != SCALAR_VALUE) { in adjust_reg_min_max_vals()
9189 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
9193 insn->dst_reg, in adjust_reg_min_max_vals()
9201 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
9205 src_reg, dst_reg); in adjust_reg_min_max_vals()
9213 dst_reg, src_reg); in adjust_reg_min_max_vals()
9238 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
9266 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9270 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
9272 insn->dst_reg); in check_alu_op()
9277 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
9301 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9307 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; in check_alu_op() local
9319 *dst_reg = *src_reg; in check_alu_op()
9320 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
9321 dst_reg->subreg_def = DEF_NOT_SUBREG; in check_alu_op()
9330 *dst_reg = *src_reg; in check_alu_op()
9335 dst_reg->id = 0; in check_alu_op()
9336 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
9337 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
9340 insn->dst_reg); in check_alu_op()
9342 zext_32_to_64(dst_reg); in check_alu_op()
9343 reg_bounds_sync(dst_reg); in check_alu_op()
9350 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
9351 regs[insn->dst_reg].type = SCALAR_VALUE; in check_alu_op()
9353 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
9356 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
9384 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9405 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9416 struct bpf_reg_state *dst_reg, in find_good_pkt_pointers() argument
9424 if (dst_reg->off < 0 || in find_good_pkt_pointers()
9425 (dst_reg->off == 0 && range_right_open)) in find_good_pkt_pointers()
9429 if (dst_reg->umax_value > MAX_PACKET_OFF || in find_good_pkt_pointers()
9430 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) in find_good_pkt_pointers()
9436 new_range = dst_reg->off; in find_good_pkt_pointers()
9488 if (reg->type == type && reg->id == dst_reg->id) in find_good_pkt_pointers()
9696 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, in is_pkt_ptr_branch_taken() argument
9703 pkt = dst_reg; in is_pkt_ptr_branch_taken()
9704 } else if (dst_reg->type == PTR_TO_PACKET_END) { in is_pkt_ptr_branch_taken()
9916 struct bpf_reg_state *dst_reg) in __reg_combine_min_max() argument
9918 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, in __reg_combine_min_max()
9919 dst_reg->umin_value); in __reg_combine_min_max()
9920 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, in __reg_combine_min_max()
9921 dst_reg->umax_value); in __reg_combine_min_max()
9922 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, in __reg_combine_min_max()
9923 dst_reg->smin_value); in __reg_combine_min_max()
9924 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, in __reg_combine_min_max()
9925 dst_reg->smax_value); in __reg_combine_min_max()
9926 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, in __reg_combine_min_max()
9927 dst_reg->var_off); in __reg_combine_min_max()
9929 reg_bounds_sync(dst_reg); in __reg_combine_min_max()
10014 struct bpf_reg_state *dst_reg, in try_match_pkt_pointers() argument
10028 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10030 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10033 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
10034 dst_reg->type, false); in try_match_pkt_pointers()
10035 mark_pkt_end(other_branch, insn->dst_reg, true); in try_match_pkt_pointers()
10036 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10038 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10049 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10051 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10054 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
10055 dst_reg->type, true); in try_match_pkt_pointers()
10056 mark_pkt_end(this_branch, insn->dst_reg, false); in try_match_pkt_pointers()
10057 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10059 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10070 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10072 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10075 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
10076 dst_reg->type, true); in try_match_pkt_pointers()
10077 mark_pkt_end(other_branch, insn->dst_reg, false); in try_match_pkt_pointers()
10078 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10080 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10091 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10093 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10096 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
10097 dst_reg->type, false); in try_match_pkt_pointers()
10098 mark_pkt_end(this_branch, insn->dst_reg, true); in try_match_pkt_pointers()
10099 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10101 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10136 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; in check_cond_jmp_op() local
10173 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
10177 dst_reg = ®s[insn->dst_reg]; in check_cond_jmp_op()
10181 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); in check_cond_jmp_op()
10184 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
10190 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
10194 } else if (reg_is_pkt_pointer_any(dst_reg) && in check_cond_jmp_op()
10197 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); in check_cond_jmp_op()
10204 if (!__is_pointer_value(false, dst_reg)) in check_cond_jmp_op()
10205 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
10253 if (dst_reg->type == SCALAR_VALUE && in check_cond_jmp_op()
10258 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
10259 dst_reg, in check_cond_jmp_op()
10263 else if (tnum_is_const(dst_reg->var_off) || in check_cond_jmp_op()
10265 tnum_is_const(tnum_subreg(dst_reg->var_off)))) in check_cond_jmp_op()
10268 dst_reg->var_off.value, in check_cond_jmp_op()
10269 tnum_subreg(dst_reg->var_off).value, in check_cond_jmp_op()
10275 &other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
10276 src_reg, dst_reg, opcode); in check_cond_jmp_op()
10284 } else if (dst_reg->type == SCALAR_VALUE) { in check_cond_jmp_op()
10285 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
10286 dst_reg, insn->imm, (u32)insn->imm, in check_cond_jmp_op()
10290 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && in check_cond_jmp_op()
10291 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { in check_cond_jmp_op()
10292 find_equal_scalars(this_branch, dst_reg); in check_cond_jmp_op()
10293 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); in check_cond_jmp_op()
10302 type_may_be_null(dst_reg->type)) { in check_cond_jmp_op()
10306 mark_ptr_or_null_regs(this_branch, insn->dst_reg, in check_cond_jmp_op()
10308 mark_ptr_or_null_regs(other_branch, insn->dst_reg, in check_cond_jmp_op()
10310 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], in check_cond_jmp_op()
10312 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
10314 insn->dst_reg); in check_cond_jmp_op()
10327 struct bpf_reg_state *dst_reg; in check_ld_imm() local
10340 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
10344 dst_reg = ®s[insn->dst_reg]; in check_ld_imm()
10348 dst_reg->type = SCALAR_VALUE; in check_ld_imm()
10349 __mark_reg_known(®s[insn->dst_reg], imm); in check_ld_imm()
10357 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
10360 dst_reg->type = aux->btf_var.reg_type; in check_ld_imm()
10361 switch (base_type(dst_reg->type)) { in check_ld_imm()
10363 dst_reg->mem_size = aux->btf_var.mem_size; in check_ld_imm()
10366 dst_reg->btf = aux->btf_var.btf; in check_ld_imm()
10367 dst_reg->btf_id = aux->btf_var.btf_id; in check_ld_imm()
10390 dst_reg->type = PTR_TO_FUNC; in check_ld_imm()
10391 dst_reg->subprogno = subprogno; in check_ld_imm()
10396 dst_reg->map_ptr = map; in check_ld_imm()
10400 dst_reg->type = PTR_TO_MAP_VALUE; in check_ld_imm()
10401 dst_reg->off = aux->map_off; in check_ld_imm()
10403 dst_reg->id = ++env->id_gen; in check_ld_imm()
10406 dst_reg->type = CONST_PTR_TO_MAP; in check_ld_imm()
10459 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || in check_ld_abs()
12265 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
12276 BPF_READ, insn->dst_reg, false); in do_check()
12322 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12326 dst_reg_type = regs[insn->dst_reg].type; in do_check()
12329 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12351 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12355 if (is_ctx_reg(env, insn->dst_reg)) { in do_check()
12357 insn->dst_reg, in do_check()
12358 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in do_check()
12363 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12380 insn->dst_reg != BPF_REG_0 || in do_check()
12404 insn->dst_reg != BPF_REG_0 || in do_check()
12417 insn->dst_reg != BPF_REG_0 || in do_check()
12778 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || in resolve_pseudo_ldimm64()
13370 rnd_hi32_patch[3].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
13395 zext_patch[1].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
13574 insn->dst_reg, in convert_ctx_accesses()
13576 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
13581 insn->dst_reg, in convert_ctx_accesses()
13583 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
13938 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), in do_misc_fixups()
13949 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), in do_misc_fixups()
14005 off_reg = issrc ? insn->src_reg : insn->dst_reg; in do_misc_fixups()
14019 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); in do_misc_fixups()