Lines Matching +full:rs +full:-
1 // SPDX-License-Identifier: GPL-2.0
68 __set_bit(reg, &ctx->flags); in bpf_to_rv_reg()
83 return test_bit(reg, &ctx->flags); in seen_reg()
90 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); in mark_fp()
95 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in mark_call()
100 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in seen_call()
105 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in mark_tail_call()
110 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in seen_tail_call()
118 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); in rv_tail_call_reg()
126 return -(1L << 31) <= val && val < (1L << 31); in is_32b_int()
132 * auipc+jalr can reach any signed PC-relative offset in the range in in_auipc_jalr_range()
133 * [-2^31 - 2^11, 2^31 - 2^11). in in_auipc_jalr_range()
135 return (-(1L << 31) - (1L << 11)) <= val && in in_auipc_jalr_range()
136 val < ((1L << 31) - (1L << 11)); in in_auipc_jalr_range()
141 /* Note that the immediate from the add is sign-extended, in emit_imm()
145 * shift. The "Loading a 32-Bit constant" example from the in emit_imm()
146 * "Computer Organization and Design, RISC-V edition" book by in emit_imm()
152 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, in emit_imm()
184 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; in __build_epilogue()
188 store_offset -= 8; in __build_epilogue()
191 store_offset -= 8; in __build_epilogue()
194 store_offset -= 8; in __build_epilogue()
198 store_offset -= 8; in __build_epilogue()
202 store_offset -= 8; in __build_epilogue()
206 store_offset -= 8; in __build_epilogue()
210 store_offset -= 8; in __build_epilogue()
214 store_offset -= 8; in __build_epilogue()
226 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, in emit_bcc() argument
231 emit(rv_beq(rd, rs, rvoff >> 1), ctx); in emit_bcc()
234 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
237 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
240 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
243 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
246 emit(rv_bne(rd, rs, rvoff >> 1), ctx); in emit_bcc()
249 emit(rv_blt(rs, rd, rvoff >> 1), ctx); in emit_bcc()
252 emit(rv_blt(rd, rs, rvoff >> 1), ctx); in emit_bcc()
255 emit(rv_bge(rd, rs, rvoff >> 1), ctx); in emit_bcc()
258 emit(rv_bge(rs, rd, rvoff >> 1), ctx); in emit_bcc()
262 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, in emit_branch() argument
268 emit_bcc(cond, rd, rs, rvoff, ctx); in emit_branch()
273 rvoff -= 4; in emit_branch()
276 * bne rd,rs,foo in emit_branch()
278 * beq rd,rs,<.L1> in emit_branch()
285 emit_bcc(cond, rd, rs, 8, ctx); in emit_branch()
296 emit_bcc(cond, rd, rs, 12, ctx); in emit_branch()
309 int tc_ninsn, off, start_insn = ctx->ninsns; in emit_bpf_tail_call()
316 * if (index >= array->map.max_entries) in emit_bpf_tail_call()
319 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : in emit_bpf_tail_call()
320 ctx->offset[0]; in emit_bpf_tail_call()
325 return -1; in emit_bpf_tail_call()
327 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
330 /* if (--TCC < 0) in emit_bpf_tail_call()
333 emit_addi(RV_REG_TCC, tcc, -1, ctx); in emit_bpf_tail_call()
334 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
337 /* prog = array->ptrs[index]; in emit_bpf_tail_call()
345 return -1; in emit_bpf_tail_call()
347 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
350 /* goto *(prog->bpf_func + 4); */ in emit_bpf_tail_call()
353 return -1; in emit_bpf_tail_call()
359 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, in init_regs() argument
362 u8 code = insn->code; in init_regs()
371 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); in init_regs()
377 *rs = bpf_to_rv_reg(insn->src_reg, ctx); in init_regs()
380 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_zext_32_rd_rs() argument
384 emit_mv(RV_REG_T1, *rs, ctx); in emit_zext_32_rd_rs()
387 *rs = RV_REG_T1; in emit_zext_32_rd_rs()
390 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_sext_32_rd_rs() argument
393 emit_addiw(RV_REG_T1, *rs, 0, ctx); in emit_sext_32_rd_rs()
395 *rs = RV_REG_T1; in emit_sext_32_rd_rs()
428 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); in emit_jump_and_link()
429 return -ERANGE; in emit_jump_and_link()
445 if (addr && ctx->insns) { in emit_call()
446 ip = (u64)(long)(ctx->insns + ctx->ninsns); in emit_call()
447 off = addr - ip; in emit_call()
458 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, in emit_atomic() argument
477 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : in emit_atomic()
478 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); in emit_atomic()
481 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : in emit_atomic()
482 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); in emit_atomic()
485 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : in emit_atomic()
486 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); in emit_atomic()
489 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : in emit_atomic()
490 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); in emit_atomic()
494 emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) : in emit_atomic()
495 rv_amoadd_w(rs, rs, rd, 0, 0), ctx); in emit_atomic()
497 emit_zext_32(rs, ctx); in emit_atomic()
500 emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) : in emit_atomic()
501 rv_amoand_w(rs, rs, rd, 0, 0), ctx); in emit_atomic()
503 emit_zext_32(rs, ctx); in emit_atomic()
506 emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) : in emit_atomic()
507 rv_amoor_w(rs, rs, rd, 0, 0), ctx); in emit_atomic()
509 emit_zext_32(rs, ctx); in emit_atomic()
512 emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) : in emit_atomic()
513 rv_amoxor_w(rs, rs, rd, 0, 0), ctx); in emit_atomic()
515 emit_zext_32(rs, ctx); in emit_atomic()
519 emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) : in emit_atomic()
520 rv_amoswap_w(rs, rs, rd, 0, 0), ctx); in emit_atomic()
522 emit_zext_32(rs, ctx); in emit_atomic()
533 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : in emit_atomic()
534 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); in emit_atomic()
535 jmp_offset = ninsns_rvoff(-6); in emit_atomic()
548 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); in ex_handler_bpf()
549 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); in ex_handler_bpf()
552 regs->epc = (unsigned long)&ex->fixup - offset; in ex_handler_bpf()
566 if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM) in add_exception_handler()
569 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) in add_exception_handler()
570 return -EINVAL; in add_exception_handler()
572 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) in add_exception_handler()
573 return -EINVAL; in add_exception_handler()
576 return -EINVAL; in add_exception_handler()
578 ex = &ctx->prog->aux->extable[ctx->nexentries]; in add_exception_handler()
579 pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len]; in add_exception_handler()
581 offset = pc - (long)&ex->insn; in add_exception_handler()
583 return -ERANGE; in add_exception_handler()
584 ex->insn = offset; in add_exception_handler()
594 offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); in add_exception_handler()
596 return -ERANGE; in add_exception_handler()
598 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | in add_exception_handler()
600 ex->type = EX_TYPE_BPF; in add_exception_handler()
602 ctx->nexentries++; in add_exception_handler()
609 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || in bpf_jit_emit_insn()
610 BPF_CLASS(insn->code) == BPF_JMP; in bpf_jit_emit_insn()
611 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
612 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
613 u8 rd = -1, rs = -1, code = insn->code; in bpf_jit_emit_insn() local
614 s16 off = insn->off; in bpf_jit_emit_insn()
615 s32 imm = insn->imm; in bpf_jit_emit_insn()
617 init_regs(&rd, &rs, insn, ctx); in bpf_jit_emit_insn()
628 emit_mv(rd, rs, ctx); in bpf_jit_emit_insn()
629 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
636 emit_add(rd, rd, rs, ctx); in bpf_jit_emit_insn()
637 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
643 emit_sub(rd, rd, rs, ctx); in bpf_jit_emit_insn()
645 emit_subw(rd, rd, rs, ctx); in bpf_jit_emit_insn()
647 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
652 emit_and(rd, rd, rs, ctx); in bpf_jit_emit_insn()
653 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
658 emit_or(rd, rd, rs, ctx); in bpf_jit_emit_insn()
659 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
664 emit_xor(rd, rd, rs, ctx); in bpf_jit_emit_insn()
665 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
670 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
671 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
676 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
677 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
682 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
683 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
688 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
689 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
694 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
695 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
700 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
701 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
705 /* dst = -dst */ in bpf_jit_emit_insn()
709 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
721 if (!aux->verifier_zext) in bpf_jit_emit_insn()
782 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
795 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
800 if (is_12b_int(-imm)) { in bpf_jit_emit_insn()
801 emit_addi(rd, rd, -imm, ctx); in bpf_jit_emit_insn()
806 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
817 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
828 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
839 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
847 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
855 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
863 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
870 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
880 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
890 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
927 s = ctx->ninsns; in bpf_jit_emit_insn()
929 emit_sext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
931 emit_zext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
932 e = ctx->ninsns; in bpf_jit_emit_insn()
935 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
940 rvoff -= 4; in bpf_jit_emit_insn()
941 emit_and(RV_REG_T1, rd, rs, ctx); in bpf_jit_emit_insn()
945 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
971 s = ctx->ninsns; in bpf_jit_emit_insn()
974 rs = RV_REG_T1; in bpf_jit_emit_insn()
977 rs = RV_REG_ZERO; in bpf_jit_emit_insn()
985 e = ctx->ninsns; in bpf_jit_emit_insn()
988 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
989 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
995 s = ctx->ninsns; in bpf_jit_emit_insn()
1003 * sign-extension is sufficient here and saves one instruction, in bpf_jit_emit_insn()
1008 e = ctx->ninsns; in bpf_jit_emit_insn()
1009 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
1020 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, in bpf_jit_emit_insn()
1032 return -1; in bpf_jit_emit_insn()
1037 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
1072 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1073 emit(rv_lbu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1074 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1079 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
1080 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1082 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1088 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1089 emit(rv_lhu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1090 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1095 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
1096 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1098 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1104 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1105 emit(rv_lwu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1106 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1111 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
1112 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1114 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1120 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1121 emit_ld(rd, off, rs, ctx); in bpf_jit_emit_insn()
1122 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1127 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
1128 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1130 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1193 emit(rv_sb(rd, off, rs), ctx); in bpf_jit_emit_insn()
1199 emit(rv_sb(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1203 emit(rv_sh(rd, off, rs), ctx); in bpf_jit_emit_insn()
1209 emit(rv_sh(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1213 emit_sw(rd, off, rs, ctx); in bpf_jit_emit_insn()
1219 emit_sw(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1223 emit_sd(rd, off, rs, ctx); in bpf_jit_emit_insn()
1229 emit_sd(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1233 emit_atomic(rd, rs, off, imm, in bpf_jit_emit_insn()
1237 pr_err("bpf-jit: unknown opcode %02x\n", code); in bpf_jit_emit_insn()
1238 return -EINVAL; in bpf_jit_emit_insn()
1248 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); in bpf_jit_build_prologue()
1271 store_offset = stack_adjust - 8; in bpf_jit_build_prologue()
1273 /* First instruction is always setting the tail-call-counter in bpf_jit_build_prologue()
1275 * Force using a 4-byte (non-compressed) instruction. in bpf_jit_build_prologue()
1279 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); in bpf_jit_build_prologue()
1283 store_offset -= 8; in bpf_jit_build_prologue()
1286 store_offset -= 8; in bpf_jit_build_prologue()
1289 store_offset -= 8; in bpf_jit_build_prologue()
1293 store_offset -= 8; in bpf_jit_build_prologue()
1297 store_offset -= 8; in bpf_jit_build_prologue()
1301 store_offset -= 8; in bpf_jit_build_prologue()
1305 store_offset -= 8; in bpf_jit_build_prologue()
1309 store_offset -= 8; in bpf_jit_build_prologue()
1323 ctx->stack_size = stack_adjust; in bpf_jit_build_prologue()