Lines Matching +full:rs +full:-
1 // SPDX-License-Identifier: GPL-2.0
52 __set_bit(reg, &ctx->flags); in bpf_to_rv_reg()
67 return test_bit(reg, &ctx->flags); in seen_reg()
74 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); in mark_fp()
79 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in mark_call()
84 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in seen_call()
89 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in mark_tail_call()
94 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in seen_tail_call()
102 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); in rv_tail_call_reg()
110 return -(1L << 31) <= val && val < (1L << 31); in is_32b_int()
116 * auipc+jalr can reach any signed PC-relative offset in the range in in_auipc_jalr_range()
117 * [-2^31 - 2^11, 2^31 - 2^11). in in_auipc_jalr_range()
119 return (-(1L << 31) - (1L << 11)) <= val && in in_auipc_jalr_range()
120 val < ((1L << 31) - (1L << 11)); in in_auipc_jalr_range()
125 /* Note that the immediate from the add is sign-extended, in emit_imm()
129 * shift. The "Loading a 32-Bit constant" example from the in emit_imm()
130 * "Computer Organization and Design, RISC-V edition" book by in emit_imm()
136 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, in emit_imm()
168 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; in __build_epilogue()
172 store_offset -= 8; in __build_epilogue()
175 store_offset -= 8; in __build_epilogue()
178 store_offset -= 8; in __build_epilogue()
182 store_offset -= 8; in __build_epilogue()
186 store_offset -= 8; in __build_epilogue()
190 store_offset -= 8; in __build_epilogue()
194 store_offset -= 8; in __build_epilogue()
198 store_offset -= 8; in __build_epilogue()
210 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, in emit_bcc() argument
215 emit(rv_beq(rd, rs, rvoff >> 1), ctx); in emit_bcc()
218 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
221 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
224 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
227 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
230 emit(rv_bne(rd, rs, rvoff >> 1), ctx); in emit_bcc()
233 emit(rv_blt(rs, rd, rvoff >> 1), ctx); in emit_bcc()
236 emit(rv_blt(rd, rs, rvoff >> 1), ctx); in emit_bcc()
239 emit(rv_bge(rd, rs, rvoff >> 1), ctx); in emit_bcc()
242 emit(rv_bge(rs, rd, rvoff >> 1), ctx); in emit_bcc()
246 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, in emit_branch() argument
252 emit_bcc(cond, rd, rs, rvoff, ctx); in emit_branch()
257 rvoff -= 4; in emit_branch()
260 * bne rd,rs,foo in emit_branch()
262 * beq rd,rs,<.L1> in emit_branch()
269 emit_bcc(cond, rd, rs, 8, ctx); in emit_branch()
280 emit_bcc(cond, rd, rs, 12, ctx); in emit_branch()
293 int tc_ninsn, off, start_insn = ctx->ninsns; in emit_bpf_tail_call()
300 * if (index >= array->map.max_entries) in emit_bpf_tail_call()
303 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : in emit_bpf_tail_call()
304 ctx->offset[0]; in emit_bpf_tail_call()
309 return -1; in emit_bpf_tail_call()
311 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
314 /* if (TCC-- < 0) in emit_bpf_tail_call()
317 emit_addi(RV_REG_T1, tcc, -1, ctx); in emit_bpf_tail_call()
318 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
321 /* prog = array->ptrs[index]; in emit_bpf_tail_call()
329 return -1; in emit_bpf_tail_call()
331 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
334 /* goto *(prog->bpf_func + 4); */ in emit_bpf_tail_call()
337 return -1; in emit_bpf_tail_call()
344 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, in init_regs() argument
347 u8 code = insn->code; in init_regs()
356 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); in init_regs()
362 *rs = bpf_to_rv_reg(insn->src_reg, ctx); in init_regs()
365 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_zext_32_rd_rs() argument
369 emit_mv(RV_REG_T1, *rs, ctx); in emit_zext_32_rd_rs()
372 *rs = RV_REG_T1; in emit_zext_32_rd_rs()
375 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_sext_32_rd_rs() argument
378 emit_addiw(RV_REG_T1, *rs, 0, ctx); in emit_sext_32_rd_rs()
380 *rs = RV_REG_T1; in emit_sext_32_rd_rs()
413 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); in emit_jump_and_link()
414 return -ERANGE; in emit_jump_and_link()
430 if (addr && ctx->insns) { in emit_call()
431 ip = (u64)(long)(ctx->insns + ctx->ninsns); in emit_call()
432 off = addr - ip; in emit_call()
446 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || in bpf_jit_emit_insn()
447 BPF_CLASS(insn->code) == BPF_JMP; in bpf_jit_emit_insn()
448 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
449 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
450 u8 rd = -1, rs = -1, code = insn->code; in bpf_jit_emit_insn() local
451 s16 off = insn->off; in bpf_jit_emit_insn()
452 s32 imm = insn->imm; in bpf_jit_emit_insn()
454 init_regs(&rd, &rs, insn, ctx); in bpf_jit_emit_insn()
465 emit_mv(rd, rs, ctx); in bpf_jit_emit_insn()
466 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
473 emit_add(rd, rd, rs, ctx); in bpf_jit_emit_insn()
474 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
480 emit_sub(rd, rd, rs, ctx); in bpf_jit_emit_insn()
482 emit_subw(rd, rd, rs, ctx); in bpf_jit_emit_insn()
484 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
489 emit_and(rd, rd, rs, ctx); in bpf_jit_emit_insn()
490 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
495 emit_or(rd, rd, rs, ctx); in bpf_jit_emit_insn()
496 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
501 emit_xor(rd, rd, rs, ctx); in bpf_jit_emit_insn()
502 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
507 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
508 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
513 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
514 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
519 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
520 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
525 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
526 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
531 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
532 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
537 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
538 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
542 /* dst = -dst */ in bpf_jit_emit_insn()
546 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
558 if (!aux->verifier_zext) in bpf_jit_emit_insn()
619 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
632 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
637 if (is_12b_int(-imm)) { in bpf_jit_emit_insn()
638 emit_addi(rd, rd, -imm, ctx); in bpf_jit_emit_insn()
643 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
654 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
665 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
676 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
684 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
692 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
700 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
707 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
717 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
727 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
764 s = ctx->ninsns; in bpf_jit_emit_insn()
766 emit_sext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
768 emit_zext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
769 e = ctx->ninsns; in bpf_jit_emit_insn()
772 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
777 rvoff -= 4; in bpf_jit_emit_insn()
778 emit_and(RV_REG_T1, rd, rs, ctx); in bpf_jit_emit_insn()
782 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
808 s = ctx->ninsns; in bpf_jit_emit_insn()
811 rs = RV_REG_T1; in bpf_jit_emit_insn()
814 rs = RV_REG_ZERO; in bpf_jit_emit_insn()
822 e = ctx->ninsns; in bpf_jit_emit_insn()
825 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
826 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
832 s = ctx->ninsns; in bpf_jit_emit_insn()
840 * sign-extension is sufficient here and saves one instruction, in bpf_jit_emit_insn()
845 e = ctx->ninsns; in bpf_jit_emit_insn()
846 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
857 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, in bpf_jit_emit_insn()
869 return -1; in bpf_jit_emit_insn()
874 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
897 emit(rv_lbu(rd, off, rs), ctx); in bpf_jit_emit_insn()
902 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
909 emit(rv_lhu(rd, off, rs), ctx); in bpf_jit_emit_insn()
914 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
921 emit(rv_lwu(rd, off, rs), ctx); in bpf_jit_emit_insn()
926 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
933 emit_ld(rd, off, rs, ctx); in bpf_jit_emit_insn()
938 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
996 emit(rv_sb(rd, off, rs), ctx); in bpf_jit_emit_insn()
1002 emit(rv_sb(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1006 emit(rv_sh(rd, off, rs), ctx); in bpf_jit_emit_insn()
1012 emit(rv_sh(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1016 emit_sw(rd, off, rs, ctx); in bpf_jit_emit_insn()
1022 emit_sw(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1026 emit_sd(rd, off, rs, ctx); in bpf_jit_emit_insn()
1032 emit_sd(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1036 if (insn->imm != BPF_ADD) { in bpf_jit_emit_insn()
1037 pr_err("bpf-jit: not supported: atomic operation %02x ***\n", in bpf_jit_emit_insn()
1038 insn->imm); in bpf_jit_emit_insn()
1039 return -EINVAL; in bpf_jit_emit_insn()
1058 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) : in bpf_jit_emit_insn()
1059 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx); in bpf_jit_emit_insn()
1062 pr_err("bpf-jit: unknown opcode %02x\n", code); in bpf_jit_emit_insn()
1063 return -EINVAL; in bpf_jit_emit_insn()
1073 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); in bpf_jit_build_prologue()
1096 store_offset = stack_adjust - 8; in bpf_jit_build_prologue()
1098 /* First instruction is always setting the tail-call-counter in bpf_jit_build_prologue()
1100 * Force using a 4-byte (non-compressed) instruction. in bpf_jit_build_prologue()
1104 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); in bpf_jit_build_prologue()
1108 store_offset -= 8; in bpf_jit_build_prologue()
1111 store_offset -= 8; in bpf_jit_build_prologue()
1114 store_offset -= 8; in bpf_jit_build_prologue()
1118 store_offset -= 8; in bpf_jit_build_prologue()
1122 store_offset -= 8; in bpf_jit_build_prologue()
1126 store_offset -= 8; in bpf_jit_build_prologue()
1130 store_offset -= 8; in bpf_jit_build_prologue()
1134 store_offset -= 8; in bpf_jit_build_prologue()
1148 ctx->stack_size = stack_adjust; in bpf_jit_build_prologue()