Lines Matching +full:lower +full:- +full:case
1 // SPDX-License-Identifier: GPL-2.0
46 case RV_CTX_F_SEEN_S1: in bpf_to_rv_reg()
47 case RV_CTX_F_SEEN_S2: in bpf_to_rv_reg()
48 case RV_CTX_F_SEEN_S3: in bpf_to_rv_reg()
49 case RV_CTX_F_SEEN_S4: in bpf_to_rv_reg()
50 case RV_CTX_F_SEEN_S5: in bpf_to_rv_reg()
51 case RV_CTX_F_SEEN_S6: in bpf_to_rv_reg()
52 __set_bit(reg, &ctx->flags); in bpf_to_rv_reg()
60 case RV_CTX_F_SEEN_CALL: in seen_reg()
61 case RV_CTX_F_SEEN_S1: in seen_reg()
62 case RV_CTX_F_SEEN_S2: in seen_reg()
63 case RV_CTX_F_SEEN_S3: in seen_reg()
64 case RV_CTX_F_SEEN_S4: in seen_reg()
65 case RV_CTX_F_SEEN_S5: in seen_reg()
66 case RV_CTX_F_SEEN_S6: in seen_reg()
67 return test_bit(reg, &ctx->flags); in seen_reg()
74 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); in mark_fp()
79 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in mark_call()
84 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in seen_call()
89 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in mark_tail_call()
94 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in seen_tail_call()
102 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); in rv_tail_call_reg()
110 return -(1L << 31) <= val && val < (1L << 31); in is_32b_int()
116 * auipc+jalr can reach any signed PC-relative offset in the range in in_auipc_jalr_range()
117 * [-2^31 - 2^11, 2^31 - 2^11). in in_auipc_jalr_range()
119 return (-(1L << 31) - (1L << 11)) <= val && in in_auipc_jalr_range()
120 val < ((1L << 31) - (1L << 11)); in in_auipc_jalr_range()
125 /* Note that the immediate from the add is sign-extended, in emit_imm()
129 * shift. The "Loading a 32-Bit constant" example from the in emit_imm()
130 * "Computer Organization and Design, RISC-V edition" book by in emit_imm()
136 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, in emit_imm()
139 s64 lower = ((val & 0xfff) << 52) >> 52; in emit_imm() local
147 emit_li(rd, lower, ctx); in emit_imm()
151 emit_addiw(rd, rd, lower, ctx); in emit_imm()
162 if (lower) in emit_imm()
163 emit_addi(rd, rd, lower, ctx); in emit_imm()
168 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; in __build_epilogue()
172 store_offset -= 8; in __build_epilogue()
175 store_offset -= 8; in __build_epilogue()
178 store_offset -= 8; in __build_epilogue()
182 store_offset -= 8; in __build_epilogue()
186 store_offset -= 8; in __build_epilogue()
190 store_offset -= 8; in __build_epilogue()
194 store_offset -= 8; in __build_epilogue()
198 store_offset -= 8; in __build_epilogue()
214 case BPF_JEQ: in emit_bcc()
217 case BPF_JGT: in emit_bcc()
220 case BPF_JLT: in emit_bcc()
223 case BPF_JGE: in emit_bcc()
226 case BPF_JLE: in emit_bcc()
229 case BPF_JNE: in emit_bcc()
232 case BPF_JSGT: in emit_bcc()
235 case BPF_JSLT: in emit_bcc()
238 case BPF_JSGE: in emit_bcc()
241 case BPF_JSLE: in emit_bcc()
249 s64 upper, lower; in emit_branch() local
257 rvoff -= 4; in emit_branch()
278 lower = rvoff & 0xfff; in emit_branch()
282 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); in emit_branch()
293 int tc_ninsn, off, start_insn = ctx->ninsns; in emit_bpf_tail_call()
300 * if (index >= array->map.max_entries) in emit_bpf_tail_call()
303 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : in emit_bpf_tail_call()
304 ctx->offset[0]; in emit_bpf_tail_call()
309 return -1; in emit_bpf_tail_call()
311 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
314 /* if (TCC-- < 0) in emit_bpf_tail_call()
317 emit_addi(RV_REG_T1, tcc, -1, ctx); in emit_bpf_tail_call()
318 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
321 /* prog = array->ptrs[index]; in emit_bpf_tail_call()
329 return -1; in emit_bpf_tail_call()
331 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
334 /* goto *(prog->bpf_func + 4); */ in emit_bpf_tail_call()
337 return -1; in emit_bpf_tail_call()
347 u8 code = insn->code; in init_regs()
350 case BPF_JMP | BPF_JA: in init_regs()
351 case BPF_JMP | BPF_CALL: in init_regs()
352 case BPF_JMP | BPF_EXIT: in init_regs()
353 case BPF_JMP | BPF_TAIL_CALL: in init_regs()
356 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); in init_regs()
362 *rs = bpf_to_rv_reg(insn->src_reg, ctx); in init_regs()
400 s64 upper, lower; in emit_jump_and_link() local
407 lower = rvoff & 0xfff; in emit_jump_and_link()
409 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); in emit_jump_and_link()
413 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); in emit_jump_and_link()
414 return -ERANGE; in emit_jump_and_link()
430 if (addr && ctx->insns) { in emit_call()
431 ip = (u64)(long)(ctx->insns + ctx->ninsns); in emit_call()
432 off = addr - ip; in emit_call()
446 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || in bpf_jit_emit_insn()
447 BPF_CLASS(insn->code) == BPF_JMP; in bpf_jit_emit_insn()
448 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
449 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
450 u8 rd = -1, rs = -1, code = insn->code; in bpf_jit_emit_insn()
451 s16 off = insn->off; in bpf_jit_emit_insn()
452 s32 imm = insn->imm; in bpf_jit_emit_insn()
458 case BPF_ALU | BPF_MOV | BPF_X: in bpf_jit_emit_insn()
459 case BPF_ALU64 | BPF_MOV | BPF_X: in bpf_jit_emit_insn()
466 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
471 case BPF_ALU | BPF_ADD | BPF_X: in bpf_jit_emit_insn()
472 case BPF_ALU64 | BPF_ADD | BPF_X: in bpf_jit_emit_insn()
474 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
477 case BPF_ALU | BPF_SUB | BPF_X: in bpf_jit_emit_insn()
478 case BPF_ALU64 | BPF_SUB | BPF_X: in bpf_jit_emit_insn()
484 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
487 case BPF_ALU | BPF_AND | BPF_X: in bpf_jit_emit_insn()
488 case BPF_ALU64 | BPF_AND | BPF_X: in bpf_jit_emit_insn()
490 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
493 case BPF_ALU | BPF_OR | BPF_X: in bpf_jit_emit_insn()
494 case BPF_ALU64 | BPF_OR | BPF_X: in bpf_jit_emit_insn()
496 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
499 case BPF_ALU | BPF_XOR | BPF_X: in bpf_jit_emit_insn()
500 case BPF_ALU64 | BPF_XOR | BPF_X: in bpf_jit_emit_insn()
502 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
505 case BPF_ALU | BPF_MUL | BPF_X: in bpf_jit_emit_insn()
506 case BPF_ALU64 | BPF_MUL | BPF_X: in bpf_jit_emit_insn()
508 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
511 case BPF_ALU | BPF_DIV | BPF_X: in bpf_jit_emit_insn()
512 case BPF_ALU64 | BPF_DIV | BPF_X: in bpf_jit_emit_insn()
514 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
517 case BPF_ALU | BPF_MOD | BPF_X: in bpf_jit_emit_insn()
518 case BPF_ALU64 | BPF_MOD | BPF_X: in bpf_jit_emit_insn()
520 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
523 case BPF_ALU | BPF_LSH | BPF_X: in bpf_jit_emit_insn()
524 case BPF_ALU64 | BPF_LSH | BPF_X: in bpf_jit_emit_insn()
526 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
529 case BPF_ALU | BPF_RSH | BPF_X: in bpf_jit_emit_insn()
530 case BPF_ALU64 | BPF_RSH | BPF_X: in bpf_jit_emit_insn()
532 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
535 case BPF_ALU | BPF_ARSH | BPF_X: in bpf_jit_emit_insn()
536 case BPF_ALU64 | BPF_ARSH | BPF_X: in bpf_jit_emit_insn()
538 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
542 /* dst = -dst */ in bpf_jit_emit_insn()
543 case BPF_ALU | BPF_NEG: in bpf_jit_emit_insn()
544 case BPF_ALU64 | BPF_NEG: in bpf_jit_emit_insn()
546 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
551 case BPF_ALU | BPF_END | BPF_FROM_LE: in bpf_jit_emit_insn()
553 case 16: in bpf_jit_emit_insn()
557 case 32: in bpf_jit_emit_insn()
558 if (!aux->verifier_zext) in bpf_jit_emit_insn()
561 case 64: in bpf_jit_emit_insn()
567 case BPF_ALU | BPF_END | BPF_FROM_BE: in bpf_jit_emit_insn()
616 case BPF_ALU | BPF_MOV | BPF_K: in bpf_jit_emit_insn()
617 case BPF_ALU64 | BPF_MOV | BPF_K: in bpf_jit_emit_insn()
619 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
624 case BPF_ALU | BPF_ADD | BPF_K: in bpf_jit_emit_insn()
625 case BPF_ALU64 | BPF_ADD | BPF_K: in bpf_jit_emit_insn()
632 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
635 case BPF_ALU | BPF_SUB | BPF_K: in bpf_jit_emit_insn()
636 case BPF_ALU64 | BPF_SUB | BPF_K: in bpf_jit_emit_insn()
637 if (is_12b_int(-imm)) { in bpf_jit_emit_insn()
638 emit_addi(rd, rd, -imm, ctx); in bpf_jit_emit_insn()
643 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
646 case BPF_ALU | BPF_AND | BPF_K: in bpf_jit_emit_insn()
647 case BPF_ALU64 | BPF_AND | BPF_K: in bpf_jit_emit_insn()
654 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
657 case BPF_ALU | BPF_OR | BPF_K: in bpf_jit_emit_insn()
658 case BPF_ALU64 | BPF_OR | BPF_K: in bpf_jit_emit_insn()
665 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
668 case BPF_ALU | BPF_XOR | BPF_K: in bpf_jit_emit_insn()
669 case BPF_ALU64 | BPF_XOR | BPF_K: in bpf_jit_emit_insn()
676 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
679 case BPF_ALU | BPF_MUL | BPF_K: in bpf_jit_emit_insn()
680 case BPF_ALU64 | BPF_MUL | BPF_K: in bpf_jit_emit_insn()
684 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
687 case BPF_ALU | BPF_DIV | BPF_K: in bpf_jit_emit_insn()
688 case BPF_ALU64 | BPF_DIV | BPF_K: in bpf_jit_emit_insn()
692 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
695 case BPF_ALU | BPF_MOD | BPF_K: in bpf_jit_emit_insn()
696 case BPF_ALU64 | BPF_MOD | BPF_K: in bpf_jit_emit_insn()
700 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
703 case BPF_ALU | BPF_LSH | BPF_K: in bpf_jit_emit_insn()
704 case BPF_ALU64 | BPF_LSH | BPF_K: in bpf_jit_emit_insn()
707 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
710 case BPF_ALU | BPF_RSH | BPF_K: in bpf_jit_emit_insn()
711 case BPF_ALU64 | BPF_RSH | BPF_K: in bpf_jit_emit_insn()
717 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
720 case BPF_ALU | BPF_ARSH | BPF_K: in bpf_jit_emit_insn()
721 case BPF_ALU64 | BPF_ARSH | BPF_K: in bpf_jit_emit_insn()
727 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
732 case BPF_JMP | BPF_JA: in bpf_jit_emit_insn()
740 case BPF_JMP | BPF_JEQ | BPF_X: in bpf_jit_emit_insn()
741 case BPF_JMP32 | BPF_JEQ | BPF_X: in bpf_jit_emit_insn()
742 case BPF_JMP | BPF_JGT | BPF_X: in bpf_jit_emit_insn()
743 case BPF_JMP32 | BPF_JGT | BPF_X: in bpf_jit_emit_insn()
744 case BPF_JMP | BPF_JLT | BPF_X: in bpf_jit_emit_insn()
745 case BPF_JMP32 | BPF_JLT | BPF_X: in bpf_jit_emit_insn()
746 case BPF_JMP | BPF_JGE | BPF_X: in bpf_jit_emit_insn()
747 case BPF_JMP32 | BPF_JGE | BPF_X: in bpf_jit_emit_insn()
748 case BPF_JMP | BPF_JLE | BPF_X: in bpf_jit_emit_insn()
749 case BPF_JMP32 | BPF_JLE | BPF_X: in bpf_jit_emit_insn()
750 case BPF_JMP | BPF_JNE | BPF_X: in bpf_jit_emit_insn()
751 case BPF_JMP32 | BPF_JNE | BPF_X: in bpf_jit_emit_insn()
752 case BPF_JMP | BPF_JSGT | BPF_X: in bpf_jit_emit_insn()
753 case BPF_JMP32 | BPF_JSGT | BPF_X: in bpf_jit_emit_insn()
754 case BPF_JMP | BPF_JSLT | BPF_X: in bpf_jit_emit_insn()
755 case BPF_JMP32 | BPF_JSLT | BPF_X: in bpf_jit_emit_insn()
756 case BPF_JMP | BPF_JSGE | BPF_X: in bpf_jit_emit_insn()
757 case BPF_JMP32 | BPF_JSGE | BPF_X: in bpf_jit_emit_insn()
758 case BPF_JMP | BPF_JSLE | BPF_X: in bpf_jit_emit_insn()
759 case BPF_JMP32 | BPF_JSLE | BPF_X: in bpf_jit_emit_insn()
760 case BPF_JMP | BPF_JSET | BPF_X: in bpf_jit_emit_insn()
761 case BPF_JMP32 | BPF_JSET | BPF_X: in bpf_jit_emit_insn()
764 s = ctx->ninsns; in bpf_jit_emit_insn()
769 e = ctx->ninsns; in bpf_jit_emit_insn()
772 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
777 rvoff -= 4; in bpf_jit_emit_insn()
787 case BPF_JMP | BPF_JEQ | BPF_K: in bpf_jit_emit_insn()
788 case BPF_JMP32 | BPF_JEQ | BPF_K: in bpf_jit_emit_insn()
789 case BPF_JMP | BPF_JGT | BPF_K: in bpf_jit_emit_insn()
790 case BPF_JMP32 | BPF_JGT | BPF_K: in bpf_jit_emit_insn()
791 case BPF_JMP | BPF_JLT | BPF_K: in bpf_jit_emit_insn()
792 case BPF_JMP32 | BPF_JLT | BPF_K: in bpf_jit_emit_insn()
793 case BPF_JMP | BPF_JGE | BPF_K: in bpf_jit_emit_insn()
794 case BPF_JMP32 | BPF_JGE | BPF_K: in bpf_jit_emit_insn()
795 case BPF_JMP | BPF_JLE | BPF_K: in bpf_jit_emit_insn()
796 case BPF_JMP32 | BPF_JLE | BPF_K: in bpf_jit_emit_insn()
797 case BPF_JMP | BPF_JNE | BPF_K: in bpf_jit_emit_insn()
798 case BPF_JMP32 | BPF_JNE | BPF_K: in bpf_jit_emit_insn()
799 case BPF_JMP | BPF_JSGT | BPF_K: in bpf_jit_emit_insn()
800 case BPF_JMP32 | BPF_JSGT | BPF_K: in bpf_jit_emit_insn()
801 case BPF_JMP | BPF_JSLT | BPF_K: in bpf_jit_emit_insn()
802 case BPF_JMP32 | BPF_JSLT | BPF_K: in bpf_jit_emit_insn()
803 case BPF_JMP | BPF_JSGE | BPF_K: in bpf_jit_emit_insn()
804 case BPF_JMP32 | BPF_JSGE | BPF_K: in bpf_jit_emit_insn()
805 case BPF_JMP | BPF_JSLE | BPF_K: in bpf_jit_emit_insn()
806 case BPF_JMP32 | BPF_JSLE | BPF_K: in bpf_jit_emit_insn()
808 s = ctx->ninsns; in bpf_jit_emit_insn()
822 e = ctx->ninsns; in bpf_jit_emit_insn()
825 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
829 case BPF_JMP | BPF_JSET | BPF_K: in bpf_jit_emit_insn()
830 case BPF_JMP32 | BPF_JSET | BPF_K: in bpf_jit_emit_insn()
832 s = ctx->ninsns; in bpf_jit_emit_insn()
840 * sign-extension is sufficient here and saves one instruction, in bpf_jit_emit_insn()
845 e = ctx->ninsns; in bpf_jit_emit_insn()
846 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
851 case BPF_JMP | BPF_CALL: in bpf_jit_emit_insn()
857 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, in bpf_jit_emit_insn()
867 case BPF_JMP | BPF_TAIL_CALL: in bpf_jit_emit_insn()
869 return -1; in bpf_jit_emit_insn()
873 case BPF_JMP | BPF_EXIT: in bpf_jit_emit_insn()
874 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
884 case BPF_LD | BPF_IMM | BPF_DW: in bpf_jit_emit_insn()
895 case BPF_LDX | BPF_MEM | BPF_B: in bpf_jit_emit_insn()
907 case BPF_LDX | BPF_MEM | BPF_H: in bpf_jit_emit_insn()
919 case BPF_LDX | BPF_MEM | BPF_W: in bpf_jit_emit_insn()
931 case BPF_LDX | BPF_MEM | BPF_DW: in bpf_jit_emit_insn()
943 case BPF_ST | BPF_NOSPEC: in bpf_jit_emit_insn()
947 case BPF_ST | BPF_MEM | BPF_B: in bpf_jit_emit_insn()
959 case BPF_ST | BPF_MEM | BPF_H: in bpf_jit_emit_insn()
970 case BPF_ST | BPF_MEM | BPF_W: in bpf_jit_emit_insn()
981 case BPF_ST | BPF_MEM | BPF_DW: in bpf_jit_emit_insn()
994 case BPF_STX | BPF_MEM | BPF_B: in bpf_jit_emit_insn()
1004 case BPF_STX | BPF_MEM | BPF_H: in bpf_jit_emit_insn()
1014 case BPF_STX | BPF_MEM | BPF_W: in bpf_jit_emit_insn()
1024 case BPF_STX | BPF_MEM | BPF_DW: in bpf_jit_emit_insn()
1034 case BPF_STX | BPF_ATOMIC | BPF_W: in bpf_jit_emit_insn()
1035 case BPF_STX | BPF_ATOMIC | BPF_DW: in bpf_jit_emit_insn()
1036 if (insn->imm != BPF_ADD) { in bpf_jit_emit_insn()
1037 pr_err("bpf-jit: not supported: atomic operation %02x ***\n", in bpf_jit_emit_insn()
1038 insn->imm); in bpf_jit_emit_insn()
1039 return -EINVAL; in bpf_jit_emit_insn()
1062 pr_err("bpf-jit: unknown opcode %02x\n", code); in bpf_jit_emit_insn()
1063 return -EINVAL; in bpf_jit_emit_insn()
1073 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); in bpf_jit_build_prologue()
1096 store_offset = stack_adjust - 8; in bpf_jit_build_prologue()
1098 /* First instruction is always setting the tail-call-counter in bpf_jit_build_prologue()
1100 * Force using a 4-byte (non-compressed) instruction. in bpf_jit_build_prologue()
1104 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); in bpf_jit_build_prologue()
1108 store_offset -= 8; in bpf_jit_build_prologue()
1111 store_offset -= 8; in bpf_jit_build_prologue()
1114 store_offset -= 8; in bpf_jit_build_prologue()
1118 store_offset -= 8; in bpf_jit_build_prologue()
1122 store_offset -= 8; in bpf_jit_build_prologue()
1126 store_offset -= 8; in bpf_jit_build_prologue()
1130 store_offset -= 8; in bpf_jit_build_prologue()
1134 store_offset -= 8; in bpf_jit_build_prologue()
1148 ctx->stack_size = stack_adjust; in bpf_jit_build_prologue()