Lines Matching full:emit
231 * Emit an instruction that will be executed unconditionally.
233 static inline void emit(u32 inst, struct jit_ctx *ctx) in emit() function
417 emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx); in emit_mov_i_no8m()
419 emit(ARM_MOVW(rd, val & 0xffff), ctx); in emit_mov_i_no8m()
421 emit(ARM_MOVT(rd, val >> 16), ctx); in emit_mov_i_no8m()
430 emit(ARM_MOV_I(rd, imm12), ctx); in emit_mov_i()
438 emit(ARM_BX(tgt_reg), ctx); in emit_bx_r()
440 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); in emit_bx_r()
446 emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); in emit_blx_r()
449 emit(ARM_BLX_R(tgt_reg), ctx); in emit_blx_r()
472 emit(ARM_UDIV(rd, rm, rn), ctx); in emit_udivmod()
474 emit(ARM_UDIV(ARM_IP, rm, rn), ctx); in emit_udivmod()
475 emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx); in emit_udivmod()
490 emit(ARM_MOV_R(tmp[0], ARM_R1), ctx); in emit_udivmod()
491 emit(ARM_MOV_R(ARM_R1, rn), ctx); in emit_udivmod()
494 emit(ARM_MOV_R(tmp[1], ARM_R0), ctx); in emit_udivmod()
495 emit(ARM_MOV_R(ARM_R0, rm), ctx); in emit_udivmod()
505 emit(ARM_MOV_R(rd, ARM_R0), ctx); in emit_udivmod()
509 emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx); in emit_udivmod()
511 emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx); in emit_udivmod()
527 emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); in arm_bpf_get_reg32()
539 emit(ARM_LDRD_I(tmp[1], ARM_FP, in arm_bpf_get_reg64()
542 emit(ARM_LDR_I(tmp[1], ARM_FP, in arm_bpf_get_reg64()
544 emit(ARM_LDR_I(tmp[0], ARM_FP, in arm_bpf_get_reg64()
559 emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); in arm_bpf_put_reg32()
561 emit(ARM_MOV_R(reg, src), ctx); in arm_bpf_put_reg32()
570 emit(ARM_STRD_I(src[1], ARM_FP, in arm_bpf_put_reg64()
573 emit(ARM_STR_I(src[1], ARM_FP, in arm_bpf_put_reg64()
575 emit(ARM_STR_I(src[0], ARM_FP, in arm_bpf_put_reg64()
580 emit(ARM_MOV_R(reg[1], src[1]), ctx); in arm_bpf_put_reg64()
582 emit(ARM_MOV_R(reg[0], src[0]), ctx); in arm_bpf_put_reg64()
630 emit(ARM_ADDS_R(dst, dst, src), ctx); in emit_a32_add_r()
632 emit(ARM_ADC_R(dst, dst, src), ctx); in emit_a32_add_r()
634 emit(ARM_ADD_R(dst, dst, src), ctx); in emit_a32_add_r()
647 emit(ARM_SUBS_R(dst, dst, src), ctx); in emit_a32_sub_r()
649 emit(ARM_SBC_R(dst, dst, src), ctx); in emit_a32_sub_r()
651 emit(ARM_SUB_R(dst, dst, src), ctx); in emit_a32_sub_r()
667 emit(ARM_ORR_R(dst, dst, src), ctx); in emit_alu_r()
671 emit(ARM_AND_R(dst, dst, src), ctx); in emit_alu_r()
675 emit(ARM_EOR_R(dst, dst, src), ctx); in emit_alu_r()
679 emit(ARM_MUL(dst, dst, src), ctx); in emit_alu_r()
683 emit(ARM_LSL_R(dst, dst, src), ctx); in emit_alu_r()
687 emit(ARM_LSR_R(dst, dst, src), ctx); in emit_alu_r()
691 emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx); in emit_alu_r()
770 emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); in emit_a32_mov_r64()
771 emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); in emit_a32_mov_r64()
773 emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); in emit_a32_mov_r64()
775 emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); in emit_a32_mov_r64()
777 emit(ARM_MOV_R(dst[0], src[0]), ctx); in emit_a32_mov_r64()
778 emit(ARM_MOV_R(dst[1], src[1]), ctx); in emit_a32_mov_r64()
793 emit(ARM_LSL_I(rd, rd, val), ctx); in emit_a32_alu_i()
796 emit(ARM_LSR_I(rd, rd, val), ctx); in emit_a32_alu_i()
799 emit(ARM_ASR_I(rd, rd, val), ctx); in emit_a32_alu_i()
802 emit(ARM_RSB_I(rd, rd, val), ctx); in emit_a32_alu_i()
819 emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx); in emit_a32_neg64()
820 emit(ARM_RSC_I(rd[0], rd[0], 0), ctx); in emit_a32_neg64()
838 emit(ARM_SUB_I(ARM_IP, rt, 32), ctx); in emit_a32_lsh_r64()
839 emit(ARM_RSB_I(tmp2[0], rt, 32), ctx); in emit_a32_lsh_r64()
840 emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx); in emit_a32_lsh_r64()
841 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx); in emit_a32_lsh_r64()
842 emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx); in emit_a32_lsh_r64()
843 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx); in emit_a32_lsh_r64()
862 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); in emit_a32_arsh_r64()
863 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); in emit_a32_arsh_r64()
864 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); in emit_a32_arsh_r64()
865 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); in emit_a32_arsh_r64()
868 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx); in emit_a32_arsh_r64()
887 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); in emit_a32_rsh_r64()
888 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); in emit_a32_rsh_r64()
889 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); in emit_a32_rsh_r64()
890 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); in emit_a32_rsh_r64()
891 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx); in emit_a32_rsh_r64()
892 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx); in emit_a32_rsh_r64()
910 emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx); in emit_a32_lsh_i64()
911 emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx); in emit_a32_lsh_i64()
912 emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx); in emit_a32_lsh_i64()
915 emit(ARM_MOV_R(rd[0], rd[1]), ctx); in emit_a32_lsh_i64()
917 emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx); in emit_a32_lsh_i64()
918 emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx); in emit_a32_lsh_i64()
940 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); in emit_a32_rsh_i64()
941 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); in emit_a32_rsh_i64()
942 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx); in emit_a32_rsh_i64()
944 emit(ARM_MOV_R(rd[1], rd[0]), ctx); in emit_a32_rsh_i64()
945 emit(ARM_MOV_I(rd[0], 0), ctx); in emit_a32_rsh_i64()
947 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx); in emit_a32_rsh_i64()
948 emit(ARM_MOV_I(rd[0], 0), ctx); in emit_a32_rsh_i64()
970 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); in emit_a32_arsh_i64()
971 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); in emit_a32_arsh_i64()
972 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx); in emit_a32_arsh_i64()
974 emit(ARM_MOV_R(rd[1], rd[0]), ctx); in emit_a32_arsh_i64()
975 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); in emit_a32_arsh_i64()
977 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx); in emit_a32_arsh_i64()
978 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); in emit_a32_arsh_i64()
995 emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx); in emit_a32_mul_r64()
996 emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx); in emit_a32_mul_r64()
997 emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx); in emit_a32_mul_r64()
999 emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx); in emit_a32_mul_r64()
1000 emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx); in emit_a32_mul_r64()
1036 emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx); in emit_str_r()
1043 emit(ARM_STRB_I(src_lo, rd, off), ctx); in emit_str_r()
1047 emit(ARM_STRH_I(src_lo, rd, off), ctx); in emit_str_r()
1051 emit(ARM_STR_I(src_lo, rd, off), ctx); in emit_str_r()
1055 emit(ARM_STR_I(src_lo, rd, off), ctx); in emit_str_r()
1056 emit(ARM_STR_I(src_hi, rd, off + 4), ctx); in emit_str_r()
1070 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); in emit_ldx_r()
1074 emit(ARM_MOV_R(tmp[0], rm), ctx); in emit_ldx_r()
1080 emit(ARM_LDRB_I(rd[1], rm, off), ctx); in emit_ldx_r()
1086 emit(ARM_LDRH_I(rd[1], rm, off), ctx); in emit_ldx_r()
1092 emit(ARM_LDR_I(rd[1], rm, off), ctx); in emit_ldx_r()
1098 emit(ARM_LDR_I(rd[1], rm, off), ctx); in emit_ldx_r()
1099 emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); in emit_ldx_r()
1112 emit(ARM_AND_R(ARM_IP, rt, rn), ctx); in emit_ar_r()
1113 emit(ARM_AND_R(ARM_LR, rd, rm), ctx); in emit_ar_r()
1114 emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx); in emit_ar_r()
1116 emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx); in emit_ar_r()
1126 emit(ARM_CMP_R(rd, rm), ctx); in emit_ar_r()
1130 emit(ARM_CMP_R(rt, rn), ctx); in emit_ar_r()
1135 emit(ARM_CMP_R(rn, rt), ctx); in emit_ar_r()
1137 emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx); in emit_ar_r()
1141 emit(ARM_CMP_R(rt, rn), ctx); in emit_ar_r()
1143 emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx); in emit_ar_r()
1176 emit(ARM_LDR_I(tmp[1], r_array, off), ctx); in emit_bpf_tail_call()
1178 emit(ARM_CMP_R(r_index, tmp[1]), ctx); in emit_bpf_tail_call()
1190 emit(ARM_CMP_I(tc[0], hi), ctx); in emit_bpf_tail_call()
1193 emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx); in emit_bpf_tail_call()
1194 emit(ARM_ADC_I(tc[0], tc[0], 0), ctx); in emit_bpf_tail_call()
1203 emit(ARM_ADD_I(tmp[1], r_array, off), ctx); in emit_bpf_tail_call()
1204 emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); in emit_bpf_tail_call()
1205 emit(ARM_CMP_I(tmp[1], 0), ctx); in emit_bpf_tail_call()
1212 emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx); in emit_bpf_tail_call()
1213 emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx); in emit_bpf_tail_call()
1235 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); in emit_rev16()
1236 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx); in emit_rev16()
1237 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); in emit_rev16()
1238 emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx); in emit_rev16()
1240 emit(ARM_REV16(rd, rn), ctx); in emit_rev16()
1250 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); in emit_rev32()
1251 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx); in emit_rev32()
1252 emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx); in emit_rev32()
1254 emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx); in emit_rev32()
1255 emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx); in emit_rev32()
1256 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx); in emit_rev32()
1257 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); in emit_rev32()
1258 emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx); in emit_rev32()
1259 emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx); in emit_rev32()
1260 emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx); in emit_rev32()
1263 emit(ARM_REV(rd, rn), ctx); in emit_rev32()
1277 emit(ARM_PUSH(reg_set), ctx); in emit_push_r64()
1290 emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); in build_prologue()
1291 emit(ARM_PUSH(reg_set), ctx); in build_prologue()
1292 emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); in build_prologue()
1294 emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx); in build_prologue()
1295 emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx); in build_prologue()
1299 emit(ARM_MOV_I(bpf_r1[0], 0), ctx); in build_prologue()
1300 emit(ARM_SUB_I(bpf_r1[1], ARM_SP, SCRATCH_SIZE), ctx); in build_prologue()
1305 emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx); in build_prologue()
1311 emit(ARM_MOV_I(bpf_r1[1], 0), ctx); in build_prologue()
1315 emit(ARM_MOV_R(bpf_r1[1], arm_r0), ctx); in build_prologue()
1327 emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx); in build_epilogue()
1328 emit(ARM_LDM(ARM_SP, reg_set), ctx); in build_epilogue()
1331 emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx); in build_epilogue()
1332 emit(ARM_POP(CALLEE_POP_MASK), ctx); in build_epilogue()
1559 emit(ARM_MOV_R(rd[0], ARM_LR), ctx); in build_insn()
1569 emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx); in build_insn()
1571 emit(ARM_UXTH(rd[1], rd[1]), ctx); in build_insn()
1574 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); in build_insn()
1579 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); in build_insn()
1763 emit(ARM_B(jmp_offset), ctx); in build_insn()
1791 emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean in build_insn()
1803 emit(ARM_B(jmp_offset), ctx); in build_insn()