Lines Matching refs:nfp_prog
52 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \ argument
53 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
55 &(nfp_prog)->insns != &pos->l && \
56 &(nfp_prog)->insns != &next->l; \
60 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \ argument
61 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
64 &(nfp_prog)->insns != &pos->l && \
65 &(nfp_prog)->insns != &next->l && \
66 &(nfp_prog)->insns != &next2->l; \
72 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_meta_has_prev() argument
74 return meta->l.prev != &nfp_prog->insns; in nfp_meta_has_prev()
77 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) in nfp_prog_push() argument
79 if (nfp_prog->__prog_alloc_len / sizeof(u64) == nfp_prog->prog_len) { in nfp_prog_push()
81 nfp_prog->prog_len); in nfp_prog_push()
82 nfp_prog->error = -ENOSPC; in nfp_prog_push()
86 nfp_prog->prog[nfp_prog->prog_len] = insn; in nfp_prog_push()
87 nfp_prog->prog_len++; in nfp_prog_push()
90 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog) in nfp_prog_current_offset() argument
92 return nfp_prog->prog_len; in nfp_prog_current_offset()
96 nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off) in nfp_prog_confirm_current_offset() argument
102 if (nfp_prog->error) in nfp_prog_confirm_current_offset()
104 return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off); in nfp_prog_confirm_current_offset()
109 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, in __emit_cmd() argument
126 nfp_prog_push(nfp_prog, insn); in __emit_cmd()
130 emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, in emit_cmd_any() argument
138 nfp_prog->error = err; in emit_cmd_any()
143 nfp_prog->error = -EFAULT; in emit_cmd_any()
148 nfp_prog->error = -EFAULT; in emit_cmd_any()
152 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, ctx, in emit_cmd_any()
157 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, in emit_cmd() argument
160 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, false); in emit_cmd()
164 emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, in emit_cmd_indir() argument
167 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, true); in emit_cmd_indir()
171 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, in __emit_br() argument
188 nfp_prog_push(nfp_prog, insn); in __emit_br()
192 emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer, in emit_br_relo() argument
197 nfp_prog->error = -EFAULT; in emit_br_relo()
201 __emit_br(nfp_prog, mask, in emit_br_relo()
205 nfp_prog->prog[nfp_prog->prog_len - 1] |= in emit_br_relo()
210 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) in emit_br() argument
212 emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL); in emit_br()
216 __emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer, in __emit_br_bit() argument
234 nfp_prog_push(nfp_prog, insn); in __emit_br_bit()
238 emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, in emit_br_bit_relo() argument
252 nfp_prog->error = err; in emit_br_bit_relo()
256 __emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set, in emit_br_bit_relo()
259 nfp_prog->prog[nfp_prog->prog_len - 1] |= in emit_br_bit_relo()
264 emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer) in emit_br_bset() argument
266 emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL); in emit_br_bset()
270 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, in __emit_immed() argument
288 nfp_prog_push(nfp_prog, insn); in __emit_immed()
292 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm, in emit_immed() argument
299 nfp_prog->error = -EFAULT; in emit_immed()
305 nfp_prog->error = err; in emit_immed()
310 __emit_immed(nfp_prog, in emit_immed()
317 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, in __emit_shf() argument
325 nfp_prog->error = -EFAULT; in __emit_shf()
346 nfp_prog_push(nfp_prog, insn); in __emit_shf()
350 emit_shf(struct nfp_prog *nfp_prog, swreg dst, in emit_shf() argument
358 nfp_prog->error = err; in emit_shf()
362 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift, in emit_shf()
368 emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst, in emit_shf_indir() argument
373 nfp_prog->error = -EFAULT; in emit_shf_indir()
377 emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0); in emit_shf_indir()
381 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, in __emit_alu() argument
398 nfp_prog_push(nfp_prog, insn); in __emit_alu()
402 emit_alu(struct nfp_prog *nfp_prog, swreg dst, in emit_alu() argument
410 nfp_prog->error = err; in emit_alu()
414 __emit_alu(nfp_prog, reg.dst, reg.dst_ab, in emit_alu()
420 __emit_mul(struct nfp_prog *nfp_prog, enum alu_dst_ab dst_ab, u16 areg, in __emit_mul() argument
437 nfp_prog_push(nfp_prog, insn); in __emit_mul()
441 emit_mul(struct nfp_prog *nfp_prog, swreg lreg, enum mul_type type, in emit_mul() argument
449 nfp_prog->error = -EINVAL; in emit_mul()
465 nfp_prog->error = err; in emit_mul()
469 __emit_mul(nfp_prog, reg.dst_ab, areg, type, step, reg.breg, reg.swap, in emit_mul()
474 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc, in __emit_ld_field() argument
494 nfp_prog_push(nfp_prog, insn); in __emit_ld_field()
498 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, in emit_ld_field_any() argument
507 nfp_prog->error = err; in emit_ld_field_any()
511 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift, in emit_ld_field_any()
517 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, in emit_ld_field() argument
520 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false); in emit_ld_field()
524 __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr, in __emit_lcsr() argument
537 nfp_prog_push(nfp_prog, insn); in __emit_lcsr()
540 static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr) in emit_csr_wr() argument
557 nfp_prog->error = err; in emit_csr_wr()
561 __emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr, in emit_csr_wr()
566 static void __emit_csr_rd(struct nfp_prog *nfp_prog, u16 addr) in __emit_csr_rd() argument
568 __emit_lcsr(nfp_prog, 0, 0, false, addr, false, false); in __emit_csr_rd()
571 static void emit_nop(struct nfp_prog *nfp_prog) in emit_nop() argument
573 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0); in emit_nop()
595 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm) in wrp_immed() argument
601 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift); in wrp_immed()
603 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift); in wrp_immed()
605 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL, in wrp_immed()
607 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD, in wrp_immed()
613 wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm, in wrp_immed_relo() argument
618 nfp_prog->error = -EFAULT; in wrp_immed_relo()
621 emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B); in wrp_immed_relo()
623 nfp_prog->prog[nfp_prog->prog_len - 1] |= in wrp_immed_relo()
631 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) in ur_load_imm_any() argument
636 wrp_immed(nfp_prog, tmp_reg, imm); in ur_load_imm_any()
644 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) in re_load_imm_any() argument
649 wrp_immed(nfp_prog, tmp_reg, imm); in re_load_imm_any()
653 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) in wrp_nops() argument
656 emit_nop(nfp_prog); in wrp_nops()
659 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src) in wrp_mov() argument
661 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src); in wrp_mov()
664 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src) in wrp_reg_mov() argument
666 wrp_mov(nfp_prog, reg_both(dst), reg_b(src)); in wrp_reg_mov()
673 wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len, in wrp_reg_subpart() argument
679 emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true); in wrp_reg_subpart()
686 wrp_reg_or_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, in wrp_reg_or_subpart() argument
692 emit_ld_field(nfp_prog, dst, mask, src, sc, 32 - offset * 8); in wrp_reg_or_subpart()
696 addr40_offset(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, in addr40_offset() argument
705 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(src_gpr), ALU_OP_ADD, offset); in addr40_offset()
706 emit_alu(nfp_prog, imm_b(nfp_prog), reg_b(src_gpr + 1), ALU_OP_ADD_C, in addr40_offset()
708 *rega = imm_a(nfp_prog); in addr40_offset()
709 *regb = imm_b(nfp_prog); in addr40_offset()
713 static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_cpp_memcpy() argument
722 off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
728 addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base, in nfp_cpp_memcpy()
733 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
737 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, in nfp_cpp_memcpy()
743 wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i)); in nfp_cpp_memcpy()
745 off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
749 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
754 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
759 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
761 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
766 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
768 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
775 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
779 off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32, in nfp_cpp_memcpy()
780 imm_b(nfp_prog)); in nfp_cpp_memcpy()
781 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8, in nfp_cpp_memcpy()
790 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
792 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
796 off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
797 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, in nfp_cpp_memcpy()
821 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
826 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
830 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
834 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
836 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), in nfp_cpp_memcpy()
842 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in nfp_cpp_memcpy()
848 data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size) in data_ld() argument
859 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0, in data_ld()
860 pptr_reg(nfp_prog), offset, sz - 1, CMD_CTX_SWAP); in data_ld()
864 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE, in data_ld()
868 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); in data_ld()
871 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); in data_ld()
877 data_ld_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, in data_ld_host_order() argument
889 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, mode, 0, in data_ld_host_order()
894 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask, in data_ld_host_order()
898 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); in data_ld_host_order()
901 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); in data_ld_host_order()
907 data_ld_host_order_addr32(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, in data_ld_host_order_addr32() argument
910 return data_ld_host_order(nfp_prog, dst_gpr, reg_a(src_gpr), offset, in data_ld_host_order_addr32()
915 data_ld_host_order_addr40(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, in data_ld_host_order_addr40() argument
920 addr40_offset(nfp_prog, src_gpr, offset, ®a, ®b); in data_ld_host_order_addr40()
922 return data_ld_host_order(nfp_prog, dst_gpr, rega, regb, in data_ld_host_order_addr40()
927 construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size) in construct_data_ind_ld() argument
932 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); in construct_data_ind_ld()
933 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg); in construct_data_ind_ld()
936 emit_alu(nfp_prog, imm_a(nfp_prog), in construct_data_ind_ld()
937 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size)); in construct_data_ind_ld()
938 emit_alu(nfp_prog, reg_none(), in construct_data_ind_ld()
939 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog)); in construct_data_ind_ld()
940 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); in construct_data_ind_ld()
943 return data_ld(nfp_prog, imm_b(nfp_prog), 0, size); in construct_data_ind_ld()
946 static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size) in construct_data_ld() argument
951 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog)); in construct_data_ld()
952 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg); in construct_data_ld()
953 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); in construct_data_ld()
956 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); in construct_data_ld()
957 return data_ld(nfp_prog, tmp_reg, 0, size); in construct_data_ld()
961 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, in data_stx_host_order() argument
967 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i)); in data_stx_host_order()
969 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in data_stx_host_order()
976 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, in data_st_host_order() argument
979 wrp_immed(nfp_prog, reg_xfer(0), imm); in data_st_host_order()
981 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32); in data_st_host_order()
983 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in data_st_host_order()
990 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off,
995 wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off, in wrp_lmem_load() argument
1013 wrp_mov(nfp_prog, reg_both(dst), in wrp_lmem_load()
1045 reg = imm_a(nfp_prog); in wrp_lmem_load()
1052 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); in wrp_lmem_load()
1055 emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr); in wrp_lmem_load()
1058 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); in wrp_lmem_load()
1064 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off, in wrp_lmem_store() argument
1082 wrp_mov(nfp_prog, in wrp_lmem_store()
1115 reg = imm_a(nfp_prog); in wrp_lmem_store()
1120 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); in wrp_lmem_store()
1123 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf); in wrp_lmem_store()
1127 wrp_mov(nfp_prog, reg_lm(0, idx), reg); in wrp_lmem_store()
1129 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); in wrp_lmem_store()
1136 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_op_stack() argument
1140 s32 off = nfp_prog->stack_depth + meta->insn.off + ptr_off; in mem_op_stack()
1154 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off, in mem_op_stack()
1155 stack_imm(nfp_prog)); in mem_op_stack()
1157 emit_alu(nfp_prog, imm_b(nfp_prog), in mem_op_stack()
1171 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32), in mem_op_stack()
1172 stack_imm(nfp_prog)); in mem_op_stack()
1173 emit_alu(nfp_prog, imm_b(nfp_prog), in mem_op_stack()
1174 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); in mem_op_stack()
1178 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4), in mem_op_stack()
1179 stack_imm(nfp_prog)); in mem_op_stack()
1181 emit_alu(nfp_prog, imm_b(nfp_prog), in mem_op_stack()
1182 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); in mem_op_stack()
1187 emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3); in mem_op_stack()
1189 wrp_nops(nfp_prog, clr_gpr && size < 8 ? 2 : 3); in mem_op_stack()
1193 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); in mem_op_stack()
1208 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size, in mem_op_stack()
1230 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm) in wrp_alu_imm() argument
1236 wrp_immed(nfp_prog, reg_both(dst), 0); in wrp_alu_imm()
1242 wrp_immed(nfp_prog, reg_both(dst), ~0U); in wrp_alu_imm()
1248 emit_alu(nfp_prog, reg_both(dst), reg_none(), in wrp_alu_imm()
1254 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); in wrp_alu_imm()
1255 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg); in wrp_alu_imm()
1259 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu64_imm() argument
1270 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U); in wrp_alu64_imm()
1271 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32); in wrp_alu64_imm()
1277 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu64_reg() argument
1282 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); in wrp_alu64_reg()
1283 emit_alu(nfp_prog, reg_both(dst + 1), in wrp_alu64_reg()
1290 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu32_imm() argument
1300 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm); in wrp_alu32_imm()
1301 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); in wrp_alu32_imm()
1307 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu32_reg() argument
1312 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); in wrp_alu32_reg()
1313 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in wrp_alu32_reg()
1319 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src, in wrp_test_reg_one() argument
1322 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src)); in wrp_test_reg_one()
1323 emit_br(nfp_prog, br_mask, off, 0); in wrp_test_reg_one()
1327 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_test_reg() argument
1332 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op, in wrp_test_reg()
1334 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op, in wrp_test_reg()
1368 static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in cmp_imm() argument
1384 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in cmp_imm()
1386 emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg); in cmp_imm()
1388 emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg)); in cmp_imm()
1390 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); in cmp_imm()
1392 emit_alu(nfp_prog, reg_none(), in cmp_imm()
1395 emit_alu(nfp_prog, reg_none(), in cmp_imm()
1398 emit_br(nfp_prog, code->br_mask, insn->off, 0); in cmp_imm()
1403 static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in cmp_reg() argument
1422 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); in cmp_reg()
1423 emit_alu(nfp_prog, reg_none(), in cmp_reg()
1425 emit_br(nfp_prog, code->br_mask, insn->off, 0); in cmp_reg()
1430 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out) in wrp_end32() argument
1432 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in, in wrp_end32()
1434 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out), in wrp_end32()
1439 wrp_mul_u32(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg, in wrp_mul_u32() argument
1442 emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg); in wrp_mul_u32()
1443 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_1, rreg); in wrp_mul_u32()
1444 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_2, rreg); in wrp_mul_u32()
1445 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_3, rreg); in wrp_mul_u32()
1446 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_4, rreg); in wrp_mul_u32()
1447 emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_32x32, MUL_LAST, reg_none()); in wrp_mul_u32()
1449 emit_mul(nfp_prog, dst_hi, MUL_TYPE_STEP_32x32, MUL_LAST_2, in wrp_mul_u32()
1452 wrp_immed(nfp_prog, dst_hi, 0); in wrp_mul_u32()
1456 wrp_mul_u16(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg, in wrp_mul_u16() argument
1459 emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg); in wrp_mul_u16()
1460 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_1, rreg); in wrp_mul_u16()
1461 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_2, rreg); in wrp_mul_u16()
1462 emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_16x16, MUL_LAST, reg_none()); in wrp_mul_u16()
1466 wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_mul() argument
1485 multiplier = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); in wrp_mul()
1489 wrp_mul_u32(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier, in wrp_mul()
1492 wrp_mul_u16(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier); in wrp_mul()
1497 static int wrp_div_imm(struct nfp_prog *nfp_prog, u8 dst, u64 imm) in wrp_div_imm() argument
1505 wrp_immed(nfp_prog, dst_both, 0); in wrp_div_imm()
1521 swreg tmp_b = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); in wrp_div_imm()
1523 emit_alu(nfp_prog, reg_none(), dst_a, ALU_OP_SUB, tmp_b); in wrp_div_imm()
1524 wrp_immed(nfp_prog, imm_a(nfp_prog), 0); in wrp_div_imm()
1525 emit_alu(nfp_prog, dst_both, imm_a(nfp_prog), ALU_OP_ADD_C, in wrp_div_imm()
1538 magic = ur_load_imm_any(nfp_prog, rvalue.m, imm_b(nfp_prog)); in wrp_div_imm()
1540 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, in wrp_div_imm()
1543 wrp_mul_u32(nfp_prog, imm_both(nfp_prog), reg_none(), dst_a, in wrp_div_imm()
1545 emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_SUB, in wrp_div_imm()
1546 imm_b(nfp_prog)); in wrp_div_imm()
1547 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, in wrp_div_imm()
1549 emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_ADD, in wrp_div_imm()
1550 imm_b(nfp_prog)); in wrp_div_imm()
1551 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, in wrp_div_imm()
1555 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, in wrp_div_imm()
1557 wrp_mul_u32(nfp_prog, dst_both, reg_none(), dst_a, magic, true); in wrp_div_imm()
1558 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, in wrp_div_imm()
1565 static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in adjust_head() argument
1567 swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog); in adjust_head()
1571 adjust_head = &nfp_prog->bpf->adjust_head; in adjust_head()
1574 if (nfp_prog->adjust_head_location != UINT_MAX) { in adjust_head()
1575 if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n)) in adjust_head()
1578 emit_alu(nfp_prog, pptr_reg(nfp_prog), in adjust_head()
1579 reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog)); in adjust_head()
1580 emit_alu(nfp_prog, plen_reg(nfp_prog), in adjust_head()
1581 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1582 emit_alu(nfp_prog, pv_len(nfp_prog), in adjust_head()
1583 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1585 wrp_immed(nfp_prog, reg_both(0), 0); in adjust_head()
1586 wrp_immed(nfp_prog, reg_both(1), 0); in adjust_head()
1595 ret_einval = nfp_prog_current_offset(nfp_prog) + 14; in adjust_head()
1599 emit_alu(nfp_prog, tmp, in adjust_head()
1600 reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog)); in adjust_head()
1603 emit_alu(nfp_prog, reg_none(), in adjust_head()
1605 emit_br(nfp_prog, BR_BLO, ret_einval, 0); in adjust_head()
1606 emit_alu(nfp_prog, reg_none(), in adjust_head()
1608 emit_br(nfp_prog, BR_BLO, ret_einval, 0); in adjust_head()
1611 emit_alu(nfp_prog, tmp_len, in adjust_head()
1612 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1613 emit_alu(nfp_prog, reg_none(), in adjust_head()
1615 emit_br(nfp_prog, BR_BMI, ret_einval, 0); in adjust_head()
1618 wrp_immed(nfp_prog, reg_both(0), 0); in adjust_head()
1619 wrp_immed(nfp_prog, reg_both(1), 0); in adjust_head()
1622 emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0); in adjust_head()
1625 emit_br(nfp_prog, BR_UNC, end, 2); in adjust_head()
1627 emit_alu(nfp_prog, plen_reg(nfp_prog), in adjust_head()
1628 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1629 emit_alu(nfp_prog, pv_len(nfp_prog), in adjust_head()
1630 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1633 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval)) in adjust_head()
1636 wrp_immed(nfp_prog, reg_both(0), -22); in adjust_head()
1637 wrp_immed(nfp_prog, reg_both(1), ~0); in adjust_head()
1639 if (!nfp_prog_confirm_current_offset(nfp_prog, end)) in adjust_head()
1645 static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in adjust_tail() argument
1650 BUILD_BUG_ON(plen_reg(nfp_prog) != reg_b(STATIC_REG_PKT_LEN)); in adjust_tail()
1652 plen = imm_a(nfp_prog); in adjust_tail()
1655 ret_einval = nfp_prog_current_offset(nfp_prog) + 9; in adjust_tail()
1656 end = nfp_prog_current_offset(nfp_prog) + 11; in adjust_tail()
1659 emit_alu(nfp_prog, plen, plen_reg(nfp_prog), ALU_OP_ADD, delta); in adjust_tail()
1663 emit_br(nfp_prog, BR_BCC, ret_einval, 0); in adjust_tail()
1666 emit_alu(nfp_prog, reg_none(), plen, ALU_OP_SUB, reg_imm(ETH_HLEN)); in adjust_tail()
1667 emit_br(nfp_prog, BR_BMI, ret_einval, 0); in adjust_tail()
1669 emit_alu(nfp_prog, plen_reg(nfp_prog), in adjust_tail()
1670 plen_reg(nfp_prog), ALU_OP_ADD, delta); in adjust_tail()
1671 emit_alu(nfp_prog, pv_len(nfp_prog), in adjust_tail()
1672 pv_len(nfp_prog), ALU_OP_ADD, delta); in adjust_tail()
1674 emit_br(nfp_prog, BR_UNC, end, 2); in adjust_tail()
1675 wrp_immed(nfp_prog, reg_both(0), 0); in adjust_tail()
1676 wrp_immed(nfp_prog, reg_both(1), 0); in adjust_tail()
1678 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval)) in adjust_tail()
1681 wrp_immed(nfp_prog, reg_both(0), -22); in adjust_tail()
1682 wrp_immed(nfp_prog, reg_both(1), ~0); in adjust_tail()
1684 if (!nfp_prog_confirm_current_offset(nfp_prog, end)) in adjust_tail()
1691 map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in map_call_stack_common() argument
1698 lm_off = nfp_prog->stack_depth; in map_call_stack_common()
1704 emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0); in map_call_stack_common()
1706 emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2); in map_call_stack_common()
1708 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, in map_call_stack_common()
1710 ret_tgt = nfp_prog_current_offset(nfp_prog) + 2; in map_call_stack_common()
1713 wrp_mov(nfp_prog, reg_a(0), reg_a(2)); in map_call_stack_common()
1716 wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); in map_call_stack_common()
1718 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) in map_call_stack_common()
1725 emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0); in map_call_stack_common()
1726 wrp_nops(nfp_prog, 3); in map_call_stack_common()
1732 nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_get_prandom_u32() argument
1734 __emit_csr_rd(nfp_prog, NFP_CSR_PSEUDO_RND_NUM); in nfp_get_prandom_u32()
1736 emit_immed(nfp_prog, reg_both(0), 0, in nfp_get_prandom_u32()
1738 emit_immed(nfp_prog, reg_both(1), 0, in nfp_get_prandom_u32()
1744 nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_perf_event_output() argument
1749 ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog)); in nfp_perf_event_output()
1751 ret_tgt = nfp_prog_current_offset(nfp_prog) + 3; in nfp_perf_event_output()
1753 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, in nfp_perf_event_output()
1757 wrp_mov(nfp_prog, reg_a(1), ptr_type); in nfp_perf_event_output()
1760 wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); in nfp_perf_event_output()
1762 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) in nfp_perf_event_output()
1769 nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_queue_select() argument
1773 jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5; in nfp_queue_select()
1776 emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2), in nfp_queue_select()
1778 emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2); in nfp_queue_select()
1781 emit_shf(nfp_prog, pv_qsel_set(nfp_prog), in nfp_queue_select()
1782 pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1), in nfp_queue_select()
1784 emit_ld_field(nfp_prog, in nfp_queue_select()
1785 pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2), in nfp_queue_select()
1790 emit_ld_field(nfp_prog, in nfp_queue_select()
1791 pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX), in nfp_queue_select()
1794 if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt)) in nfp_queue_select()
1801 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_reg64() argument
1810 stack_depth_reg = ur_load_imm_any(nfp_prog, in mov_reg64()
1811 nfp_prog->stack_depth, in mov_reg64()
1812 stack_imm(nfp_prog)); in mov_reg64()
1813 emit_alu(nfp_prog, reg_both(dst), in mov_reg64()
1814 stack_reg(nfp_prog), ALU_OP_ADD, stack_depth_reg); in mov_reg64()
1815 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in mov_reg64()
1817 wrp_reg_mov(nfp_prog, dst, src); in mov_reg64()
1818 wrp_reg_mov(nfp_prog, dst + 1, src + 1); in mov_reg64()
1824 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_imm64() argument
1828 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); in mov_imm64()
1829 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); in mov_imm64()
1834 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_reg64() argument
1836 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); in xor_reg64()
1839 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_imm64() argument
1841 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); in xor_imm64()
1844 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_reg64() argument
1846 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); in and_reg64()
1849 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_imm64() argument
1851 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); in and_imm64()
1854 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_reg64() argument
1856 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); in or_reg64()
1859 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_imm64() argument
1861 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); in or_imm64()
1864 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_reg64() argument
1868 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), in add_reg64()
1871 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), in add_reg64()
1878 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_imm64() argument
1883 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U); in add_imm64()
1884 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32); in add_imm64()
1889 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_reg64() argument
1893 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), in sub_reg64()
1896 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), in sub_reg64()
1903 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_imm64() argument
1908 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U); in sub_imm64()
1909 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32); in sub_imm64()
1914 static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_reg64() argument
1916 return wrp_mul(nfp_prog, meta, true, true); in mul_reg64()
1919 static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_imm64() argument
1921 return wrp_mul(nfp_prog, meta, true, false); in mul_imm64()
1924 static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_imm64() argument
1928 return wrp_div_imm(nfp_prog, insn->dst_reg * 2, insn->imm); in div_imm64()
1931 static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_reg64() argument
1936 return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src); in div_reg64()
1939 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in neg_reg64() argument
1943 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0), in neg_reg64()
1945 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0), in neg_reg64()
1961 static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) in __shl_imm64() argument
1964 emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), in __shl_imm64()
1967 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in __shl_imm64()
1970 wrp_reg_mov(nfp_prog, dst + 1, dst); in __shl_imm64()
1971 wrp_immed(nfp_prog, reg_both(dst), 0); in __shl_imm64()
1973 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in __shl_imm64()
1975 wrp_immed(nfp_prog, reg_both(dst), 0); in __shl_imm64()
1981 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_imm64() argument
1986 return __shl_imm64(nfp_prog, dst, insn->imm); in shl_imm64()
1989 static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_lt32_high() argument
1991 emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB, in shl_reg64_lt32_high()
1993 emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0)); in shl_reg64_lt32_high()
1994 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE, in shl_reg64_lt32_high()
1999 static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_lt32_low() argument
2001 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shl_reg64_lt32_low()
2002 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in shl_reg64_lt32_low()
2006 static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_lt32() argument
2008 shl_reg64_lt32_high(nfp_prog, dst, src); in shl_reg64_lt32()
2009 shl_reg64_lt32_low(nfp_prog, dst, src); in shl_reg64_lt32()
2012 static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_ge32() argument
2014 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shl_reg64_ge32()
2015 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in shl_reg64_ge32()
2017 wrp_immed(nfp_prog, reg_both(dst), 0); in shl_reg64_ge32()
2020 static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_reg64() argument
2030 return __shl_imm64(nfp_prog, dst, umin); in shl_reg64()
2034 shl_reg64_lt32(nfp_prog, dst, src); in shl_reg64()
2036 shl_reg64_ge32(nfp_prog, dst, src); in shl_reg64()
2043 label_ge32 = nfp_prog_current_offset(nfp_prog) + 7; in shl_reg64()
2044 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); in shl_reg64()
2046 shl_reg64_lt32_high(nfp_prog, dst, src); in shl_reg64()
2047 label_end = nfp_prog_current_offset(nfp_prog) + 6; in shl_reg64()
2048 emit_br(nfp_prog, BR_UNC, label_end, 2); in shl_reg64()
2050 shl_reg64_lt32_low(nfp_prog, dst, src); in shl_reg64()
2052 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) in shl_reg64()
2054 shl_reg64_ge32(nfp_prog, dst, src); in shl_reg64()
2056 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) in shl_reg64()
2073 static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) in __shr_imm64() argument
2076 emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, in __shr_imm64()
2078 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in __shr_imm64()
2081 wrp_reg_mov(nfp_prog, dst, dst + 1); in __shr_imm64()
2082 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in __shr_imm64()
2084 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in __shr_imm64()
2086 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in __shr_imm64()
2092 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_imm64() argument
2097 return __shr_imm64(nfp_prog, dst, insn->imm); in shr_imm64()
2101 static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_lt32_high() argument
2103 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shr_reg64_lt32_high()
2104 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in shr_reg64_lt32_high()
2108 static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_lt32_low() argument
2110 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shr_reg64_lt32_low()
2111 emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, in shr_reg64_lt32_low()
2115 static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_lt32() argument
2117 shr_reg64_lt32_low(nfp_prog, dst, src); in shr_reg64_lt32()
2118 shr_reg64_lt32_high(nfp_prog, dst, src); in shr_reg64_lt32()
2121 static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_ge32() argument
2123 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shr_reg64_ge32()
2124 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in shr_reg64_ge32()
2126 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in shr_reg64_ge32()
2129 static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_reg64() argument
2139 return __shr_imm64(nfp_prog, dst, umin); in shr_reg64()
2143 shr_reg64_lt32(nfp_prog, dst, src); in shr_reg64()
2145 shr_reg64_ge32(nfp_prog, dst, src); in shr_reg64()
2152 label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; in shr_reg64()
2153 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); in shr_reg64()
2154 shr_reg64_lt32_low(nfp_prog, dst, src); in shr_reg64()
2155 label_end = nfp_prog_current_offset(nfp_prog) + 6; in shr_reg64()
2156 emit_br(nfp_prog, BR_UNC, label_end, 2); in shr_reg64()
2158 shr_reg64_lt32_high(nfp_prog, dst, src); in shr_reg64()
2160 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) in shr_reg64()
2162 shr_reg64_ge32(nfp_prog, dst, src); in shr_reg64()
2164 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) in shr_reg64()
2174 static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) in __ashr_imm64() argument
2177 emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, in __ashr_imm64()
2180 emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, in __ashr_imm64()
2182 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2186 wrp_reg_mov(nfp_prog, dst, dst + 1); in __ashr_imm64()
2187 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2190 emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, in __ashr_imm64()
2192 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2194 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2201 static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_imm64() argument
2206 return __ashr_imm64(nfp_prog, dst, insn->imm); in ashr_imm64()
2209 static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_lt32_high() argument
2214 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); in ashr_reg64_lt32_high()
2215 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in ashr_reg64_lt32_high()
2219 static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_lt32_low() argument
2224 return shr_reg64_lt32_low(nfp_prog, dst, src); in ashr_reg64_lt32_low()
2227 static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_lt32() argument
2229 ashr_reg64_lt32_low(nfp_prog, dst, src); in ashr_reg64_lt32()
2230 ashr_reg64_lt32_high(nfp_prog, dst, src); in ashr_reg64_lt32()
2233 static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_ge32() argument
2235 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); in ashr_reg64_ge32()
2236 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, in ashr_reg64_ge32()
2238 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in ashr_reg64_ge32()
2243 static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_reg64() argument
2253 return __ashr_imm64(nfp_prog, dst, umin); in ashr_reg64()
2257 ashr_reg64_lt32(nfp_prog, dst, src); in ashr_reg64()
2259 ashr_reg64_ge32(nfp_prog, dst, src); in ashr_reg64()
2263 label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; in ashr_reg64()
2264 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); in ashr_reg64()
2265 ashr_reg64_lt32_low(nfp_prog, dst, src); in ashr_reg64()
2266 label_end = nfp_prog_current_offset(nfp_prog) + 6; in ashr_reg64()
2267 emit_br(nfp_prog, BR_UNC, label_end, 2); in ashr_reg64()
2269 ashr_reg64_lt32_high(nfp_prog, dst, src); in ashr_reg64()
2271 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) in ashr_reg64()
2273 ashr_reg64_ge32(nfp_prog, dst, src); in ashr_reg64()
2275 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) in ashr_reg64()
2282 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_reg() argument
2286 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); in mov_reg()
2287 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); in mov_reg()
2292 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_imm() argument
2296 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); in mov_imm()
2297 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); in mov_imm()
2302 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_reg() argument
2304 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); in xor_reg()
2307 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_imm() argument
2309 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm); in xor_imm()
2312 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_reg() argument
2314 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); in and_reg()
2317 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_imm() argument
2319 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); in and_imm()
2322 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_reg() argument
2324 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); in or_reg()
2327 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_imm() argument
2329 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); in or_imm()
2332 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_reg() argument
2334 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); in add_reg()
2337 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_imm() argument
2339 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm); in add_imm()
2342 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_reg() argument
2344 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); in sub_reg()
2347 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_imm() argument
2349 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm); in sub_imm()
2352 static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_reg() argument
2354 return wrp_mul(nfp_prog, meta, false, true); in mul_reg()
2357 static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_imm() argument
2359 return wrp_mul(nfp_prog, meta, false, false); in mul_imm()
2362 static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_reg() argument
2364 return div_reg64(nfp_prog, meta); in div_reg()
2367 static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_imm() argument
2369 return div_imm64(nfp_prog, meta); in div_imm()
2372 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in neg_reg() argument
2376 emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst)); in neg_reg()
2377 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in neg_reg()
2382 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_imm() argument
2389 emit_shf(nfp_prog, reg_both(insn->dst_reg * 2), in shl_imm()
2392 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); in shl_imm()
2397 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in end_reg32() argument
2404 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr), in end_reg32()
2406 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr), in end_reg32()
2409 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); in end_reg32()
2412 wrp_end32(nfp_prog, reg_a(gpr), gpr); in end_reg32()
2413 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); in end_reg32()
2416 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1)); in end_reg32()
2418 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1); in end_reg32()
2419 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr); in end_reg32()
2426 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in imm_ld8_part2() argument
2436 wrp_immed(nfp_prog, reg_both(dst), imm_lo); in imm_ld8_part2()
2440 wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst)); in imm_ld8_part2()
2442 wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi); in imm_ld8_part2()
2447 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in imm_ld8() argument
2453 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld1() argument
2455 return construct_data_ld(nfp_prog, meta->insn.imm, 1); in data_ld1()
2458 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld2() argument
2460 return construct_data_ld(nfp_prog, meta->insn.imm, 2); in data_ld2()
2463 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld4() argument
2465 return construct_data_ld(nfp_prog, meta->insn.imm, 4); in data_ld4()
2468 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld1() argument
2470 return construct_data_ind_ld(nfp_prog, meta->insn.imm, in data_ind_ld1()
2474 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld2() argument
2476 return construct_data_ind_ld(nfp_prog, meta->insn.imm, in data_ind_ld2()
2480 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld4() argument
2482 return construct_data_ind_ld(nfp_prog, meta->insn.imm, in data_ind_ld4()
2487 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_stack() argument
2490 return mem_op_stack(nfp_prog, meta, size, ptr_off, in mem_ldx_stack()
2495 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_skb() argument
2504 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); in mem_ldx_skb()
2509 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); in mem_ldx_skb()
2514 emit_alu(nfp_prog, dst, in mem_ldx_skb()
2515 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); in mem_ldx_skb()
2521 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in mem_ldx_skb()
2526 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_xdp() argument
2535 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); in mem_ldx_xdp()
2540 emit_alu(nfp_prog, dst, in mem_ldx_xdp()
2541 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); in mem_ldx_xdp()
2547 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in mem_ldx_xdp()
2553 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_data() argument
2558 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_ldx_data()
2560 return data_ld_host_order_addr32(nfp_prog, meta->insn.src_reg * 2, in mem_ldx_data()
2565 mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_emem() argument
2570 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_ldx_emem()
2572 return data_ld_host_order_addr40(nfp_prog, meta->insn.src_reg * 2, in mem_ldx_emem()
2577 mem_ldx_data_init_pktcache(struct nfp_prog *nfp_prog, in mem_ldx_data_init_pktcache() argument
2586 off = re_load_imm_any(nfp_prog, range_start, imm_b(nfp_prog)); in mem_ldx_data_init_pktcache()
2594 wrp_immed(nfp_prog, reg_none(), in mem_ldx_data_init_pktcache()
2598 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base, in mem_ldx_data_init_pktcache()
2603 mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog, in mem_ldx_data_from_pktcache_unaligned() argument
2630 wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off); in mem_ldx_data_from_pktcache_unaligned()
2633 wrp_immed(nfp_prog, dst_hi, 0); in mem_ldx_data_from_pktcache_unaligned()
2640 wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo); in mem_ldx_data_from_pktcache_unaligned()
2641 wrp_immed(nfp_prog, dst_hi, 0); in mem_ldx_data_from_pktcache_unaligned()
2645 wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, in mem_ldx_data_from_pktcache_unaligned()
2647 wrp_reg_subpart(nfp_prog, dst_hi, src_mid, len_lo, in mem_ldx_data_from_pktcache_unaligned()
2649 wrp_reg_or_subpart(nfp_prog, dst_hi, src_hi, REG_WIDTH - len_lo, in mem_ldx_data_from_pktcache_unaligned()
2657 mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog, in mem_ldx_data_from_pktcache_aligned() argument
2671 wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0); in mem_ldx_data_from_pktcache_aligned()
2672 wrp_immed(nfp_prog, dst_hi, 0); in mem_ldx_data_from_pktcache_aligned()
2674 wrp_mov(nfp_prog, dst_lo, src_lo); in mem_ldx_data_from_pktcache_aligned()
2675 wrp_immed(nfp_prog, dst_hi, 0); in mem_ldx_data_from_pktcache_aligned()
2679 wrp_mov(nfp_prog, dst_lo, src_lo); in mem_ldx_data_from_pktcache_aligned()
2680 wrp_mov(nfp_prog, dst_hi, src_hi); in mem_ldx_data_from_pktcache_aligned()
2687 mem_ldx_data_from_pktcache(struct nfp_prog *nfp_prog, in mem_ldx_data_from_pktcache() argument
2693 return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size); in mem_ldx_data_from_pktcache()
2695 return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size); in mem_ldx_data_from_pktcache()
2699 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx() argument
2703 return nfp_cpp_memcpy(nfp_prog, meta); in mem_ldx()
2706 if (nfp_prog->type == BPF_PROG_TYPE_XDP) in mem_ldx()
2707 return mem_ldx_xdp(nfp_prog, meta, size); in mem_ldx()
2709 return mem_ldx_skb(nfp_prog, meta, size); in mem_ldx()
2715 mem_ldx_data_init_pktcache(nfp_prog, meta); in mem_ldx()
2717 return mem_ldx_data_from_pktcache(nfp_prog, meta, size); in mem_ldx()
2719 return mem_ldx_data(nfp_prog, meta, size); in mem_ldx()
2724 return mem_ldx_stack(nfp_prog, meta, size, in mem_ldx()
2728 return mem_ldx_emem(nfp_prog, meta, size); in mem_ldx()
2733 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx1() argument
2735 return mem_ldx(nfp_prog, meta, 1); in mem_ldx1()
2738 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx2() argument
2740 return mem_ldx(nfp_prog, meta, 2); in mem_ldx2()
2743 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx4() argument
2745 return mem_ldx(nfp_prog, meta, 4); in mem_ldx4()
2748 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx8() argument
2750 return mem_ldx(nfp_prog, meta, 8); in mem_ldx8()
2754 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_st_data() argument
2760 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_st_data()
2762 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, in mem_st_data()
2766 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_st() argument
2770 return mem_st_data(nfp_prog, meta, size); in mem_st()
2775 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st1() argument
2777 return mem_st(nfp_prog, meta, 1); in mem_st1()
2780 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st2() argument
2782 return mem_st(nfp_prog, meta, 2); in mem_st2()
2785 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st4() argument
2787 return mem_st(nfp_prog, meta, 4); in mem_st4()
2790 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st8() argument
2792 return mem_st(nfp_prog, meta, 8); in mem_st8()
2796 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx_data() argument
2801 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_stx_data()
2803 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, in mem_stx_data()
2808 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx_stack() argument
2811 return mem_op_stack(nfp_prog, meta, size, ptr_off, in mem_stx_stack()
2816 static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx_xdp() argument
2820 return nfp_queue_select(nfp_prog, meta); in mem_stx_xdp()
2828 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx() argument
2832 return mem_stx_data(nfp_prog, meta, size); in mem_stx()
2835 return mem_stx_stack(nfp_prog, meta, size, in mem_stx()
2841 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx1() argument
2843 return mem_stx(nfp_prog, meta, 1); in mem_stx1()
2846 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx2() argument
2848 return mem_stx(nfp_prog, meta, 2); in mem_stx2()
2851 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx4() argument
2854 if (nfp_prog->type == BPF_PROG_TYPE_XDP) in mem_stx4()
2855 return mem_stx_xdp(nfp_prog, meta); in mem_stx4()
2856 return mem_stx(nfp_prog, meta, 4); in mem_stx4()
2859 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx8() argument
2861 return mem_stx(nfp_prog, meta, 8); in mem_stx8()
2865 mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64) in mem_xadd() argument
2872 off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_xadd()
2879 out = nfp_prog_current_offset(nfp_prog); in mem_xadd()
2880 full_add = nfp_prog_current_offset(nfp_prog); in mem_xadd()
2899 swreg max_imm = imm_a(nfp_prog); in mem_xadd()
2901 wrp_immed(nfp_prog, max_imm, 0xffff); in mem_xadd()
2902 emit_alu(nfp_prog, reg_none(), in mem_xadd()
2904 emit_alu(nfp_prog, reg_none(), in mem_xadd()
2906 emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0); in mem_xadd()
2915 emit_alu(nfp_prog, imma_a(nfp_prog), in mem_xadd()
2917 emit_alu(nfp_prog, imma_b(nfp_prog), in mem_xadd()
2919 addra = imma_a(nfp_prog); in mem_xadd()
2920 addrb = imma_b(nfp_prog); in mem_xadd()
2925 swreg prev_alu = imm_a(nfp_prog); in mem_xadd()
2927 wrp_immed(nfp_prog, prev_alu, in mem_xadd()
2931 wrp_reg_or_subpart(nfp_prog, prev_alu, reg_b(src_gpr), 2, 2); in mem_xadd()
2932 emit_cmd_indir(nfp_prog, CMD_TGT_ADD_IMM, CMD_MODE_40b_BA, 0, in mem_xadd()
2936 emit_br(nfp_prog, BR_UNC, out, 0); in mem_xadd()
2939 if (!nfp_prog_confirm_current_offset(nfp_prog, full_add)) in mem_xadd()
2944 emit_cmd(nfp_prog, CMD_TGT_ADD, CMD_MODE_40b_BA, 0, in mem_xadd()
2948 wrp_mov(nfp_prog, reg_xfer(0), reg_a(src_gpr)); in mem_xadd()
2950 wrp_mov(nfp_prog, reg_xfer(1), reg_a(src_gpr + 1)); in mem_xadd()
2953 if (!nfp_prog_confirm_current_offset(nfp_prog, out)) in mem_xadd()
2959 static int mem_xadd4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_xadd4() argument
2961 return mem_xadd(nfp_prog, meta, false); in mem_xadd4()
2964 static int mem_xadd8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_xadd8() argument
2966 return mem_xadd(nfp_prog, meta, true); in mem_xadd8()
2969 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jump() argument
2971 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); in jump()
2976 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jeq_imm() argument
2986 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in jeq_imm()
2987 emit_alu(nfp_prog, imm_a(nfp_prog), in jeq_imm()
2989 or1 = imm_a(nfp_prog); in jeq_imm()
2993 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); in jeq_imm()
2994 emit_alu(nfp_prog, imm_b(nfp_prog), in jeq_imm()
2996 or2 = imm_b(nfp_prog); in jeq_imm()
2999 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2); in jeq_imm()
3000 emit_br(nfp_prog, BR_BEQ, insn->off, 0); in jeq_imm()
3005 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jset_imm() argument
3017 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in jset_imm()
3018 emit_alu(nfp_prog, reg_none(), in jset_imm()
3020 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jset_imm()
3024 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); in jset_imm()
3025 emit_alu(nfp_prog, reg_none(), in jset_imm()
3027 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jset_imm()
3033 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jne_imm() argument
3040 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2), in jne_imm()
3042 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jne_imm()
3046 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in jne_imm()
3047 emit_alu(nfp_prog, reg_none(), in jne_imm()
3049 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jne_imm()
3051 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); in jne_imm()
3052 emit_alu(nfp_prog, reg_none(), in jne_imm()
3054 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jne_imm()
3059 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jeq_reg() argument
3063 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2), in jeq_reg()
3065 emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1), in jeq_reg()
3067 emit_alu(nfp_prog, reg_none(), in jeq_reg()
3068 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog)); in jeq_reg()
3069 emit_br(nfp_prog, BR_BEQ, insn->off, 0); in jeq_reg()
3074 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jset_reg() argument
3076 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); in jset_reg()
3079 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jne_reg() argument
3081 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); in jne_reg()
3084 static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in call() argument
3088 return adjust_head(nfp_prog, meta); in call()
3090 return adjust_tail(nfp_prog, meta); in call()
3094 return map_call_stack_common(nfp_prog, meta); in call()
3096 return nfp_get_prandom_u32(nfp_prog, meta); in call()
3098 return nfp_perf_event_output(nfp_prog, meta); in call()
3105 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in goto_out() argument
3107 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT); in goto_out()
3204 static int nfp_fixup_branches(struct nfp_prog *nfp_prog) in nfp_fixup_branches() argument
3209 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_fixup_branches()
3217 if (list_is_last(&meta->l, &nfp_prog->insns)) in nfp_fixup_branches()
3218 br_idx = nfp_prog->last_bpf_off; in nfp_fixup_branches()
3222 if (!nfp_is_br(nfp_prog->prog[br_idx])) { in nfp_fixup_branches()
3224 br_idx, meta->insn.code, nfp_prog->prog[br_idx]); in nfp_fixup_branches()
3228 if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) != in nfp_fixup_branches()
3245 if (!nfp_is_br(nfp_prog->prog[idx])) in nfp_fixup_branches()
3247 br_set_offset(&nfp_prog->prog[idx], jmp_dst->off); in nfp_fixup_branches()
3254 static void nfp_intro(struct nfp_prog *nfp_prog) in nfp_intro() argument
3256 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0)); in nfp_intro()
3257 emit_alu(nfp_prog, plen_reg(nfp_prog), in nfp_intro()
3258 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); in nfp_intro()
3261 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) in nfp_outro_tc_da() argument
3276 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); in nfp_outro_tc_da()
3278 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_tc_da()
3280 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_tc_da()
3281 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); in nfp_outro_tc_da()
3284 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); in nfp_outro_tc_da()
3287 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0)); in nfp_outro_tc_da()
3288 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); in nfp_outro_tc_da()
3289 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_tc_da()
3291 wrp_immed(nfp_prog, reg_b(2), 0x41221211); in nfp_outro_tc_da()
3292 wrp_immed(nfp_prog, reg_b(3), 0x41001211); in nfp_outro_tc_da()
3294 emit_shf(nfp_prog, reg_a(1), in nfp_outro_tc_da()
3297 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); in nfp_outro_tc_da()
3298 emit_shf(nfp_prog, reg_a(2), in nfp_outro_tc_da()
3301 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); in nfp_outro_tc_da()
3302 emit_shf(nfp_prog, reg_b(2), in nfp_outro_tc_da()
3305 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_tc_da()
3307 emit_shf(nfp_prog, reg_b(2), in nfp_outro_tc_da()
3309 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); in nfp_outro_tc_da()
3312 static void nfp_outro_xdp(struct nfp_prog *nfp_prog) in nfp_outro_xdp() argument
3322 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); in nfp_outro_xdp()
3324 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_xdp()
3326 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_xdp()
3327 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16); in nfp_outro_xdp()
3330 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); in nfp_outro_xdp()
3333 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0)); in nfp_outro_xdp()
3334 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); in nfp_outro_xdp()
3336 wrp_immed(nfp_prog, reg_b(2), 0x44112282); in nfp_outro_xdp()
3338 emit_shf(nfp_prog, reg_a(1), in nfp_outro_xdp()
3341 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); in nfp_outro_xdp()
3342 emit_shf(nfp_prog, reg_b(2), in nfp_outro_xdp()
3345 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_xdp()
3347 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_xdp()
3348 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); in nfp_outro_xdp()
3351 static void nfp_outro(struct nfp_prog *nfp_prog) in nfp_outro() argument
3353 switch (nfp_prog->type) { in nfp_outro()
3355 nfp_outro_tc_da(nfp_prog); in nfp_outro()
3358 nfp_outro_xdp(nfp_prog); in nfp_outro()
3365 static int nfp_translate(struct nfp_prog *nfp_prog) in nfp_translate() argument
3370 nfp_intro(nfp_prog); in nfp_translate()
3371 if (nfp_prog->error) in nfp_translate()
3372 return nfp_prog->error; in nfp_translate()
3374 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_translate()
3377 meta->off = nfp_prog_current_offset(nfp_prog); in nfp_translate()
3380 nfp_prog->n_translated++; in nfp_translate()
3384 if (nfp_meta_has_prev(nfp_prog, meta) && in nfp_translate()
3389 err = cb(nfp_prog, meta); in nfp_translate()
3392 if (nfp_prog->error) in nfp_translate()
3393 return nfp_prog->error; in nfp_translate()
3395 nfp_prog->n_translated++; in nfp_translate()
3398 nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1; in nfp_translate()
3400 nfp_outro(nfp_prog); in nfp_translate()
3401 if (nfp_prog->error) in nfp_translate()
3402 return nfp_prog->error; in nfp_translate()
3404 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW); in nfp_translate()
3405 if (nfp_prog->error) in nfp_translate()
3406 return nfp_prog->error; in nfp_translate()
3408 return nfp_fixup_branches(nfp_prog); in nfp_translate()
3412 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) in nfp_bpf_opt_reg_init() argument
3416 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_reg_init()
3438 static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog) in nfp_bpf_opt_neg_add_sub() argument
3442 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_neg_add_sub()
3484 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) in nfp_bpf_opt_ld_mask() argument
3493 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { in nfp_bpf_opt_ld_mask()
3523 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) in nfp_bpf_opt_ld_shift() argument
3527 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) { in nfp_bpf_opt_ld_shift()
3712 static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) in nfp_bpf_opt_ldst_gather() argument
3721 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { in nfp_bpf_opt_ldst_gather()
3801 static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog) in nfp_bpf_opt_pkt_cache() argument
3810 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_pkt_cache()
3899 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_pkt_cache()
3915 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) in nfp_bpf_optimize() argument
3917 nfp_bpf_opt_reg_init(nfp_prog); in nfp_bpf_optimize()
3919 nfp_bpf_opt_neg_add_sub(nfp_prog); in nfp_bpf_optimize()
3920 nfp_bpf_opt_ld_mask(nfp_prog); in nfp_bpf_optimize()
3921 nfp_bpf_opt_ld_shift(nfp_prog); in nfp_bpf_optimize()
3922 nfp_bpf_opt_ldst_gather(nfp_prog); in nfp_bpf_optimize()
3923 nfp_bpf_opt_pkt_cache(nfp_prog); in nfp_bpf_optimize()
3928 static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog) in nfp_bpf_replace_map_ptrs() argument
3935 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { in nfp_bpf_replace_map_ptrs()
3977 static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog) in nfp_bpf_prog_trim() argument
3981 prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL); in nfp_bpf_prog_trim()
3985 nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64); in nfp_bpf_prog_trim()
3986 memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len); in nfp_bpf_prog_trim()
3987 kvfree(nfp_prog->prog); in nfp_bpf_prog_trim()
3988 nfp_prog->prog = prog; in nfp_bpf_prog_trim()
3991 int nfp_bpf_jit(struct nfp_prog *nfp_prog) in nfp_bpf_jit() argument
3995 ret = nfp_bpf_replace_map_ptrs(nfp_prog); in nfp_bpf_jit()
3999 ret = nfp_bpf_optimize(nfp_prog); in nfp_bpf_jit()
4003 ret = nfp_translate(nfp_prog); in nfp_bpf_jit()
4006 ret, nfp_prog->n_translated); in nfp_bpf_jit()
4010 nfp_bpf_prog_trim(nfp_prog); in nfp_bpf_jit()
4015 void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt) in nfp_bpf_jit_prepare() argument
4020 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_jit_prepare()
4029 dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_indx, in nfp_bpf_jit_prepare()
4043 void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv) in nfp_bpf_relo_for_vnic() argument
4049 prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64), in nfp_bpf_relo_for_vnic()
4054 for (i = 0; i < nfp_prog->prog_len; i++) { in nfp_bpf_relo_for_vnic()
4067 nfp_prog->tgt_out + bv->start_off); in nfp_bpf_relo_for_vnic()
4071 nfp_prog->tgt_abort + bv->start_off); in nfp_bpf_relo_for_vnic()
4081 val = nfp_prog->bpf->helpers.map_lookup; in nfp_bpf_relo_for_vnic()
4084 val = nfp_prog->bpf->helpers.map_update; in nfp_bpf_relo_for_vnic()
4087 val = nfp_prog->bpf->helpers.map_delete; in nfp_bpf_relo_for_vnic()
4090 val = nfp_prog->bpf->helpers.perf_event_output; in nfp_bpf_relo_for_vnic()
4108 err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len); in nfp_bpf_relo_for_vnic()