Lines Matching refs:meta
72 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_meta_has_prev() argument
74 return meta->l.prev != &nfp_prog->insns; in nfp_meta_has_prev()
713 static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_cpp_memcpy() argument
715 bool descending_seq = meta->ldst_gather_len < 0; in nfp_cpp_memcpy()
716 s16 len = abs(meta->ldst_gather_len); in nfp_cpp_memcpy()
722 off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
723 src_40bit_addr = meta->ptr.type == PTR_TO_MAP_VALUE; in nfp_cpp_memcpy()
724 src_base = reg_a(meta->insn.src_reg * 2); in nfp_cpp_memcpy()
728 addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base, in nfp_cpp_memcpy()
745 off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
750 reg_a(meta->paired_st->dst_reg * 2), off, len - 1, in nfp_cpp_memcpy()
755 reg_a(meta->paired_st->dst_reg * 2), off, xfer_num - 1, in nfp_cpp_memcpy()
762 reg_a(meta->paired_st->dst_reg * 2), off, in nfp_cpp_memcpy()
769 reg_a(meta->paired_st->dst_reg * 2), off, in nfp_cpp_memcpy()
776 reg_a(meta->paired_st->dst_reg * 2), off, 7, in nfp_cpp_memcpy()
779 off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32, in nfp_cpp_memcpy()
782 reg_a(meta->paired_st->dst_reg * 2), off, len - 33, in nfp_cpp_memcpy()
793 reg_a(meta->paired_st->dst_reg * 2), off, in nfp_cpp_memcpy()
795 new_off = meta->paired_st->off + (xfer_num - 1) * 4; in nfp_cpp_memcpy()
798 xfer_num - 1, reg_a(meta->paired_st->dst_reg * 2), off, in nfp_cpp_memcpy()
814 else if (BPF_SIZE(meta->insn.code) != BPF_DW) in nfp_cpp_memcpy()
819 switch (BPF_SIZE(meta->insn.code)) { in nfp_cpp_memcpy()
821 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
826 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
830 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
834 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
836 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), in nfp_cpp_memcpy()
841 if (BPF_SIZE(meta->insn.code) != BPF_DW) in nfp_cpp_memcpy()
842 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in nfp_cpp_memcpy()
1136 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_op_stack() argument
1140 s32 off = nfp_prog->stack_depth + meta->insn.off + ptr_off; in mem_op_stack()
1149 if (meta->ptr_not_const) { in mem_op_stack()
1154 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off, in mem_op_stack()
1259 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu64_imm() argument
1262 const struct bpf_insn *insn = &meta->insn; in wrp_alu64_imm()
1266 meta->skip = true; in wrp_alu64_imm()
1277 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu64_reg() argument
1280 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; in wrp_alu64_reg()
1290 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu32_imm() argument
1293 const struct bpf_insn *insn = &meta->insn; in wrp_alu32_imm()
1296 meta->skip = true; in wrp_alu32_imm()
1307 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu32_reg() argument
1310 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; in wrp_alu32_reg()
1313 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in wrp_alu32_reg()
1327 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_test_reg() argument
1330 const struct bpf_insn *insn = &meta->insn; in wrp_test_reg()
1354 static const struct jmp_code_map *nfp_jmp_code_get(struct nfp_insn_meta *meta) in nfp_jmp_code_get() argument
1358 op = BPF_OP(meta->insn.code) >> 4; in nfp_jmp_code_get()
1368 static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in cmp_imm() argument
1370 const struct bpf_insn *insn = &meta->insn; in cmp_imm()
1377 code = nfp_jmp_code_get(meta); in cmp_imm()
1381 alu_op = meta->jump_neg_op ? ALU_OP_ADD : ALU_OP_SUB; in cmp_imm()
1382 carry_op = meta->jump_neg_op ? ALU_OP_ADD_C : ALU_OP_SUB_C; in cmp_imm()
1403 static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in cmp_reg() argument
1405 const struct bpf_insn *insn = &meta->insn; in cmp_reg()
1409 code = nfp_jmp_code_get(meta); in cmp_reg()
1466 wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_mul() argument
1470 const struct bpf_insn *insn = &meta->insn; in wrp_mul()
1478 lopnd_max = meta->umax_dst; in wrp_mul()
1481 ropnd_max = meta->umax_src; in wrp_mul()
1565 static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in adjust_head() argument
1575 if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n)) in adjust_head()
1645 static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in adjust_tail() argument
1691 map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in map_call_stack_common() argument
1699 lm_off += meta->arg2.reg.var_off.value + meta->arg2.reg.off; in map_call_stack_common()
1700 load_lm_ptr = meta->arg2.var_off || lm_off; in map_call_stack_common()
1705 if (meta->func_id == BPF_FUNC_map_update_elem) in map_call_stack_common()
1708 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, in map_call_stack_common()
1732 nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_get_prandom_u32() argument
1744 nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_perf_event_output() argument
1749 ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog)); in nfp_perf_event_output()
1753 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, in nfp_perf_event_output()
1769 nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_queue_select() argument
1776 emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2), in nfp_queue_select()
1785 pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2), in nfp_queue_select()
1801 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_reg64() argument
1803 const struct bpf_insn *insn = &meta->insn; in mov_reg64()
1824 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_imm64() argument
1826 u64 imm = meta->insn.imm; /* sign extend */ in mov_imm64()
1828 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); in mov_imm64()
1829 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); in mov_imm64()
1834 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_reg64() argument
1836 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); in xor_reg64()
1839 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_imm64() argument
1841 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); in xor_imm64()
1844 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_reg64() argument
1846 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); in and_reg64()
1849 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_imm64() argument
1851 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); in and_imm64()
1854 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_reg64() argument
1856 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); in or_reg64()
1859 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_imm64() argument
1861 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); in or_imm64()
1864 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_reg64() argument
1866 const struct bpf_insn *insn = &meta->insn; in add_reg64()
1878 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_imm64() argument
1880 const struct bpf_insn *insn = &meta->insn; in add_imm64()
1889 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_reg64() argument
1891 const struct bpf_insn *insn = &meta->insn; in sub_reg64()
1903 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_imm64() argument
1905 const struct bpf_insn *insn = &meta->insn; in sub_imm64()
1914 static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_reg64() argument
1916 return wrp_mul(nfp_prog, meta, true, true); in mul_reg64()
1919 static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_imm64() argument
1921 return wrp_mul(nfp_prog, meta, true, false); in mul_imm64()
1924 static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_imm64() argument
1926 const struct bpf_insn *insn = &meta->insn; in div_imm64()
1931 static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_reg64() argument
1936 return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src); in div_reg64()
1939 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in neg_reg64() argument
1941 const struct bpf_insn *insn = &meta->insn; in neg_reg64()
1981 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_imm64() argument
1983 const struct bpf_insn *insn = &meta->insn; in shl_imm64()
2020 static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_reg64() argument
2022 const struct bpf_insn *insn = &meta->insn; in shl_reg64()
2027 umin = meta->umin_src; in shl_reg64()
2028 umax = meta->umax_src; in shl_reg64()
2092 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_imm64() argument
2094 const struct bpf_insn *insn = &meta->insn; in shr_imm64()
2129 static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_reg64() argument
2131 const struct bpf_insn *insn = &meta->insn; in shr_reg64()
2136 umin = meta->umin_src; in shr_reg64()
2137 umax = meta->umax_src; in shr_reg64()
2201 static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_imm64() argument
2203 const struct bpf_insn *insn = &meta->insn; in ashr_imm64()
2243 static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_reg64() argument
2245 const struct bpf_insn *insn = &meta->insn; in ashr_reg64()
2250 umin = meta->umin_src; in ashr_reg64()
2251 umax = meta->umax_src; in ashr_reg64()
2282 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_reg() argument
2284 const struct bpf_insn *insn = &meta->insn; in mov_reg()
2292 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_imm() argument
2294 const struct bpf_insn *insn = &meta->insn; in mov_imm()
2302 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_reg() argument
2304 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); in xor_reg()
2307 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_imm() argument
2309 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm); in xor_imm()
2312 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_reg() argument
2314 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); in and_reg()
2317 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_imm() argument
2319 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); in and_imm()
2322 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_reg() argument
2324 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); in or_reg()
2327 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_imm() argument
2329 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); in or_imm()
2332 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_reg() argument
2334 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); in add_reg()
2337 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_imm() argument
2339 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm); in add_imm()
2342 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_reg() argument
2344 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); in sub_reg()
2347 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_imm() argument
2349 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm); in sub_imm()
2352 static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_reg() argument
2354 return wrp_mul(nfp_prog, meta, false, true); in mul_reg()
2357 static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_imm() argument
2359 return wrp_mul(nfp_prog, meta, false, false); in mul_imm()
2362 static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_reg() argument
2364 return div_reg64(nfp_prog, meta); in div_reg()
2367 static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_imm() argument
2369 return div_imm64(nfp_prog, meta); in div_imm()
2372 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in neg_reg() argument
2374 u8 dst = meta->insn.dst_reg * 2; in neg_reg()
2377 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in neg_reg()
2382 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_imm() argument
2384 const struct bpf_insn *insn = &meta->insn; in shl_imm()
2397 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in end_reg32() argument
2399 const struct bpf_insn *insn = &meta->insn; in end_reg32()
2426 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in imm_ld8_part2() argument
2428 struct nfp_insn_meta *prev = nfp_meta_prev(meta); in imm_ld8_part2()
2434 imm_hi = meta->insn.imm; in imm_ld8_part2()
2447 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in imm_ld8() argument
2449 meta->double_cb = imm_ld8_part2; in imm_ld8()
2453 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld1() argument
2455 return construct_data_ld(nfp_prog, meta->insn.imm, 1); in data_ld1()
2458 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld2() argument
2460 return construct_data_ld(nfp_prog, meta->insn.imm, 2); in data_ld2()
2463 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld4() argument
2465 return construct_data_ld(nfp_prog, meta->insn.imm, 4); in data_ld4()
2468 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld1() argument
2470 return construct_data_ind_ld(nfp_prog, meta->insn.imm, in data_ind_ld1()
2471 meta->insn.src_reg * 2, 1); in data_ind_ld1()
2474 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld2() argument
2476 return construct_data_ind_ld(nfp_prog, meta->insn.imm, in data_ind_ld2()
2477 meta->insn.src_reg * 2, 2); in data_ind_ld2()
2480 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld4() argument
2482 return construct_data_ind_ld(nfp_prog, meta->insn.imm, in data_ind_ld4()
2483 meta->insn.src_reg * 2, 4); in data_ind_ld4()
2487 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_stack() argument
2490 return mem_op_stack(nfp_prog, meta, size, ptr_off, in mem_ldx_stack()
2491 meta->insn.dst_reg * 2, meta->insn.src_reg * 2, in mem_ldx_stack()
2495 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_skb() argument
2498 swreg dst = reg_both(meta->insn.dst_reg * 2); in mem_ldx_skb()
2500 switch (meta->insn.off) { in mem_ldx_skb()
2521 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in mem_ldx_skb()
2526 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_xdp() argument
2529 swreg dst = reg_both(meta->insn.dst_reg * 2); in mem_ldx_xdp()
2531 switch (meta->insn.off) { in mem_ldx_xdp()
2547 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in mem_ldx_xdp()
2553 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_data() argument
2558 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_ldx_data()
2560 return data_ld_host_order_addr32(nfp_prog, meta->insn.src_reg * 2, in mem_ldx_data()
2561 tmp_reg, meta->insn.dst_reg * 2, size); in mem_ldx_data()
2565 mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_emem() argument
2570 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_ldx_emem()
2572 return data_ld_host_order_addr40(nfp_prog, meta->insn.src_reg * 2, in mem_ldx_emem()
2573 tmp_reg, meta->insn.dst_reg * 2, size); in mem_ldx_emem()
2578 struct nfp_insn_meta *meta) in mem_ldx_data_init_pktcache() argument
2580 s16 range_start = meta->pkt_cache.range_start; in mem_ldx_data_init_pktcache()
2581 s16 range_end = meta->pkt_cache.range_end; in mem_ldx_data_init_pktcache()
2587 src_base = reg_a(meta->insn.src_reg * 2); in mem_ldx_data_init_pktcache()
2604 struct nfp_insn_meta *meta, in mem_ldx_data_from_pktcache_unaligned() argument
2607 s16 range_start = meta->pkt_cache.range_start; in mem_ldx_data_from_pktcache_unaligned()
2608 s16 insn_off = meta->insn.off - range_start; in mem_ldx_data_from_pktcache_unaligned()
2610 u8 dst_gpr = meta->insn.dst_reg * 2; in mem_ldx_data_from_pktcache_unaligned()
2658 struct nfp_insn_meta *meta, in mem_ldx_data_from_pktcache_aligned() argument
2664 idx = (meta->insn.off - meta->pkt_cache.range_start) / REG_WIDTH; in mem_ldx_data_from_pktcache_aligned()
2665 dst_gpr = meta->insn.dst_reg * 2; in mem_ldx_data_from_pktcache_aligned()
2688 struct nfp_insn_meta *meta, unsigned int size) in mem_ldx_data_from_pktcache() argument
2690 u8 off = meta->insn.off - meta->pkt_cache.range_start; in mem_ldx_data_from_pktcache()
2693 return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size); in mem_ldx_data_from_pktcache()
2695 return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size); in mem_ldx_data_from_pktcache()
2699 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx() argument
2702 if (meta->ldst_gather_len) in mem_ldx()
2703 return nfp_cpp_memcpy(nfp_prog, meta); in mem_ldx()
2705 if (meta->ptr.type == PTR_TO_CTX) { in mem_ldx()
2707 return mem_ldx_xdp(nfp_prog, meta, size); in mem_ldx()
2709 return mem_ldx_skb(nfp_prog, meta, size); in mem_ldx()
2712 if (meta->ptr.type == PTR_TO_PACKET) { in mem_ldx()
2713 if (meta->pkt_cache.range_end) { in mem_ldx()
2714 if (meta->pkt_cache.do_init) in mem_ldx()
2715 mem_ldx_data_init_pktcache(nfp_prog, meta); in mem_ldx()
2717 return mem_ldx_data_from_pktcache(nfp_prog, meta, size); in mem_ldx()
2719 return mem_ldx_data(nfp_prog, meta, size); in mem_ldx()
2723 if (meta->ptr.type == PTR_TO_STACK) in mem_ldx()
2724 return mem_ldx_stack(nfp_prog, meta, size, in mem_ldx()
2725 meta->ptr.off + meta->ptr.var_off.value); in mem_ldx()
2727 if (meta->ptr.type == PTR_TO_MAP_VALUE) in mem_ldx()
2728 return mem_ldx_emem(nfp_prog, meta, size); in mem_ldx()
2733 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx1() argument
2735 return mem_ldx(nfp_prog, meta, 1); in mem_ldx1()
2738 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx2() argument
2740 return mem_ldx(nfp_prog, meta, 2); in mem_ldx2()
2743 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx4() argument
2745 return mem_ldx(nfp_prog, meta, 4); in mem_ldx4()
2748 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx8() argument
2750 return mem_ldx(nfp_prog, meta, 8); in mem_ldx8()
2754 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_st_data() argument
2757 u64 imm = meta->insn.imm; /* sign extend */ in mem_st_data()
2760 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_st_data()
2762 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, in mem_st_data()
2766 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_st() argument
2769 if (meta->ptr.type == PTR_TO_PACKET) in mem_st()
2770 return mem_st_data(nfp_prog, meta, size); in mem_st()
2775 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st1() argument
2777 return mem_st(nfp_prog, meta, 1); in mem_st1()
2780 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st2() argument
2782 return mem_st(nfp_prog, meta, 2); in mem_st2()
2785 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st4() argument
2787 return mem_st(nfp_prog, meta, 4); in mem_st4()
2790 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st8() argument
2792 return mem_st(nfp_prog, meta, 8); in mem_st8()
2796 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx_data() argument
2801 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_stx_data()
2803 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, in mem_stx_data()
2804 meta->insn.src_reg * 2, size); in mem_stx_data()
2808 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx_stack() argument
2811 return mem_op_stack(nfp_prog, meta, size, ptr_off, in mem_stx_stack()
2812 meta->insn.src_reg * 2, meta->insn.dst_reg * 2, in mem_stx_stack()
2816 static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx_xdp() argument
2818 switch (meta->insn.off) { in mem_stx_xdp()
2820 return nfp_queue_select(nfp_prog, meta); in mem_stx_xdp()
2828 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx() argument
2831 if (meta->ptr.type == PTR_TO_PACKET) in mem_stx()
2832 return mem_stx_data(nfp_prog, meta, size); in mem_stx()
2834 if (meta->ptr.type == PTR_TO_STACK) in mem_stx()
2835 return mem_stx_stack(nfp_prog, meta, size, in mem_stx()
2836 meta->ptr.off + meta->ptr.var_off.value); in mem_stx()
2841 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx1() argument
2843 return mem_stx(nfp_prog, meta, 1); in mem_stx1()
2846 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx2() argument
2848 return mem_stx(nfp_prog, meta, 2); in mem_stx2()
2851 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx4() argument
2853 if (meta->ptr.type == PTR_TO_CTX) in mem_stx4()
2855 return mem_stx_xdp(nfp_prog, meta); in mem_stx4()
2856 return mem_stx(nfp_prog, meta, 4); in mem_stx4()
2859 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx8() argument
2861 return mem_stx(nfp_prog, meta, 8); in mem_stx8()
2865 mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64) in mem_xadd() argument
2867 u8 dst_gpr = meta->insn.dst_reg * 2; in mem_xadd()
2868 u8 src_gpr = meta->insn.src_reg * 2; in mem_xadd()
2872 off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_xadd()
2882 if (meta->insn.off) { in mem_xadd()
2886 if (meta->xadd_maybe_16bit) { in mem_xadd()
2890 if (meta->xadd_over_16bit) in mem_xadd()
2892 if (meta->xadd_maybe_16bit && meta->xadd_over_16bit) { in mem_xadd()
2898 if (meta->xadd_maybe_16bit && meta->xadd_over_16bit) { in mem_xadd()
2906 emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0); in mem_xadd()
2911 if (!meta->insn.off) { in mem_xadd()
2924 if (meta->xadd_maybe_16bit) { in mem_xadd()
2935 if (meta->xadd_over_16bit) in mem_xadd()
2943 if (meta->xadd_over_16bit) { in mem_xadd()
2959 static int mem_xadd4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_xadd4() argument
2961 return mem_xadd(nfp_prog, meta, false); in mem_xadd4()
2964 static int mem_xadd8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_xadd8() argument
2966 return mem_xadd(nfp_prog, meta, true); in mem_xadd8()
2969 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jump() argument
2971 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); in jump()
2976 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jeq_imm() argument
2978 const struct bpf_insn *insn = &meta->insn; in jeq_imm()
3005 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jset_imm() argument
3007 const struct bpf_insn *insn = &meta->insn; in jset_imm()
3012 meta->skip = true; in jset_imm()
3033 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jne_imm() argument
3035 const struct bpf_insn *insn = &meta->insn; in jne_imm()
3059 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jeq_reg() argument
3061 const struct bpf_insn *insn = &meta->insn; in jeq_reg()
3074 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jset_reg() argument
3076 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); in jset_reg()
3079 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jne_reg() argument
3081 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); in jne_reg()
3084 static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in call() argument
3086 switch (meta->insn.imm) { in call()
3088 return adjust_head(nfp_prog, meta); in call()
3090 return adjust_tail(nfp_prog, meta); in call()
3094 return map_call_stack_common(nfp_prog, meta); in call()
3096 return nfp_get_prandom_u32(nfp_prog, meta); in call()
3098 return nfp_perf_event_output(nfp_prog, meta); in call()
3105 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in goto_out() argument
3206 struct nfp_insn_meta *meta, *jmp_dst; in nfp_fixup_branches() local
3209 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_fixup_branches()
3210 if (meta->skip) in nfp_fixup_branches()
3212 if (meta->insn.code == (BPF_JMP | BPF_CALL)) in nfp_fixup_branches()
3214 if (BPF_CLASS(meta->insn.code) != BPF_JMP) in nfp_fixup_branches()
3217 if (list_is_last(&meta->l, &nfp_prog->insns)) in nfp_fixup_branches()
3220 br_idx = list_next_entry(meta, l)->off - 1; in nfp_fixup_branches()
3224 br_idx, meta->insn.code, nfp_prog->prog[br_idx]); in nfp_fixup_branches()
3232 if (!meta->jmp_dst) { in nfp_fixup_branches()
3237 jmp_dst = meta->jmp_dst; in nfp_fixup_branches()
3244 for (idx = meta->off; idx <= br_idx; idx++) { in nfp_fixup_branches()
3367 struct nfp_insn_meta *meta; in nfp_translate() local
3374 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_translate()
3375 instr_cb_t cb = instr_cb[meta->insn.code]; in nfp_translate()
3377 meta->off = nfp_prog_current_offset(nfp_prog); in nfp_translate()
3379 if (meta->skip) { in nfp_translate()
3384 if (nfp_meta_has_prev(nfp_prog, meta) && in nfp_translate()
3385 nfp_meta_prev(meta)->double_cb) in nfp_translate()
3386 cb = nfp_meta_prev(meta)->double_cb; in nfp_translate()
3389 err = cb(nfp_prog, meta); in nfp_translate()
3414 struct nfp_insn_meta *meta; in nfp_bpf_opt_reg_init() local
3416 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_reg_init()
3417 struct bpf_insn insn = meta->insn; in nfp_bpf_opt_reg_init()
3427 meta->skip = true; in nfp_bpf_opt_reg_init()
3430 if (!meta->skip) in nfp_bpf_opt_reg_init()
3440 struct nfp_insn_meta *meta; in nfp_bpf_opt_neg_add_sub() local
3442 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_neg_add_sub()
3443 struct bpf_insn insn = meta->insn; in nfp_bpf_opt_neg_add_sub()
3445 if (meta->skip) in nfp_bpf_opt_neg_add_sub()
3463 meta->jump_neg_op = true; in nfp_bpf_opt_neg_add_sub()
3476 meta->insn.code = insn.code | BPF_K; in nfp_bpf_opt_neg_add_sub()
3479 meta->insn.imm = -insn.imm; in nfp_bpf_opt_neg_add_sub()
3803 struct nfp_insn_meta *meta, *range_node = NULL; in nfp_bpf_opt_pkt_cache() local
3810 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_pkt_cache()
3811 if (meta->flags & FLAG_INSN_IS_JUMP_DST) in nfp_bpf_opt_pkt_cache()
3814 if (meta->skip) in nfp_bpf_opt_pkt_cache()
3817 insn = &meta->insn; in nfp_bpf_opt_pkt_cache()
3819 if (is_mbpf_store_pkt(meta) || in nfp_bpf_opt_pkt_cache()
3821 is_mbpf_classic_store_pkt(meta) || in nfp_bpf_opt_pkt_cache()
3822 is_mbpf_classic_load(meta)) { in nfp_bpf_opt_pkt_cache()
3827 if (!is_mbpf_load(meta)) in nfp_bpf_opt_pkt_cache()
3830 if (meta->ptr.type != PTR_TO_PACKET || meta->ldst_gather_len) { in nfp_bpf_opt_pkt_cache()
3853 if (meta->ptr.id == range_ptr_id && in nfp_bpf_opt_pkt_cache()
3854 meta->ptr.off == range_ptr_off) { in nfp_bpf_opt_pkt_cache()
3886 range_node = meta; in nfp_bpf_opt_pkt_cache()
3899 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_pkt_cache()
3900 if (meta->skip) in nfp_bpf_opt_pkt_cache()
3903 if (is_mbpf_load_pkt(meta) && !meta->ldst_gather_len) { in nfp_bpf_opt_pkt_cache()
3904 if (meta->pkt_cache.do_init) { in nfp_bpf_opt_pkt_cache()
3905 range_start = meta->pkt_cache.range_start; in nfp_bpf_opt_pkt_cache()
3906 range_end = meta->pkt_cache.range_end; in nfp_bpf_opt_pkt_cache()
3908 meta->pkt_cache.range_start = range_start; in nfp_bpf_opt_pkt_cache()
3909 meta->pkt_cache.range_end = range_end; in nfp_bpf_opt_pkt_cache()
4017 struct nfp_insn_meta *meta; in nfp_bpf_jit_prepare() local
4020 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_jit_prepare()
4021 u64 code = meta->insn.code; in nfp_bpf_jit_prepare()
4028 dst_indx = meta->n + 1 + meta->insn.off; in nfp_bpf_jit_prepare()
4029 dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_indx, in nfp_bpf_jit_prepare()
4032 meta->jmp_dst = dst_meta; in nfp_bpf_jit_prepare()