1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * BPF JIT compiler for LoongArch
4 *
5 * Copyright (C) 2022 Loongson Technology Corporation Limited
6 */
7 #include <linux/bitfield.h>
8 #include <linux/bpf.h>
9 #include <linux/filter.h>
10 #include <asm/cacheflush.h>
11 #include <asm/inst.h>
12
13 struct jit_ctx {
14 const struct bpf_prog *prog;
15 unsigned int idx;
16 unsigned int flags;
17 unsigned int epilogue_offset;
18 u32 *offset;
19 int num_exentries;
20 union loongarch_instruction *image;
21 u32 stack_size;
22 };
23
24 struct jit_data {
25 struct bpf_binary_header *header;
26 u8 *image;
27 struct jit_ctx ctx;
28 };
29
30 #define emit_insn(ctx, func, ...) \
31 do { \
32 if (ctx->image != NULL) { \
33 union loongarch_instruction *insn = &ctx->image[ctx->idx]; \
34 emit_##func(insn, ##__VA_ARGS__); \
35 } \
36 ctx->idx++; \
37 } while (0)
38
39 #define is_signed_imm12(val) signed_imm_check(val, 12)
40 #define is_signed_imm14(val) signed_imm_check(val, 14)
41 #define is_signed_imm16(val) signed_imm_check(val, 16)
42 #define is_signed_imm26(val) signed_imm_check(val, 26)
43 #define is_signed_imm32(val) signed_imm_check(val, 32)
44 #define is_signed_imm52(val) signed_imm_check(val, 52)
45 #define is_unsigned_imm12(val) unsigned_imm_check(val, 12)
46
bpf2la_offset(int bpf_insn,int off,const struct jit_ctx * ctx)47 static inline int bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx)
48 {
49 /* BPF JMP offset is relative to the next instruction */
50 bpf_insn++;
51 /*
52 * Whereas LoongArch branch instructions encode the offset
53 * from the branch itself, so we must subtract 1 from the
54 * instruction offset.
55 */
56 return (ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1));
57 }
58
epilogue_offset(const struct jit_ctx * ctx)59 static inline int epilogue_offset(const struct jit_ctx *ctx)
60 {
61 int from = ctx->idx;
62 int to = ctx->epilogue_offset;
63
64 return (to - from);
65 }
66
67 /* Zero-extend 32 bits into 64 bits */
emit_zext_32(struct jit_ctx * ctx,enum loongarch_gpr reg,bool is32)68 static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)
69 {
70 if (!is32)
71 return;
72
73 emit_insn(ctx, lu32id, reg, 0);
74 }
75
76 /* Signed-extend 32 bits into 64 bits */
emit_sext_32(struct jit_ctx * ctx,enum loongarch_gpr reg,bool is32)77 static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)
78 {
79 if (!is32)
80 return;
81
82 emit_insn(ctx, addiw, reg, reg, 0);
83 }
84
move_addr(struct jit_ctx * ctx,enum loongarch_gpr rd,u64 addr)85 static inline void move_addr(struct jit_ctx *ctx, enum loongarch_gpr rd, u64 addr)
86 {
87 u64 imm_11_0, imm_31_12, imm_51_32, imm_63_52;
88
89 /* lu12iw rd, imm_31_12 */
90 imm_31_12 = (addr >> 12) & 0xfffff;
91 emit_insn(ctx, lu12iw, rd, imm_31_12);
92
93 /* ori rd, rd, imm_11_0 */
94 imm_11_0 = addr & 0xfff;
95 emit_insn(ctx, ori, rd, rd, imm_11_0);
96
97 /* lu32id rd, imm_51_32 */
98 imm_51_32 = (addr >> 32) & 0xfffff;
99 emit_insn(ctx, lu32id, rd, imm_51_32);
100
101 /* lu52id rd, rd, imm_63_52 */
102 imm_63_52 = (addr >> 52) & 0xfff;
103 emit_insn(ctx, lu52id, rd, rd, imm_63_52);
104 }
105
move_imm(struct jit_ctx * ctx,enum loongarch_gpr rd,long imm,bool is32)106 static inline void move_imm(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm, bool is32)
107 {
108 long imm_11_0, imm_31_12, imm_51_32, imm_63_52, imm_51_0, imm_51_31;
109
110 /* or rd, $zero, $zero */
111 if (imm == 0) {
112 emit_insn(ctx, or, rd, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_ZERO);
113 return;
114 }
115
116 /* addiw rd, $zero, imm_11_0 */
117 if (is_signed_imm12(imm)) {
118 emit_insn(ctx, addiw, rd, LOONGARCH_GPR_ZERO, imm);
119 goto zext;
120 }
121
122 /* ori rd, $zero, imm_11_0 */
123 if (is_unsigned_imm12(imm)) {
124 emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm);
125 goto zext;
126 }
127
128 /* lu52id rd, $zero, imm_63_52 */
129 imm_63_52 = (imm >> 52) & 0xfff;
130 imm_51_0 = imm & 0xfffffffffffff;
131 if (imm_63_52 != 0 && imm_51_0 == 0) {
132 emit_insn(ctx, lu52id, rd, LOONGARCH_GPR_ZERO, imm_63_52);
133 return;
134 }
135
136 /* lu12iw rd, imm_31_12 */
137 imm_31_12 = (imm >> 12) & 0xfffff;
138 emit_insn(ctx, lu12iw, rd, imm_31_12);
139
140 /* ori rd, rd, imm_11_0 */
141 imm_11_0 = imm & 0xfff;
142 if (imm_11_0 != 0)
143 emit_insn(ctx, ori, rd, rd, imm_11_0);
144
145 if (!is_signed_imm32(imm)) {
146 if (imm_51_0 != 0) {
147 /*
148 * If bit[51:31] is all 0 or all 1,
149 * it means bit[51:32] is sign extended by lu12iw,
150 * no need to call lu32id to do a new filled operation.
151 */
152 imm_51_31 = (imm >> 31) & 0x1fffff;
153 if (imm_51_31 != 0 && imm_51_31 != 0x1fffff) {
154 /* lu32id rd, imm_51_32 */
155 imm_51_32 = (imm >> 32) & 0xfffff;
156 emit_insn(ctx, lu32id, rd, imm_51_32);
157 }
158 }
159
160 /* lu52id rd, rd, imm_63_52 */
161 if (!is_signed_imm52(imm))
162 emit_insn(ctx, lu52id, rd, rd, imm_63_52);
163 }
164
165 zext:
166 emit_zext_32(ctx, rd, is32);
167 }
168
move_reg(struct jit_ctx * ctx,enum loongarch_gpr rd,enum loongarch_gpr rj)169 static inline void move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd,
170 enum loongarch_gpr rj)
171 {
172 emit_insn(ctx, or, rd, rj, LOONGARCH_GPR_ZERO);
173 }
174
invert_jmp_cond(u8 cond)175 static inline int invert_jmp_cond(u8 cond)
176 {
177 switch (cond) {
178 case BPF_JEQ:
179 return BPF_JNE;
180 case BPF_JNE:
181 case BPF_JSET:
182 return BPF_JEQ;
183 case BPF_JGT:
184 return BPF_JLE;
185 case BPF_JGE:
186 return BPF_JLT;
187 case BPF_JLT:
188 return BPF_JGE;
189 case BPF_JLE:
190 return BPF_JGT;
191 case BPF_JSGT:
192 return BPF_JSLE;
193 case BPF_JSGE:
194 return BPF_JSLT;
195 case BPF_JSLT:
196 return BPF_JSGE;
197 case BPF_JSLE:
198 return BPF_JSGT;
199 }
200 return -1;
201 }
202
cond_jmp_offset(struct jit_ctx * ctx,u8 cond,enum loongarch_gpr rj,enum loongarch_gpr rd,int jmp_offset)203 static inline void cond_jmp_offset(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
204 enum loongarch_gpr rd, int jmp_offset)
205 {
206 switch (cond) {
207 case BPF_JEQ:
208 /* PC += jmp_offset if rj == rd */
209 emit_insn(ctx, beq, rj, rd, jmp_offset);
210 return;
211 case BPF_JNE:
212 case BPF_JSET:
213 /* PC += jmp_offset if rj != rd */
214 emit_insn(ctx, bne, rj, rd, jmp_offset);
215 return;
216 case BPF_JGT:
217 /* PC += jmp_offset if rj > rd (unsigned) */
218 emit_insn(ctx, bltu, rd, rj, jmp_offset);
219 return;
220 case BPF_JLT:
221 /* PC += jmp_offset if rj < rd (unsigned) */
222 emit_insn(ctx, bltu, rj, rd, jmp_offset);
223 return;
224 case BPF_JGE:
225 /* PC += jmp_offset if rj >= rd (unsigned) */
226 emit_insn(ctx, bgeu, rj, rd, jmp_offset);
227 return;
228 case BPF_JLE:
229 /* PC += jmp_offset if rj <= rd (unsigned) */
230 emit_insn(ctx, bgeu, rd, rj, jmp_offset);
231 return;
232 case BPF_JSGT:
233 /* PC += jmp_offset if rj > rd (signed) */
234 emit_insn(ctx, blt, rd, rj, jmp_offset);
235 return;
236 case BPF_JSLT:
237 /* PC += jmp_offset if rj < rd (signed) */
238 emit_insn(ctx, blt, rj, rd, jmp_offset);
239 return;
240 case BPF_JSGE:
241 /* PC += jmp_offset if rj >= rd (signed) */
242 emit_insn(ctx, bge, rj, rd, jmp_offset);
243 return;
244 case BPF_JSLE:
245 /* PC += jmp_offset if rj <= rd (signed) */
246 emit_insn(ctx, bge, rd, rj, jmp_offset);
247 return;
248 }
249 }
250
cond_jmp_offs26(struct jit_ctx * ctx,u8 cond,enum loongarch_gpr rj,enum loongarch_gpr rd,int jmp_offset)251 static inline void cond_jmp_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
252 enum loongarch_gpr rd, int jmp_offset)
253 {
254 cond = invert_jmp_cond(cond);
255 cond_jmp_offset(ctx, cond, rj, rd, 2);
256 emit_insn(ctx, b, jmp_offset);
257 }
258
uncond_jmp_offs26(struct jit_ctx * ctx,int jmp_offset)259 static inline void uncond_jmp_offs26(struct jit_ctx *ctx, int jmp_offset)
260 {
261 emit_insn(ctx, b, jmp_offset);
262 }
263
emit_cond_jmp(struct jit_ctx * ctx,u8 cond,enum loongarch_gpr rj,enum loongarch_gpr rd,int jmp_offset)264 static inline int emit_cond_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
265 enum loongarch_gpr rd, int jmp_offset)
266 {
267 /*
268 * A large PC-relative jump offset may overflow the immediate field of
269 * the native conditional branch instruction, triggering a conversion
270 * to use an absolute jump instead, this jump sequence is particularly
271 * nasty. For now, use cond_jmp_offs26() directly to keep it simple.
272 * In the future, maybe we can add support for far branching, the branch
273 * relaxation requires more than two passes to converge, the code seems
274 * too complex to understand, not quite sure whether it is necessary and
275 * worth the extra pain. Anyway, just leave it as it is to enhance code
276 * readability now.
277 */
278 if (is_signed_imm26(jmp_offset)) {
279 cond_jmp_offs26(ctx, cond, rj, rd, jmp_offset);
280 return 0;
281 }
282
283 return -EINVAL;
284 }
285
emit_uncond_jmp(struct jit_ctx * ctx,int jmp_offset)286 static inline int emit_uncond_jmp(struct jit_ctx *ctx, int jmp_offset)
287 {
288 if (is_signed_imm26(jmp_offset)) {
289 uncond_jmp_offs26(ctx, jmp_offset);
290 return 0;
291 }
292
293 return -EINVAL;
294 }
295
emit_tailcall_jmp(struct jit_ctx * ctx,u8 cond,enum loongarch_gpr rj,enum loongarch_gpr rd,int jmp_offset)296 static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
297 enum loongarch_gpr rd, int jmp_offset)
298 {
299 if (is_signed_imm16(jmp_offset)) {
300 cond_jmp_offset(ctx, cond, rj, rd, jmp_offset);
301 return 0;
302 }
303
304 return -EINVAL;
305 }
306
