1 // SPDX-License-Identifier: GPL-2.0
2 /* BPF JIT compiler for RV64G
3 *
4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
5 *
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/bpf.h>
10 #include <linux/filter.h>
11 #include "bpf_jit.h"
12
13 #define RV_REG_TCC RV_REG_A6
14 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
15
16 static const int regmap[] = {
17 [BPF_REG_0] = RV_REG_A5,
18 [BPF_REG_1] = RV_REG_A0,
19 [BPF_REG_2] = RV_REG_A1,
20 [BPF_REG_3] = RV_REG_A2,
21 [BPF_REG_4] = RV_REG_A3,
22 [BPF_REG_5] = RV_REG_A4,
23 [BPF_REG_6] = RV_REG_S1,
24 [BPF_REG_7] = RV_REG_S2,
25 [BPF_REG_8] = RV_REG_S3,
26 [BPF_REG_9] = RV_REG_S4,
27 [BPF_REG_FP] = RV_REG_S5,
28 [BPF_REG_AX] = RV_REG_T0,
29 };
30
31 static const int pt_regmap[] = {
32 [RV_REG_A0] = offsetof(struct pt_regs, a0),
33 [RV_REG_A1] = offsetof(struct pt_regs, a1),
34 [RV_REG_A2] = offsetof(struct pt_regs, a2),
35 [RV_REG_A3] = offsetof(struct pt_regs, a3),
36 [RV_REG_A4] = offsetof(struct pt_regs, a4),
37 [RV_REG_A5] = offsetof(struct pt_regs, a5),
38 [RV_REG_S1] = offsetof(struct pt_regs, s1),
39 [RV_REG_S2] = offsetof(struct pt_regs, s2),
40 [RV_REG_S3] = offsetof(struct pt_regs, s3),
41 [RV_REG_S4] = offsetof(struct pt_regs, s4),
42 [RV_REG_S5] = offsetof(struct pt_regs, s5),
43 [RV_REG_T0] = offsetof(struct pt_regs, t0),
44 };
45
46 enum {
47 RV_CTX_F_SEEN_TAIL_CALL = 0,
48 RV_CTX_F_SEEN_CALL = RV_REG_RA,
49 RV_CTX_F_SEEN_S1 = RV_REG_S1,
50 RV_CTX_F_SEEN_S2 = RV_REG_S2,
51 RV_CTX_F_SEEN_S3 = RV_REG_S3,
52 RV_CTX_F_SEEN_S4 = RV_REG_S4,
53 RV_CTX_F_SEEN_S5 = RV_REG_S5,
54 RV_CTX_F_SEEN_S6 = RV_REG_S6,
55 };
56
bpf_to_rv_reg(int bpf_reg,struct rv_jit_context * ctx)57 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
58 {
59 u8 reg = regmap[bpf_reg];
60
61 switch (reg) {
62 case RV_CTX_F_SEEN_S1:
63 case RV_CTX_F_SEEN_S2:
64 case RV_CTX_F_SEEN_S3:
65 case RV_CTX_F_SEEN_S4:
66 case RV_CTX_F_SEEN_S5:
67 case RV_CTX_F_SEEN_S6:
68 __set_bit(reg, &ctx->flags);
69 }
70 return reg;
71 };
72
seen_reg(int reg,struct rv_jit_context * ctx)73 static bool seen_reg(int reg, struct rv_jit_context *ctx)
74 {
75 switch (reg) {
76 case RV_CTX_F_SEEN_CALL:
77 case RV_CTX_F_SEEN_S1:
78 case RV_CTX_F_SEEN_S2:
79 case RV_CTX_F_SEEN_S3:
80 case RV_CTX_F_SEEN_S4:
81 case RV_CTX_F_SEEN_S5:
82 case RV_CTX_F_SEEN_S6:
83 return test_bit(reg, &ctx->flags);
84 }
85 return false;
86 }
87
mark_fp(struct rv_jit_context * ctx)88 static void mark_fp(struct rv_jit_context *ctx)
89 {
90 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
91 }
92
mark_call(struct rv_jit_context * ctx)93 static void mark_call(struct rv_jit_context *ctx)
94 {
95 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
96 }
97
seen_call(struct rv_jit_context * ctx)98 static bool seen_call(struct rv_jit_context *ctx)
99 {
100 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
101 }
102
mark_tail_call(struct rv_jit_context * ctx)103 static void mark_tail_call(struct rv_jit_context *ctx)
104 {
105 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
106 }
107
seen_tail_call(struct rv_jit_context * ctx)108 static bool seen_tail_call(struct rv_jit_context *ctx)
109 {
110 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
111 }
112
rv_tail_call_reg(struct rv_jit_context * ctx)113 static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
114 {
115 mark_tail_call(ctx);
116
117 if (seen_call(ctx)) {
118 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
119 return RV_REG_S6;
120 }
121 return RV_REG_A6;
122 }
123
is_32b_int(s64 val)124 static bool is_32b_int(s64 val)
125 {
126 return -(1L << 31) <= val && val < (1L << 31);
127 }
128
in_auipc_jalr_range(s64 val)129 static bool in_auipc_jalr_range(s64 val)
130 {
131 /*
132 * auipc+jalr can reach any signed PC-relative offset in the range
133 * [-2^31 - 2^11, 2^31 - 2^11).
134 */
135 return (-(1L << 31) - (1L << 11)) <= val &&
136 val < ((1L << 31) - (1L << 11));
137 }
138
emit_imm(u8 rd,s64 val,struct rv_jit_context * ctx)139 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
140 {
141 /* Note that the immediate from the add is sign-extended,
142 * which means that we need to compensate this by adding 2^12,
143 * when the 12th bit is set. A simpler way of doing this, and
144 * getting rid of the check, is to just add 2**11 before the
145 * shift. The "Loading a 32-Bit constant" example from the
146 * "Computer Organization and Design, RISC-V edition" book by
147 * Patterson/Hennessy highlights this fact.
148 *
149 * This also means that we need to process LSB to MSB.
150 */
151 s64 upper = (val + (1 << 11)) >> 12;
152 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw,
153 * and addi are signed and RVC checks will perform signed comparisons.
154 */
155 s64 lower = ((val & 0xfff) << 52) >> 52;
156 int shift;
157
158 if (is_32b_int(val)) {
159 if (upper)
160 emit_lui(rd, upper, ctx);
161
162 if (!upper) {
163 emit_li(rd, lower, ctx);
164 return;
165 }
166
167 emit_addiw(rd, rd, lower, ctx);
168 return;
169 }
170
171 shift = __ffs(upper);
172 upper >>= shift;
173 shift += 12;
174
175 emit_imm(rd, upper, ctx);
176
177 emit_slli(rd, rd, shift, ctx);
178 if (lower)
179 emit_addi(rd, rd, lower, ctx);
180 }
181
__build_epilogue(bool is_tail_call,struct rv_jit_context * ctx)182 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
183 {
184 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
185
186 if (seen_reg(RV_REG_RA, ctx)) {
187 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx);
188 store_offset -= 8;
189 }
190 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx);
191 store_offset -= 8;
192 if (seen_reg(RV_REG_S1, ctx)) {
193 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx);
194 store_offset -= 8;
195 }
196 if (seen_reg(RV_REG_S2, ctx)) {
197 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx);
198 store_offset -= 8;
199 }
200 if (seen_reg(RV_REG_S3, ctx)) {
201 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx);
202 store_offset -= 8;
203 }
204 if (seen_reg(RV_REG_S4, ctx)) {
205 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx);
206 store_offset -= 8;
207 }
208 if (seen_reg(RV_REG_S5, ctx)) {
209 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx);
210 store_offset -= 8;
211 }
212 if (seen_reg(RV_REG_S6, ctx)) {
213 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx);
214 store_offset -= 8;
215 }
216
217 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx);
218 /* Set return value. */
219 if (!is_tail_call)
220 emit_mv(RV_REG_A0, RV_REG_A5, ctx);
221 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
222 is_tail_call ? 4 : 0, /* skip TCC init */
223 ctx);
224 }
225
emit_bcc(u8 cond,u8 rd,u8 rs,int rvoff,struct rv_jit_context * ctx)226 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,
227 struct rv_jit_context *ctx)
228 {
229 switch (cond) {
230 case BPF_JEQ:
231 emit(rv_beq(rd, rs, rvoff >> 1), ctx);
232 return;
233 case BPF_JGT:
234 emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
235 return;
236 case BPF_JLT:
237 emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
238 return;
239 case BPF_JGE:
240 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
241 return;
242 case BPF_JLE:
243 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
244 return;
245 case BPF_JNE:
246 emit(rv_bne(rd, rs, rvoff >> 1), ctx);
247 return;
248 case BPF_JSGT:
249 emit(rv_blt(rs, rd, rvoff >> 1), ctx);
250 return;
251 case BPF_JSLT:
252 emit(rv_blt(rd, rs, rvoff >> 1), ctx);
253 return;
254 case BPF_JSGE:
255 emit(rv_bge(rd, rs, rvoff >> 1), ctx);
256 return;
257 case BPF_JSLE:
258 emit(rv_bge(rs, rd, rvoff >> 1), ctx);
259 }
260 }
261
emit_branch(u8 cond,u8 rd,u8 rs,int rvoff,struct rv_jit_context * ctx)262 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,
263 struct rv_jit_context *ctx)
264 {
265 s64 upper, lower;
266
267 if (is_13b_int(rvoff)) {
268 emit_bcc(cond, rd, rs, rvoff, ctx);
269 return;
270 }
271
272 /* Adjust for jal */
273 rvoff -= 4;
274
275 /* Transform, e.g.:
276 * bne rd,rs,foo
277 * to
278 * beq rd,rs,<.L1>
279 * (auipc foo)
280 * jal(r) foo
281 * .L1
282 */
283 cond = invert_bpf_cond(cond);
284 if (is_21b_int(rvoff)) {
285 emit_bcc(cond, rd, rs, 8, ctx);
286 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
287 return;
288 }
289
290 /* 32b No need for an additional rvoff adjustment, since we
291 * get that from the auipc at PC', where PC = PC' + 4.
292 */
293 upper = (rvoff + (1 << 11)) >> 12;
294 lower = rvoff & 0xfff;
295
296 emit_bcc(cond, rd, rs, 12, ctx);
297 emit(rv_auipc(RV_REG_T1, upper), ctx);
298 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx);
299 }
300
emit_zext_32(u8 reg,struct rv_jit_context * ctx)301 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
302 {
303 emit_slli(reg, reg, 32, ctx);
304 emit_srli(reg, reg, 32, ctx);
305 }
306
emit_bpf_tail_call(int insn,struct rv_jit_context * ctx)307 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
308 {
309 int tc_ninsn, off, start_insn = ctx->ninsns;
310 u8 tcc = rv_tail_call_reg(ctx);
311
312 /* a0: &ctx
313 * a1: &array
314 * a2: index
315 *
316 * if (index >= array->map.max_entries)
317 * goto out;
318 */
319 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
320 ctx->offset[0];
321 emit_zext_32(RV_REG_A2, ctx);
322
323 off = offsetof(struct bpf_array, map.max_entries);
324 if (is_12b_check(off, insn))
325 return -1;
326 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
327 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
328 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx);
329
330 /* if (--TCC < 0)
331 * goto out;
332 */
333 emit_addi(RV_REG_TCC, tcc, -1, ctx);
334 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
335 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx);
336
337 /* prog = array->ptrs[index];
338 * if (!prog)
339 * goto out;
340 */
341 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx);
342 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx);
343 off = offsetof(struct bpf_array, ptrs);
344 if (is_12b_check(off, insn))
345 return -1;
346 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx);
347 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
348 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx);
349
350 /* goto *(prog->bpf_func + 4); */
351 off = offsetof(struct bpf_prog, bpf_func);
352 if (is_12b_check(off, insn))
353 return -1;
354 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx);
355 __build_epilogue(true, ctx);
356 return 0;
357 }
358
init_regs(u8 * rd,u8 * rs,const struct bpf_insn * insn,struct rv_jit_context * ctx)359 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
360 struct rv_jit_context *ctx)
361 {
362 u8 code = insn->code;
363
364 switch (code) {
365 case BPF_JMP | BPF_JA:
366 case BPF_JMP | BPF_CALL:
367 case BPF_JMP | BPF_EXIT:
368 case BPF_JMP | BPF_TAIL_CALL:
369 break;
370 default:
371 *rd = bpf_to_rv_reg(insn->dst_reg, ctx);
372 }
373
374 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
375 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
376 code & BPF_LDX || code & BPF_STX)
377 *rs = bpf_to_rv_reg(insn->src_reg, ctx);
378 }
379
emit_zext_32_rd_rs(u8 * rd,u8 * rs,struct rv_jit_context * ctx)380 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
381 {
382 emit_mv(RV_REG_T2, *rd, ctx);
383 emit_zext_32(RV_REG_T2, ctx);
384 emit_mv(RV_REG_T1, *rs, ctx);
385 emit_zext_32(RV_REG_T1, ctx);
386 *rd = RV_REG_T2;
387 *rs = RV_REG_T1;
388 }
389
emit_sext_32_rd_rs(u8 * rd,u8 * rs,struct rv_jit_context * ctx)390 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
391 {
392 emit_addiw(RV_REG_T2, *rd, 0, ctx);
393 emit_addiw(RV_REG_T1, *rs, 0, ctx);
394 *rd = RV_REG_T2;
395 *rs = RV_REG_T1;
396 }
397
emit_zext_32_rd_t1(u8 * rd,struct rv_jit_context * ctx)398 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)
399 {
400 emit_mv(RV_REG_T2, *rd, ctx);
401 emit_zext_32(RV_REG_T2, ctx);
402 emit_zext_32(RV_REG_T1, ctx);
403 *rd = RV_REG_T2;
404 }
405
emit_sext_32_rd(u8 * rd,struct rv_jit_context * ctx)406 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
407 {
408 emit_addiw(RV_REG_T2, *rd, 0, ctx);
409 *rd = RV_REG_T2;
410 }
411
emit_jump_and_link(u8 rd,s64 rvoff,bool force_jalr,struct rv_jit_context * ctx)412 static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr,
413 struct rv_jit_context *ctx)
414 {
415 s64 upper, lower;
416
417 if (rvoff && is_21b_int(rvoff) && !force_jalr) {
418 emit(rv_jal(rd, rvoff >> 1), ctx);
419 return 0;
420 } else if (in_auipc_jalr_range(rvoff)) {
421 upper = (rvoff + (1 << 11)) >> 12;
422 lower = rvoff & 0xfff;
423 emit(rv_auipc(RV_REG_T1, upper), ctx);
424 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
425 return 0;
426 }
427
428 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff);
429 return -ERANGE;
430 }
431
is_signed_bpf_cond(u8 cond)432 static bool is_signed_bpf_cond(u8 cond)
433 {
434 return cond == BPF_JSGT || cond == BPF_JSLT ||
435 cond == BPF_JSGE || cond == BPF_JSLE;
436 }
437
emit_call(bool fixed,u64 addr,struct rv_jit_context * ctx)438 static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
439 {
440 s64 off = 0;
441 u64 ip;
442 u8 rd;
443 int ret;
444
445 if (addr && ctx->insns) {
446 ip = (u64)(long)(ctx->insns + ctx->ninsns);
447 off = addr - ip;
448 }
449
450 ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx);
451 if (ret)
452 return ret;
453 rd = bpf_to_rv_reg(BPF_REG_0, ctx);
454 emit_mv(rd, RV_REG_A0, ctx);
455 return 0;
456 }
457
emit_atomic(u8 rd,u8 rs,s16 off,s32 imm,bool is64,struct rv_jit_context * ctx)458 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
459 struct rv_jit_context *ctx)
460 {
461 u8 r0;
462 int jmp_offset;
463
464 if (off) {
465 if (is_12b_int(off)) {
466 emit_addi(RV_REG_T1, rd, off, ctx);
467 } else {
468 emit_imm(RV_REG_T1, off, ctx);
469 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
470 }
471 rd = RV_REG_T1;
472 }
473
474 switch (imm) {
475 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */
476 case BPF_ADD:
477 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) :
478 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
479 break;
480 case BPF_AND:
481 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) :
482 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
483 break;
484 case BPF_OR:
485 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) :
486 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
487 break;
488 case BPF_XOR:
489 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) :
490 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
491 break;
492 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */
493 case BPF_ADD | BPF_FETCH:
494 emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) :
495 rv_amoadd_w(rs, rs, rd, 0, 0), ctx);
496 if (!is64)
497 emit_zext_32(rs, ctx);
498 break;
499 case BPF_AND | BPF_FETCH:
500 emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) :
501 rv_amoand_w(rs, rs, rd, 0, 0), ctx);
502 if (!is64)
503 emit_zext_32(rs, ctx);
504 break;
505 case BPF_OR | BPF_FETCH:
506 emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) :
507 rv_amoor_w(rs, rs, rd, 0, 0), ctx);
508 if (!is64)
509 emit_zext_32(rs, ctx);
510 break;
511 case BPF_XOR | BPF_FETCH:
512 emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) :
513 rv_amoxor_w(rs, rs, rd, 0, 0), ctx);
514 if (!is64)
515 emit_zext_32(rs, ctx);
516 break;
517 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */
518 case BPF_XCHG:
519 emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) :
520 rv_amoswap_w(rs, rs, rd, 0, 0), ctx);
521 if (!is64)
522 emit_zext_32(rs, ctx);
523 break;
524 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
525 case BPF_CMPXCHG:
526 r0 = bpf_to_rv_reg(BPF_REG_0, ctx);
527 emit(is64 ? rv_addi(RV_REG_T2, r0, 0) :
528 rv_addiw(RV_REG_T2, r0, 0), ctx);
529 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
530 rv_lr_w(r0, 0, rd, 0, 0), ctx);
531 jmp_offset = ninsns_rvoff(8);
532 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
533 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) :
534 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx);
535 jmp_offset = ninsns_rvoff(-6);
536 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
537 emit(rv_fence(0x3, 0x3), ctx);
538 break;
539 }
540 }
541
542 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
543 #define BPF_FIXUP_REG_MASK GENMASK(31, 27)
544
ex_handler_bpf(const struct exception_table_entry * ex,struct pt_regs * regs)545 bool ex_handler_bpf(const struct exception_table_entry *ex,
546 struct pt_regs *regs)
547 {
548 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
549 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
550
551 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0;
552 regs->epc = (unsigned long)&ex->fixup - offset;
553
554 return true;
555 }
556
557 /* For accesses to BTF pointers, add an entry to the exception table */
add_exception_handler(const struct bpf_insn * insn,struct rv_jit_context * ctx,int dst_reg,int insn_len)558 static int add_exception_handler(const struct bpf_insn *insn,
559 struct rv_jit_context *ctx,
560 int dst_reg, int insn_len)
561 {
562 struct exception_table_entry *ex;
563 unsigned long pc;
564 off_t offset;
565
566 if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM)
567 return 0;
568
569 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
570 return -EINVAL;
571
572 if (WARN_ON_ONCE(insn_len > ctx->ninsns))
573 return -EINVAL;
574
575 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1))
576 return -EINVAL;
577
578 ex = &ctx->prog->aux->extable[ctx->nexentries];
579 pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len];
580
581 offset = pc - (long)&ex->insn;
582 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
583 return -ERANGE;
584 ex->insn = offset;
585
586 /*
587 * Since the extable follows the program, the fixup offset is always
588 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
589 * to keep things simple, and put the destination register in the upper
590 * bits. We don't need to worry about buildtime or runtime sort
591 * modifying the upper bits because the table is already sorted, and
592 * isn't part of the main exception table.
593 */
594 offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
595 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
596 return -ERANGE;
597
598 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
599 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
600 ex->type = EX_TYPE_BPF;
601
602 ctx->nexentries++;
603 return 0;
604 }
605
bpf_jit_emit_insn(const struct bpf_insn * insn,struct rv_jit_context * ctx,bool extra_pass)606 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
607 bool extra_pass)
608 {
609 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
610 BPF_CLASS(insn->code) == BPF_JMP;
611 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi;
612 struct bpf_prog_aux *aux = ctx->prog->aux;
613 u8 rd = -1, rs = -1, code = insn->code;
614 s16 off = insn->off;
615 s32 imm = insn->imm;
616
617 init_regs(&rd, &rs, insn, ctx);
618
619 switch (code) {
620 /* dst = src */
621 case BPF_ALU | BPF_MOV | BPF_X:
622 case BPF_ALU64 | BPF_MOV | BPF_X:
623 if (imm == 1) {
624 /* Special mov32 for zext */
625 emit_zext_32(rd, ctx);
626 break;
627 }
628 emit_mv(rd, rs, ctx);
629 if (!is64 && !aux->verifier_zext)
630 emit_zext_32(rd, ctx);
631 break;
632
633 /* dst = dst OP src */
634 case BPF_ALU | BPF_ADD | BPF_X:
635 case BPF_ALU64 | BPF_ADD | BPF_X:
636 emit_add(rd, rd, rs, ctx);
637 if (!is64 && !aux->verifier_zext)
638 emit_zext_32(rd, ctx);
639 break;
640 case BPF_ALU | BPF_SUB | BPF_X:
641 case BPF_ALU64 | BPF_SUB | BPF_X:
642 if (is64)
643 emit_sub(rd, rd, rs, ctx);
644 else
645 emit_subw(rd, rd, rs, ctx);
646
647 if (!is64 && !aux->verifier_zext)
648 emit_zext_32(rd, ctx);
649 break;
650 case BPF_ALU | BPF_AND | BPF_X:
651 case BPF_ALU64 | BPF_AND | BPF_X:
652 emit_and(rd, rd, rs, ctx);
653 if (!is64 && !aux->verifier_zext)
654 emit_zext_32(rd, ctx);
655 break;
656 case BPF_ALU | BPF_OR | BPF_X:
657 case BPF_ALU64 | BPF_OR | BPF_X:
658 emit_or(rd, rd, rs, ctx);
659 if (!is64 && !aux->verifier_zext)
660 emit_zext_32(rd, ctx);
661 break;
662 case BPF_ALU | BPF_XOR | BPF_X:
663 case BPF_ALU64 | BPF_XOR | BPF_X:
664 emit_xor(rd, rd, rs, ctx);
665 if (!is64 && !aux->verifier_zext)
666 emit_zext_32(rd, ctx);
667 break;
668 case BPF_ALU | BPF_MUL | BPF_X:
669 case BPF_ALU64 | BPF_MUL | BPF_X:
670 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
671 if (!is64 && !aux->verifier_zext)
672 emit_zext_32(rd, ctx);
673 break;
674 case BPF_ALU | BPF_DIV | BPF_X:
675 case BPF_ALU64 | BPF_DIV | BPF_X:
676 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
677 if (!is64 && !aux->verifier_zext)
678 emit_zext_32(rd, ctx);
679 break;
680 case BPF_ALU | BPF_MOD | BPF_X:
681 case BPF_ALU64 | BPF_MOD | BPF_X:
682 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
683 if (!is64 && !aux->verifier_zext)
684 emit_zext_32(rd, ctx);
685 break;
686 case BPF_ALU | BPF_LSH | BPF_X:
687 case BPF_ALU64 | BPF_LSH | BPF_X:
688 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
689 if (!is64 && !aux->verifier_zext)
690 emit_zext_32(rd, ctx);
691 break;
692 case BPF_ALU | BPF_RSH | BPF_X:
693 case BPF_ALU64 | BPF_RSH | BPF_X:
694 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
695 if (!is64 && !aux->verifier_zext)
696 emit_zext_32(rd, ctx);
697 break;
698 case BPF_ALU | BPF_ARSH | BPF_X:
699 case BPF_ALU64 | BPF_ARSH | BPF_X:
700 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
701 if (!is64 && !aux->verifier_zext)
702 emit_zext_32(rd, ctx);
703 break;
704
705 /* dst = -dst */
706 case BPF_ALU | BPF_NEG:
707 case BPF_ALU64 | BPF_NEG:
708 emit_sub(rd, RV_REG_ZERO, rd, ctx);
709 if (!is64 && !aux->verifier_zext)
710 emit_zext_32(rd, ctx);
711 break;
712
713 /* dst = BSWAP##imm(dst) */
714 case BPF_ALU | BPF_END | BPF_FROM_LE:
715 switch (imm) {
716 case 16:
717 emit_slli(rd, rd, 48, ctx);
718 emit_srli(rd, rd, 48, ctx);
719 break;
720 case 32:
721 if (!aux->verifier_zext)
722 emit_zext_32(rd, ctx);
723 break;
724 case 64:
725 /* Do nothing */
726 break;
727 }
728 break;
729
730 case BPF_ALU | BPF_END | BPF_FROM_BE:
731 emit_li(RV_REG_T2, 0, ctx);
732
733 emit_andi(RV_REG_T1, rd, 0xff, ctx);
734 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
735 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
736 emit_srli(rd, rd, 8, ctx);
737 if (imm == 16)
738 goto out_be;
739
740 emit_andi(RV_REG_T1, rd, 0xff, ctx);
741 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
742 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
743 emit_srli(rd, rd, 8, ctx);
744
745 emit_andi(RV_REG_T1, rd, 0xff, ctx);
746 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
747 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
748 emit_srli(rd, rd, 8, ctx);
749 if (imm == 32)
750 goto out_be;
751
752 emit_andi(RV_REG_T1, rd, 0xff, ctx);
753 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
754 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
755 emit_srli(rd, rd, 8, ctx);
756
757 emit_andi(RV_REG_T1, rd, 0xff, ctx);
758 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
759 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
760 emit_srli(rd, rd, 8, ctx);
761
762 emit_andi(RV_REG_T1, rd, 0xff, ctx);
763 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
764 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
765 emit_srli(rd, rd, 8, ctx);
766
767 emit_andi(RV_REG_T1, rd, 0xff, ctx);
768 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
769 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
770 emit_srli(rd, rd, 8, ctx);
771 out_be:
772 emit_andi(RV_REG_T1, rd, 0xff, ctx);
773 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
774
775 emit_mv(rd, RV_REG_T2, ctx);
776 break;
777
778 /* dst = imm */
779 case BPF_ALU | BPF_MOV | BPF_K:
780 case BPF_ALU64 | BPF_MOV | BPF_K:
781 emit_imm(rd, imm, ctx);
782 if (!is64 && !aux->verifier_zext)
783 emit_zext_32(rd, ctx);
784 break;
785
786 /* dst = dst OP imm */
787 case BPF_ALU | BPF_ADD | BPF_K:
788 case BPF_ALU64 | BPF_ADD | BPF_K:
789 if (is_12b_int(imm)) {
790 emit_addi(rd, rd, imm, ctx);
791 } else {
792 emit_imm(RV_REG_T1, imm, ctx);
793 emit_add(rd, rd, RV_REG_T1, ctx);
794 }
795 if (!is64 && !aux->verifier_zext)
796 emit_zext_32(rd, ctx);
797 break;
798 case BPF_ALU | BPF_SUB | BPF_K:
799 case BPF_ALU64 | BPF_SUB | BPF_K:
800 if (is_12b_int(-imm)) {
801 emit_addi(rd, rd, -imm, ctx);
802 } else {
803 emit_imm(RV_REG_T1, imm, ctx);
804 emit_sub(rd, rd, RV_REG_T1, ctx);
805 }
806 if (!is64 && !aux->verifier_zext)
807 emit_zext_32(rd, ctx);
808 break;
809 case BPF_ALU | BPF_AND | BPF_K:
810 case BPF_ALU64 | BPF_AND | BPF_K:
811 if (is_12b_int(imm)) {
812 emit_andi(rd, rd, imm, ctx);
813 } else {
814 emit_imm(RV_REG_T1, imm, ctx);
815 emit_and(rd, rd, RV_REG_T1, ctx);
816 }
817 if (!is64 && !aux->verifier_zext)
818 emit_zext_32(rd, ctx);
819 break;
820 case BPF_ALU | BPF_OR | BPF_K:
821 case BPF_ALU64 | BPF_OR | BPF_K:
822 if (is_12b_int(imm)) {
823 emit(rv_ori(rd, rd, imm), ctx);
824 } else {
825 emit_imm(RV_REG_T1, imm, ctx);
826 emit_or(rd, rd, RV_REG_T1, ctx);
827 }
828 if (!is64 && !aux->verifier_zext)
829 emit_zext_32(rd, ctx);
830 break;
831 case BPF_ALU | BPF_XOR | BPF_K:
832 case BPF_ALU64 | BPF_XOR | BPF_K:
833 if (is_12b_int(imm)) {
834 emit(rv_xori(rd, rd, imm), ctx);
835 } else {
836 emit_imm(RV_REG_T1, imm, ctx);
837 emit_xor(rd, rd, RV_REG_T1, ctx);
838 }
839 if (!is64 && !aux->verifier_zext)
840 emit_zext_32(rd, ctx);
841 break;
842 case BPF_ALU | BPF_MUL | BPF_K:
843 case BPF_ALU64 | BPF_MUL | BPF_K:
844 emit_imm(RV_REG_T1, imm, ctx);
845 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
846 rv_mulw(rd, rd, RV_REG_T1), ctx);
847 if (!is64 && !aux->verifier_zext)
848 emit_zext_32(rd, ctx);
849 break;
850 case BPF_ALU | BPF_DIV | BPF_K:
851 case BPF_ALU64 | BPF_DIV | BPF_K:
852 emit_imm(RV_REG_T1, imm, ctx);
853 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
854 rv_divuw(rd, rd, RV_REG_T1), ctx);
855 if (!is64 && !aux->verifier_zext)
856 emit_zext_32(rd, ctx);
857 break;
858 case BPF_ALU | BPF_MOD | BPF_K:
859 case BPF_ALU64 | BPF_MOD | BPF_K:
860 emit_imm(RV_REG_T1, imm, ctx);
861 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
862 rv_remuw(rd, rd, RV_REG_T1), ctx);
863 if (!is64 && !aux->verifier_zext)
864 emit_zext_32(rd, ctx);
865 break;
866 case BPF_ALU | BPF_LSH | BPF_K:
867 case BPF_ALU64 | BPF_LSH | BPF_K:
868 emit_slli(rd, rd, imm, ctx);
869
870 if (!is64 && !aux->verifier_zext)
871 emit_zext_32(rd, ctx);
872 break;
873 case BPF_ALU | BPF_RSH | BPF_K:
874 case BPF_ALU64 | BPF_RSH | BPF_K:
875 if (is64)
876 emit_srli(rd, rd, imm, ctx);
877 else
878 emit(rv_srliw(rd, rd, imm), ctx);
879
880 if (!is64 && !aux->verifier_zext)
881 emit_zext_32(rd, ctx);
882 break;
883 case BPF_ALU | BPF_ARSH | BPF_K:
884 case BPF_ALU64 | BPF_ARSH | BPF_K:
885 if (is64)
886 emit_srai(rd, rd, imm, ctx);
887 else
888 emit(rv_sraiw(rd, rd, imm), ctx);
889
890 if (!is64 && !aux->verifier_zext)
891 emit_zext_32(rd, ctx);
892 break;
893
894 /* JUMP off */
895 case BPF_JMP | BPF_JA:
896 rvoff = rv_offset(i, off, ctx);
897 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
898 if (ret)
899 return ret;
900 break;
901
902 /* IF (dst COND src) JUMP off */
903 case BPF_JMP | BPF_JEQ | BPF_X:
904 case BPF_JMP32 | BPF_JEQ | BPF_X:
905 case BPF_JMP | BPF_JGT | BPF_X:
906 case BPF_JMP32 | BPF_JGT | BPF_X:
907 case BPF_JMP | BPF_JLT | BPF_X:
908 case BPF_JMP32 | BPF_JLT | BPF_X:
909 case BPF_JMP | BPF_JGE | BPF_X:
910 case BPF_JMP32 | BPF_JGE | BPF_X:
911 case BPF_JMP | BPF_JLE | BPF_X:
912 case BPF_JMP32 | BPF_JLE | BPF_X:
913 case BPF_JMP | BPF_JNE | BPF_X:
914 case BPF_JMP32 | BPF_JNE | BPF_X:
915 case BPF_JMP | BPF_JSGT | BPF_X:
916 case BPF_JMP32 | BPF_JSGT | BPF_X:
917 case BPF_JMP | BPF_JSLT | BPF_X:
918 case BPF_JMP32 | BPF_JSLT | BPF_X:
919 case BPF_JMP | BPF_JSGE | BPF_X:
920 case BPF_JMP32 | BPF_JSGE | BPF_X:
921 case BPF_JMP | BPF_JSLE | BPF_X:
922 case BPF_JMP32 | BPF_JSLE | BPF_X:
923 case BPF_JMP | BPF_JSET | BPF_X:
924 case BPF_JMP32 | BPF_JSET | BPF_X:
925 rvoff = rv_offset(i, off, ctx);
926 if (!is64) {
927 s = ctx->ninsns;
928 if (is_signed_bpf_cond(BPF_OP(code)))
929 emit_sext_32_rd_rs(&rd, &rs, ctx);
930 else
931 emit_zext_32_rd_rs(&rd, &rs, ctx);
932 e = ctx->ninsns;
933
934 /* Adjust for extra insns */
935 rvoff -= ninsns_rvoff(e - s);
936 }
937
938 if (BPF_OP(code) == BPF_JSET) {
939 /* Adjust for and */
940 rvoff -= 4;
941 emit_and(RV_REG_T1, rd, rs, ctx);
942 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
943 ctx);
944 } else {
945 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
946 }
947 break;
948
949 /* IF (dst COND imm) JUMP off */
950 case BPF_JMP | BPF_JEQ | BPF_K:
951 case BPF_JMP32 | BPF_JEQ | BPF_K:
952 case BPF_JMP | BPF_JGT | BPF_K:
953 case BPF_JMP32 | BPF_JGT | BPF_K:
954 case BPF_JMP | BPF_JLT | BPF_K:
955 case BPF_JMP32 | BPF_JLT | BPF_K:
956 case BPF_JMP | BPF_JGE | BPF_K:
957 case BPF_JMP32 | BPF_JGE | BPF_K:
958 case BPF_JMP | BPF_JLE | BPF_K:
959 case BPF_JMP32 | BPF_JLE | BPF_K:
960 case BPF_JMP | BPF_JNE | BPF_K:
961 case BPF_JMP32 | BPF_JNE | BPF_K:
962 case BPF_JMP | BPF_JSGT | BPF_K:
963 case BPF_JMP32 | BPF_JSGT | BPF_K:
964 case BPF_JMP | BPF_JSLT | BPF_K:
965 case BPF_JMP32 | BPF_JSLT | BPF_K:
966 case BPF_JMP | BPF_JSGE | BPF_K:
967 case BPF_JMP32 | BPF_JSGE | BPF_K:
968 case BPF_JMP | BPF_JSLE | BPF_K:
969 case BPF_JMP32 | BPF_JSLE | BPF_K:
970 rvoff = rv_offset(i, off, ctx);
971 s = ctx->ninsns;
972 if (imm) {
973 emit_imm(RV_REG_T1, imm, ctx);
974 rs = RV_REG_T1;
975 } else {
976 /* If imm is 0, simply use zero register. */
977 rs = RV_REG_ZERO;
978 }
979 if (!is64) {
980 if (is_signed_bpf_cond(BPF_OP(code)))
981 emit_sext_32_rd(&rd, ctx);
982 else
983 emit_zext_32_rd_t1(&rd, ctx);
984 }
985 e = ctx->ninsns;
986
987 /* Adjust for extra insns */
988 rvoff -= ninsns_rvoff(e - s);
989 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
990 break;
991
992 case BPF_JMP | BPF_JSET | BPF_K:
993 case BPF_JMP32 | BPF_JSET | BPF_K:
994 rvoff = rv_offset(i, off, ctx);
995 s = ctx->ninsns;
996 if (is_12b_int(imm)) {
997 emit_andi(RV_REG_T1, rd, imm, ctx);
998 } else {
999 emit_imm(RV_REG_T1, imm, ctx);
1000 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx);
1001 }
1002 /* For jset32, we should clear the upper 32 bits of t1, but
1003 * sign-extension is sufficient here and saves one instruction,
1004 * as t1 is used only in comparison against zero.
1005 */
1006 if (!is64 && imm < 0)
1007 emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx);
1008 e = ctx->ninsns;
1009 rvoff -= ninsns_rvoff(e - s);
1010 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx);
1011 break;
1012
1013 /* function call */
1014 case BPF_JMP | BPF_CALL:
1015 {
1016 bool fixed;
1017 u64 addr;
1018
1019 mark_call(ctx);
1020 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
1021 &fixed);
1022 if (ret < 0)
1023 return ret;
1024 ret = emit_call(fixed, addr, ctx);
1025 if (ret)
1026 return ret;
1027 break;
1028 }
1029 /* tail call */
1030 case BPF_JMP | BPF_TAIL_CALL:
1031 if (emit_bpf_tail_call(i, ctx))
1032 return -1;
1033 break;
1034
1035 /* function return */
1036 case BPF_JMP | BPF_EXIT:
1037 if (i == ctx->prog->len - 1)
1038 break;
1039
1040 rvoff = epilogue_offset(ctx);
1041 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1042 if (ret)
1043 return ret;
1044 break;
1045
1046 /* dst = imm64 */
1047 case BPF_LD | BPF_IMM | BPF_DW:
1048 {
1049 struct bpf_insn insn1 = insn[1];
1050 u64 imm64;
1051
1052 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1053 emit_imm(rd, imm64, ctx);
1054 return 1;
1055 }
1056
1057 /* LDX: dst = *(size *)(src + off) */
1058 case BPF_LDX | BPF_MEM | BPF_B:
1059 case BPF_LDX | BPF_MEM | BPF_H:
1060 case BPF_LDX | BPF_MEM | BPF_W:
1061 case BPF_LDX | BPF_MEM | BPF_DW:
1062 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1063 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1064 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1065 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1066 {
1067 int insn_len, insns_start;
1068
1069 switch (BPF_SIZE(code)) {
1070 case BPF_B:
1071 if (is_12b_int(off)) {
1072 insns_start = ctx->ninsns;
1073 emit(rv_lbu(rd, off, rs), ctx);
1074 insn_len = ctx->ninsns - insns_start;
1075 break;
1076 }
1077
1078 emit_imm(RV_REG_T1, off, ctx);
1079 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1080 insns_start = ctx->ninsns;
1081 emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
1082 insn_len = ctx->ninsns - insns_start;
1083 if (insn_is_zext(&insn[1]))
1084 return 1;
1085 break;
1086 case BPF_H:
1087 if (is_12b_int(off)) {
1088 insns_start = ctx->ninsns;
1089 emit(rv_lhu(rd, off, rs), ctx);
1090 insn_len = ctx->ninsns - insns_start;
1091 break;
1092 }
1093
1094 emit_imm(RV_REG_T1, off, ctx);
1095 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1096 insns_start = ctx->ninsns;
1097 emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
1098 insn_len = ctx->ninsns - insns_start;
1099 if (insn_is_zext(&insn[1]))
1100 return 1;
1101 break;
1102 case BPF_W:
1103 if (is_12b_int(off)) {
1104 insns_start = ctx->ninsns;
1105 emit(rv_lwu(rd, off, rs), ctx);
1106 insn_len = ctx->ninsns - insns_start;
1107 break;
1108 }
1109
1110 emit_imm(RV_REG_T1, off, ctx);
1111 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1112 insns_start = ctx->ninsns;
1113 emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
1114 insn_len = ctx->ninsns - insns_start;
1115 if (insn_is_zext(&insn[1]))
1116 return 1;
1117 break;
1118 case BPF_DW:
1119 if (is_12b_int(off)) {
1120 insns_start = ctx->ninsns;
1121 emit_ld(rd, off, rs, ctx);
1122 insn_len = ctx->ninsns - insns_start;
1123 break;
1124 }
1125
1126 emit_imm(RV_REG_T1, off, ctx);
1127 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1128 insns_start = ctx->ninsns;
1129 emit_ld(rd, 0, RV_REG_T1, ctx);
1130 insn_len = ctx->ninsns - insns_start;
1131 break;
1132 }
1133
1134 ret = add_exception_handler(insn, ctx, rd, insn_len);
1135 if (ret)
1136 return ret;
1137 break;
1138 }
1139 /* speculation barrier */
1140 case BPF_ST | BPF_NOSPEC:
1141 break;
1142
1143 /* ST: *(size *)(dst + off) = imm */
1144 case BPF_ST | BPF_MEM | BPF_B:
1145 emit_imm(RV_REG_T1, imm, ctx);
1146 if (is_12b_int(off)) {
1147 emit(rv_sb(rd, off, RV_REG_T1), ctx);
1148 break;
1149 }
1150
1151 emit_imm(RV_REG_T2, off, ctx);
1152 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1153 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx);
1154 break;
1155
1156 case BPF_ST | BPF_MEM | BPF_H:
1157 emit_imm(RV_REG_T1, imm, ctx);
1158 if (is_12b_int(off)) {
1159 emit(rv_sh(rd, off, RV_REG_T1), ctx);
1160 break;
1161 }
1162
1163 emit_imm(RV_REG_T2, off, ctx);
1164 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1165 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx);
1166 break;
1167 case BPF_ST | BPF_MEM | BPF_W:
1168 emit_imm(RV_REG_T1, imm, ctx);
1169 if (is_12b_int(off)) {
1170 emit_sw(rd, off, RV_REG_T1, ctx);
1171 break;
1172 }
1173
1174 emit_imm(RV_REG_T2, off, ctx);
1175 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1176 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx);
1177 break;
1178 case BPF_ST | BPF_MEM | BPF_DW:
1179 emit_imm(RV_REG_T1, imm, ctx);
1180 if (is_12b_int(off)) {
1181 emit_sd(rd, off, RV_REG_T1, ctx);
1182 break;
1183 }
1184
1185 emit_imm(RV_REG_T2, off, ctx);
1186 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1187 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx);
1188 break;
1189
1190 /* STX: *(size *)(dst + off) = src */
1191 case BPF_STX | BPF_MEM | BPF_B:
1192 if (is_12b_int(off)) {
1193 emit(rv_sb(rd, off, rs), ctx);
1194 break;
1195 }
1196
1197 emit_imm(RV_REG_T1, off, ctx);
1198 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1199 emit(rv_sb(RV_REG_T1, 0, rs), ctx);
1200 break;
1201 case BPF_STX | BPF_MEM | BPF_H:
1202 if (is_12b_int(off)) {
1203 emit(rv_sh(rd, off, rs), ctx);
1204 break;
1205 }
1206
1207 emit_imm(RV_REG_T1, off, ctx);
1208 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1209 emit(rv_sh(RV_REG_T1, 0, rs), ctx);
1210 break;
1211 case BPF_STX | BPF_MEM | BPF_W:
1212 if (is_12b_int(off)) {
1213 emit_sw(rd, off, rs, ctx);
1214 break;
1215 }
1216
1217 emit_imm(RV_REG_T1, off, ctx);
1218 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1219 emit_sw(RV_REG_T1, 0, rs, ctx);
1220 break;
1221 case BPF_STX | BPF_MEM | BPF_DW:
1222 if (is_12b_int(off)) {
1223 emit_sd(rd, off, rs, ctx);
1224 break;
1225 }
1226
1227 emit_imm(RV_REG_T1, off, ctx);
1228 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1229 emit_sd(RV_REG_T1, 0, rs, ctx);
1230 break;
1231 case BPF_STX | BPF_ATOMIC | BPF_W:
1232 case BPF_STX | BPF_ATOMIC | BPF_DW:
1233 emit_atomic(rd, rs, off, imm,
1234 BPF_SIZE(code) == BPF_DW, ctx);
1235 break;
1236 default:
1237 pr_err("bpf-jit: unknown opcode %02x\n", code);
1238 return -EINVAL;
1239 }
1240
1241 return 0;
1242 }
1243
bpf_jit_build_prologue(struct rv_jit_context * ctx)1244 void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1245 {
1246 int stack_adjust = 0, store_offset, bpf_stack_adjust;
1247
1248 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1249 if (bpf_stack_adjust)
1250 mark_fp(ctx);
1251
1252 if (seen_reg(RV_REG_RA, ctx))
1253 stack_adjust += 8;
1254 stack_adjust += 8; /* RV_REG_FP */
1255 if (seen_reg(RV_REG_S1, ctx))
1256 stack_adjust += 8;
1257 if (seen_reg(RV_REG_S2, ctx))
1258 stack_adjust += 8;
1259 if (seen_reg(RV_REG_S3, ctx))
1260 stack_adjust += 8;
1261 if (seen_reg(RV_REG_S4, ctx))
1262 stack_adjust += 8;
1263 if (seen_reg(RV_REG_S5, ctx))
1264 stack_adjust += 8;
1265 if (seen_reg(RV_REG_S6, ctx))
1266 stack_adjust += 8;
1267
1268 stack_adjust = round_up(stack_adjust, 16);
1269 stack_adjust += bpf_stack_adjust;
1270
1271 store_offset = stack_adjust - 8;
1272
1273 /* First instruction is always setting the tail-call-counter
1274 * (TCC) register. This instruction is skipped for tail calls.
1275 * Force using a 4-byte (non-compressed) instruction.
1276 */
1277 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1278
1279 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx);
1280
1281 if (seen_reg(RV_REG_RA, ctx)) {
1282 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx);
1283 store_offset -= 8;
1284 }
1285 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx);
1286 store_offset -= 8;
1287 if (seen_reg(RV_REG_S1, ctx)) {
1288 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx);
1289 store_offset -= 8;
1290 }
1291 if (seen_reg(RV_REG_S2, ctx)) {
1292 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx);
1293 store_offset -= 8;
1294 }
1295 if (seen_reg(RV_REG_S3, ctx)) {
1296 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx);
1297 store_offset -= 8;
1298 }
1299 if (seen_reg(RV_REG_S4, ctx)) {
1300 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx);
1301 store_offset -= 8;
1302 }
1303 if (seen_reg(RV_REG_S5, ctx)) {
1304 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx);
1305 store_offset -= 8;
1306 }
1307 if (seen_reg(RV_REG_S6, ctx)) {
1308 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx);
1309 store_offset -= 8;
1310 }
1311
1312 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx);
1313
1314 if (bpf_stack_adjust)
1315 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx);
1316
1317 /* Program contains calls and tail calls, so RV_REG_TCC need
1318 * to be saved across calls.
1319 */
1320 if (seen_tail_call(ctx) && seen_call(ctx))
1321 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx);
1322
1323 ctx->stack_size = stack_adjust;
1324 }
1325
bpf_jit_build_epilogue(struct rv_jit_context * ctx)1326 void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1327 {
1328 __build_epilogue(false, ctx);
1329 }
1330
