1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * eBPF JIT compiler
4 *
5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
9 */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <asm/kprobes.h>
17 #include <linux/bpf.h>
18
19 #include "bpf_jit.h"
20
bpf_jit_fill_ill_insns(void * area,unsigned int size)21 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
22 {
23 memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
24 }
25
26 /* Fix updated addresses (for subprog calls, ldimm64, et al) during extra pass */
bpf_jit_fixup_addresses(struct bpf_prog * fp,u32 * image,struct codegen_context * ctx,u32 * addrs)27 static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image,
28 struct codegen_context *ctx, u32 *addrs)
29 {
30 const struct bpf_insn *insn = fp->insnsi;
31 bool func_addr_fixed;
32 u64 func_addr;
33 u32 tmp_idx;
34 int i, j, ret;
35
36 for (i = 0; i < fp->len; i++) {
37 /*
38 * During the extra pass, only the branch target addresses for
39 * the subprog calls need to be fixed. All other instructions
40 * can left untouched.
41 *
42 * The JITed image length does not change because we already
43 * ensure that the JITed instruction sequence for these calls
44 * are of fixed length by padding them with NOPs.
45 */
46 if (insn[i].code == (BPF_JMP | BPF_CALL) &&
47 insn[i].src_reg == BPF_PSEUDO_CALL) {
48 ret = bpf_jit_get_func_addr(fp, &insn[i], true,
49 &func_addr,
50 &func_addr_fixed);
51 if (ret < 0)
52 return ret;
53
54 /*
55 * Save ctx->idx as this would currently point to the
56 * end of the JITed image and set it to the offset of
57 * the instruction sequence corresponding to the
58 * subprog call temporarily.
59 */
60 tmp_idx = ctx->idx;
61 ctx->idx = addrs[i] / 4;
62 ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
63 if (ret)
64 return ret;
65
66 /*
67 * Restore ctx->idx here. This is safe as the length
68 * of the JITed sequence remains unchanged.
69 */
70 ctx->idx = tmp_idx;
71 } else if (insn[i].code == (BPF_LD | BPF_IMM | BPF_DW)) {
72 tmp_idx = ctx->idx;
73 ctx->idx = addrs[i] / 4;
74 #ifdef CONFIG_PPC32
75 PPC_LI32(bpf_to_ppc(insn[i].dst_reg) - 1, (u32)insn[i + 1].imm);
76 PPC_LI32(bpf_to_ppc(insn[i].dst_reg), (u32)insn[i].imm);
77 for (j = ctx->idx - addrs[i] / 4; j < 4; j++)
78 EMIT(PPC_RAW_NOP());
79 #else
80 func_addr = ((u64)(u32)insn[i].imm) | (((u64)(u32)insn[i + 1].imm) << 32);
81 PPC_LI64(bpf_to_ppc(insn[i].dst_reg), func_addr);
82 /* overwrite rest with nops */
83 for (j = ctx->idx - addrs[i] / 4; j < 5; j++)
84 EMIT(PPC_RAW_NOP());
85 #endif
86 ctx->idx = tmp_idx;
87 i++;
88 }
89 }
90
91 return 0;
92 }
93
bpf_jit_emit_exit_insn(u32 * image,struct codegen_context * ctx,int tmp_reg,long exit_addr)94 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
95 {
96 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
97 PPC_JMP(exit_addr);
98 } else if (ctx->alt_exit_addr) {
99 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
100 return -1;
101 PPC_JMP(ctx->alt_exit_addr);
102 } else {
103 ctx->alt_exit_addr = ctx->idx * 4;
104 bpf_jit_build_epilogue(image, ctx);
105 }
106
107 return 0;
108 }
109
110 struct powerpc64_jit_data {
111 struct bpf_binary_header *header;
112 u32 *addrs;
113 u8 *image;
114 u32 proglen;
115 struct codegen_context ctx;
116 };
117
bpf_jit_needs_zext(void)118 bool bpf_jit_needs_zext(void)
119 {
120 return true;
121 }
122
bpf_int_jit_compile(struct bpf_prog * fp)123 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
124 {
125 u32 proglen;
126 u32 alloclen;
127 u8 *image = NULL;
128 u32 *code_base;
129 u32 *addrs;
130 struct powerpc64_jit_data *jit_data;
131 struct codegen_context cgctx;
132 int pass;
133 int flen;
134 struct bpf_binary_header *bpf_hdr;
135 struct bpf_prog *org_fp = fp;
136 struct bpf_prog *tmp_fp;
137 bool bpf_blinded = false;
138 bool extra_pass = false;
139 u32 extable_len;
140 u32 fixup_len;
141
142 if (!fp->jit_requested)
143 return org_fp;
144
145 tmp_fp = bpf_jit_blind_constants(org_fp);
146 if (IS_ERR(tmp_fp))
147 return org_fp;
148
149 if (tmp_fp != org_fp) {
150 bpf_blinded = true;
151 fp = tmp_fp;
152 }
153
154 jit_data = fp->aux->jit_data;
155 if (!jit_data) {
156 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
157 if (!jit_data) {
158 fp = org_fp;
159 goto out;
160 }
161 fp->aux->jit_data = jit_data;
162 }
163
164 flen = fp->len;
165 addrs = jit_data->addrs;
166 if (addrs) {
167 cgctx = jit_data->ctx;
168 image = jit_data->image;
169 bpf_hdr = jit_data->header;
170 proglen = jit_data->proglen;
171 extra_pass = true;
172 goto skip_init_ctx;
173 }
174
175 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
176 if (addrs == NULL) {
177 fp = org_fp;
178 goto out_addrs;
179 }
180
181 memset(&cgctx, 0, sizeof(struct codegen_context));
182 bpf_jit_init_reg_mapping(&cgctx);
183
184 /* Make sure that the stack is quadword aligned. */
185 cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
186
187 /* Scouting faux-generate pass 0 */
188 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
189 /* We hit something illegal or unsupported. */
190 fp = org_fp;
191 goto out_addrs;
192 }
193
194 /*
195 * If we have seen a tail call, we need a second pass.
196 * This is because bpf_jit_emit_common_epilogue() is called
197 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
198 * We also need a second pass if we ended up with too large
199 * a program so as to ensure BPF_EXIT branches are in range.
200 */
201 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
202 cgctx.idx = 0;
203 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
204 fp = org_fp;
205 goto out_addrs;
206 }
207 }
208
209 bpf_jit_realloc_regs(&cgctx);
210 /*
211 * Pretend to build prologue, given the features we've seen. This will
212 * update ctgtx.idx as it pretends to output instructions, then we can
213 * calculate total size from idx.
214 */
215 bpf_jit_build_prologue(0, &cgctx);
216 addrs[fp->len] = cgctx.idx * 4;
217 bpf_jit_build_epilogue(0, &cgctx);
218
219 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
220 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
221
222 proglen = cgctx.idx * 4;
223 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
224
225 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
226 if (!bpf_hdr) {
227 fp = org_fp;
228 goto out_addrs;
229 }
230
231 if (extable_len)
232 fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len;
233
234 skip_init_ctx:
235 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
236
237 if (extra_pass) {
238 /*
239 * Do not touch the prologue and epilogue as they will remain
240 * unchanged. Only fix the branch target address for subprog
241 * calls in the body, and ldimm64 instructions.
242 *
243 * This does not change the offsets and lengths of the subprog
244 * call instruction sequences and hence, the size of the JITed
245 * image as well.
246 */
247 bpf_jit_fixup_addresses(fp, code_base, &cgctx, addrs);
248
249 /* There is no need to perform the usual passes. */
250 goto skip_codegen_passes;
251 }
252
253 /* Code generation passes 1-2 */
254 for (pass = 1; pass < 3; pass++) {
255 /* Now build the prologue, body code & epilogue for real. */
256 cgctx.idx = 0;
257 cgctx.alt_exit_addr = 0;
258 bpf_jit_build_prologue(code_base, &cgctx);
259 if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {
260 bpf_jit_binary_free(bpf_hdr);
261 fp = org_fp;
262 goto out_addrs;
263 }
264 bpf_jit_build_epilogue(code_base, &cgctx);
265
266 if (bpf_jit_enable > 1)
267 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
268 proglen - (cgctx.idx * 4), cgctx.seen);
269 }
270
271 skip_codegen_passes:
272 if (bpf_jit_enable > 1)
273 /*
274 * Note that we output the base address of the code_base
275 * rather than image, since opcodes are in code_base.
276 */
277 bpf_jit_dump(flen, proglen, pass, code_base);
278
279 #ifdef CONFIG_PPC64_ELF_ABI_V1
280 /* Function descriptor nastiness: Address + TOC */
281 ((u64 *)image)[0] = (u64)code_base;
282 ((u64 *)image)[1] = local_paca->kernel_toc;
283 #endif
284
285 fp->bpf_func = (void *)image;
286 fp->jited = 1;
287 fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
288
289 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size);
290 if (!fp->is_func || extra_pass) {
291 bpf_jit_binary_lock_ro(bpf_hdr);
292 bpf_prog_fill_jited_linfo(fp, addrs);
293 out_addrs:
294 kfree(addrs);
295 kfree(jit_data);
296 fp->aux->jit_data = NULL;
297 } else {
298 jit_data->addrs = addrs;
299 jit_data->ctx = cgctx;
300 jit_data->proglen = proglen;
301 jit_data->image = image;
302 jit_data->header = bpf_hdr;
303 }
304
305 out:
306 if (bpf_blinded)
307 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
308
309 return fp;
310 }
311
312 /*
313 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
314 * this function, as this only applies to BPF_PROBE_MEM, for now.
315 */
bpf_add_extable_entry(struct bpf_prog * fp,u32 * image,int pass,struct codegen_context * ctx,int insn_idx,int jmp_off,int dst_reg)316 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
317 int insn_idx, int jmp_off, int dst_reg)
318 {
319 off_t offset;
320 unsigned long pc;
321 struct exception_table_entry *ex;
322 u32 *fixup;
323
324 /* Populate extable entries only in the last pass */
325 if (pass != 2)
326 return 0;
327
328 if (!fp->aux->extable ||
329 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
330 return -EINVAL;
331
332 pc = (unsigned long)&image[insn_idx];
333
334 fixup = (void *)fp->aux->extable -
335 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
336 (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
337
338 fixup[0] = PPC_RAW_LI(dst_reg, 0);
339 if (IS_ENABLED(CONFIG_PPC32))
340 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
341
342 fixup[BPF_FIXUP_LEN - 1] =
343 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
344
345 ex = &fp->aux->extable[ctx->exentry_idx];
346
347 offset = pc - (long)&ex->insn;
348 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
349 return -ERANGE;
350 ex->insn = offset;
351
352 offset = (long)fixup - (long)&ex->fixup;
353 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
354 return -ERANGE;
355 ex->fixup = offset;
356
357 ctx->exentry_idx++;
358 return 0;
359 }
360
