1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * eBPF JIT compiler
4 *
5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
9 */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <asm/kprobes.h>
17 #include <linux/bpf.h>
18
19 #include "bpf_jit.h"
20
bpf_jit_fill_ill_insns(void * area,unsigned int size)21 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
22 {
23 memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
24 }
25
bpf_jit_emit_exit_insn(u32 * image,struct codegen_context * ctx,int tmp_reg,long exit_addr)26 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
27 {
28 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
29 PPC_JMP(exit_addr);
30 } else if (ctx->alt_exit_addr) {
31 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
32 return -1;
33 PPC_JMP(ctx->alt_exit_addr);
34 } else {
35 ctx->alt_exit_addr = ctx->idx * 4;
36 bpf_jit_build_epilogue(image, ctx);
37 }
38
39 return 0;
40 }
41
42 struct powerpc64_jit_data {
43 struct bpf_binary_header *header;
44 u32 *addrs;
45 u8 *image;
46 u32 proglen;
47 struct codegen_context ctx;
48 };
49
bpf_jit_needs_zext(void)50 bool bpf_jit_needs_zext(void)
51 {
52 return true;
53 }
54
bpf_int_jit_compile(struct bpf_prog * fp)55 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
56 {
57 u32 proglen;
58 u32 alloclen;
59 u8 *image = NULL;
60 u32 *code_base;
61 u32 *addrs;
62 struct powerpc64_jit_data *jit_data;
63 struct codegen_context cgctx;
64 int pass;
65 int flen;
66 struct bpf_binary_header *bpf_hdr;
67 struct bpf_prog *org_fp = fp;
68 struct bpf_prog *tmp_fp;
69 bool bpf_blinded = false;
70 bool extra_pass = false;
71 u32 extable_len;
72 u32 fixup_len;
73
74 if (!fp->jit_requested)
75 return org_fp;
76
77 tmp_fp = bpf_jit_blind_constants(org_fp);
78 if (IS_ERR(tmp_fp))
79 return org_fp;
80
81 if (tmp_fp != org_fp) {
82 bpf_blinded = true;
83 fp = tmp_fp;
84 }
85
86 jit_data = fp->aux->jit_data;
87 if (!jit_data) {
88 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
89 if (!jit_data) {
90 fp = org_fp;
91 goto out;
92 }
93 fp->aux->jit_data = jit_data;
94 }
95
96 flen = fp->len;
97 addrs = jit_data->addrs;
98 if (addrs) {
99 cgctx = jit_data->ctx;
100 image = jit_data->image;
101 bpf_hdr = jit_data->header;
102 proglen = jit_data->proglen;
103 extra_pass = true;
104 /* During extra pass, ensure index is reset before repopulating extable entries */
105 cgctx.exentry_idx = 0;
106 goto skip_init_ctx;
107 }
108
109 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
110 if (addrs == NULL) {
111 fp = org_fp;
112 goto out_addrs;
113 }
114
115 memset(&cgctx, 0, sizeof(struct codegen_context));
116 bpf_jit_init_reg_mapping(&cgctx);
117
118 /* Make sure that the stack is quadword aligned. */
119 cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
120
121 /* Scouting faux-generate pass 0 */
122 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) {
123 /* We hit something illegal or unsupported. */
124 fp = org_fp;
125 goto out_addrs;
126 }
127
128 /*
129 * If we have seen a tail call, we need a second pass.
130 * This is because bpf_jit_emit_common_epilogue() is called
131 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
132 * We also need a second pass if we ended up with too large
133 * a program so as to ensure BPF_EXIT branches are in range.
134 */
135 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
136 cgctx.idx = 0;
137 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) {
138 fp = org_fp;
139 goto out_addrs;
140 }
141 }
142
143 bpf_jit_realloc_regs(&cgctx);
144 /*
145 * Pretend to build prologue, given the features we've seen. This will
146 * update ctgtx.idx as it pretends to output instructions, then we can
147 * calculate total size from idx.
148 */
149 bpf_jit_build_prologue(0, &cgctx);
150 addrs[fp->len] = cgctx.idx * 4;
151 bpf_jit_build_epilogue(0, &cgctx);
152
153 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
154 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
155
156 proglen = cgctx.idx * 4;
157 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
158
159 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
160 if (!bpf_hdr) {
161 fp = org_fp;
162 goto out_addrs;
163 }
164
165 if (extable_len)
166 fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len;
167
168 skip_init_ctx:
169 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
170
171 /* Code generation passes 1-2 */
172 for (pass = 1; pass < 3; pass++) {
173 /* Now build the prologue, body code & epilogue for real. */
174 cgctx.idx = 0;
175 cgctx.alt_exit_addr = 0;
176 bpf_jit_build_prologue(code_base, &cgctx);
177 if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass, extra_pass)) {
178 bpf_jit_binary_free(bpf_hdr);
179 fp = org_fp;
180 goto out_addrs;
181 }
182 bpf_jit_build_epilogue(code_base, &cgctx);
183
184 if (bpf_jit_enable > 1)
185 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
186 proglen - (cgctx.idx * 4), cgctx.seen);
187 }
188
189 if (bpf_jit_enable > 1)
190 /*
191 * Note that we output the base address of the code_base
192 * rather than image, since opcodes are in code_base.
193 */
194 bpf_jit_dump(flen, proglen, pass, code_base);
195
196 #ifdef CONFIG_PPC64_ELF_ABI_V1
197 /* Function descriptor nastiness: Address + TOC */
198 ((u64 *)image)[0] = (u64)code_base;
199 ((u64 *)image)[1] = local_paca->kernel_toc;
200 #endif
201
202 fp->bpf_func = (void *)image;
203 fp->jited = 1;
204 fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
205
206 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size);
207 if (!fp->is_func || extra_pass) {
208 bpf_jit_binary_lock_ro(bpf_hdr);
209 bpf_prog_fill_jited_linfo(fp, addrs);
210 out_addrs:
211 kfree(addrs);
212 kfree(jit_data);
213 fp->aux->jit_data = NULL;
214 } else {
215 jit_data->addrs = addrs;
216 jit_data->ctx = cgctx;
217 jit_data->proglen = proglen;
218 jit_data->image = image;
219 jit_data->header = bpf_hdr;
220 }
221
222 out:
223 if (bpf_blinded)
224 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
225
226 return fp;
227 }
228
229 /*
230 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
231 * this function, as this only applies to BPF_PROBE_MEM, for now.
232 */
bpf_add_extable_entry(struct bpf_prog * fp,u32 * image,int pass,struct codegen_context * ctx,int insn_idx,int jmp_off,int dst_reg)233 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
234 int insn_idx, int jmp_off, int dst_reg)
235 {
236 off_t offset;
237 unsigned long pc;
238 struct exception_table_entry *ex;
239 u32 *fixup;
240
241 /* Populate extable entries only in the last pass */
242 if (pass != 2)
243 return 0;
244
245 if (!fp->aux->extable ||
246 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
247 return -EINVAL;
248
249 pc = (unsigned long)&image[insn_idx];
250
251 fixup = (void *)fp->aux->extable -
252 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
253 (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
254
255 fixup[0] = PPC_RAW_LI(dst_reg, 0);
256 if (IS_ENABLED(CONFIG_PPC32))
257 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
258
259 fixup[BPF_FIXUP_LEN - 1] =
260 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
261
262 ex = &fp->aux->extable[ctx->exentry_idx];
263
264 offset = pc - (long)&ex->insn;
265 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
266 return -ERANGE;
267 ex->insn = offset;
268
269 offset = (long)fixup - (long)&ex->fixup;
270 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
271 return -ERANGE;
272 ex->fixup = offset;
273
274 ctx->exentry_idx++;
275 return 0;
276 }
277
