1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <asm/asm-compat.h>
26 /* Fix updated addresses (for subprog calls, ldimm64, et al) during extra pass */
30 	const struct bpf_insn *insn = fp->insnsi;  in bpf_jit_fixup_addresses()
36 	for (i = 0; i < fp->len; i++) {  in bpf_jit_fixup_addresses()
38 		 * During the extra pass, only the branch target addresses for  in bpf_jit_fixup_addresses()
55 			 * Save ctx->idx as this would currently point to the  in bpf_jit_fixup_addresses()
60 			tmp_idx = ctx->idx;  in bpf_jit_fixup_addresses()
61 			ctx->idx = addrs[i] / 4;  in bpf_jit_fixup_addresses()
67 			 * Restore ctx->idx here. This is safe as the length  in bpf_jit_fixup_addresses()
70 			ctx->idx = tmp_idx;  in bpf_jit_fixup_addresses()
72 			tmp_idx = ctx->idx;  in bpf_jit_fixup_addresses()
73 			ctx->idx = addrs[i] / 4;  in bpf_jit_fixup_addresses()
75 			PPC_LI32(bpf_to_ppc(insn[i].dst_reg) - 1, (u32)insn[i + 1].imm);  in bpf_jit_fixup_addresses()
77 			for (j = ctx->idx - addrs[i] / 4; j < 4; j++)  in bpf_jit_fixup_addresses()
80 			func_addr = ((u64)(u32)insn[i].imm) | (((u64)(u32)insn[i + 1].imm) << 32);  in bpf_jit_fixup_addresses()
83 			for (j = ctx->idx - addrs[i] / 4; j < 5; j++)  in bpf_jit_fixup_addresses()
86 			ctx->idx = tmp_idx;  in bpf_jit_fixup_addresses()
96 	if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {  in bpf_jit_emit_exit_insn()
98 	} else if (ctx->alt_exit_addr) {  in bpf_jit_emit_exit_insn()
99 		if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))  in bpf_jit_emit_exit_insn()
100 			return -1;  in bpf_jit_emit_exit_insn()
101 		PPC_JMP(ctx->alt_exit_addr);  in bpf_jit_emit_exit_insn()
103 		ctx->alt_exit_addr = ctx->idx * 4;  in bpf_jit_emit_exit_insn()
132 	int pass;  in bpf_int_jit_compile()  local
142 	if (!fp->jit_requested)  in bpf_int_jit_compile()
154 	jit_data = fp->aux->jit_data;  in bpf_int_jit_compile()
161 		fp->aux->jit_data = jit_data;  in bpf_int_jit_compile()
164 	flen = fp->len;  in bpf_int_jit_compile()
165 	addrs = jit_data->addrs;  in bpf_int_jit_compile()
167 		cgctx = jit_data->ctx;  in bpf_int_jit_compile()
168 		image = jit_data->image;  in bpf_int_jit_compile()
169 		bpf_hdr = jit_data->header;  in bpf_int_jit_compile()
170 		proglen = jit_data->proglen;  in bpf_int_jit_compile()
175 	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);  in bpf_int_jit_compile()
185 	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);  in bpf_int_jit_compile()
187 	/* Scouting faux-generate pass 0 */  in bpf_int_jit_compile()
195 	 * If we have seen a tail call, we need a second pass.  in bpf_int_jit_compile()
197 	 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.  in bpf_int_jit_compile()
198 	 * We also need a second pass if we ended up with too large  in bpf_int_jit_compile()
216 	addrs[fp->len] = cgctx.idx * 4;  in bpf_int_jit_compile()
219 	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;  in bpf_int_jit_compile()
220 	extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);  in bpf_int_jit_compile()
232 		fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len;  in bpf_int_jit_compile()
253 	/* Code generation passes 1-2 */  in bpf_int_jit_compile()
254 	for (pass = 1; pass < 3; pass++) {  in bpf_int_jit_compile()
259 		if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {  in bpf_int_jit_compile()
266 		if (bpf_jit_enable > 1)  in bpf_int_jit_compile()
267 			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,  in bpf_int_jit_compile()
268 				proglen - (cgctx.idx * 4), cgctx.seen);  in bpf_int_jit_compile()
272 	if (bpf_jit_enable > 1)  in bpf_int_jit_compile()
277 		bpf_jit_dump(flen, proglen, pass, code_base);  in bpf_int_jit_compile()
282 	((u64 *)image)[1] = local_paca->kernel_toc;  in bpf_int_jit_compile()
285 	fp->bpf_func = (void *)image;  in bpf_int_jit_compile()
286 	fp->jited = 1;  in bpf_int_jit_compile()
287 	fp->jited_len = proglen + FUNCTION_DESCR_SIZE;  in bpf_int_jit_compile()
289 	bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size);  in bpf_int_jit_compile()
290 	if (!fp->is_func || extra_pass) {  in bpf_int_jit_compile()
296 		fp->aux->jit_data = NULL;  in bpf_int_jit_compile()
298 		jit_data->addrs = addrs;  in bpf_int_jit_compile()
299 		jit_data->ctx = cgctx;  in bpf_int_jit_compile()
300 		jit_data->proglen = proglen;  in bpf_int_jit_compile()
301 		jit_data->image = image;  in bpf_int_jit_compile()
302 		jit_data->header = bpf_hdr;  in bpf_int_jit_compile()
316 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,  in bpf_add_extable_entry()  argument
324 	/* Populate extable entries only in the last pass */  in bpf_add_extable_entry()
325 	if (pass != 2)  in bpf_add_extable_entry()
328 	if (!fp->aux->extable ||  in bpf_add_extable_entry()
329 	    WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))  in bpf_add_extable_entry()
330 		return -EINVAL;  in bpf_add_extable_entry()
334 	fixup = (void *)fp->aux->extable -  in bpf_add_extable_entry()
335 		(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +  in bpf_add_extable_entry()
336 		(ctx->exentry_idx * BPF_FIXUP_LEN * 4);  in bpf_add_extable_entry()
340 		fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */  in bpf_add_extable_entry()
342 	fixup[BPF_FIXUP_LEN - 1] =  in bpf_add_extable_entry()
343 		PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);  in bpf_add_extable_entry()
345 	ex = &fp->aux->extable[ctx->exentry_idx];  in bpf_add_extable_entry()
347 	offset = pc - (long)&ex->insn;  in bpf_add_extable_entry()
349 		return -ERANGE;  in bpf_add_extable_entry()
350 	ex->insn = offset;  in bpf_add_extable_entry()
352 	offset = (long)fixup - (long)&ex->fixup;  in bpf_add_extable_entry()
354 		return -ERANGE;  in bpf_add_extable_entry()
355 	ex->fixup = offset;  in bpf_add_extable_entry()
357 	ctx->exentry_idx++;  in bpf_add_extable_entry()