27 static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image,  in bpf_jit_fixup_addresses()  argument
30 	const struct bpf_insn *insn = fp->insnsi;  in bpf_jit_fixup_addresses()
36 	for (i = 0; i < fp->len; i++) {  in bpf_jit_fixup_addresses()
48 			ret = bpf_jit_get_func_addr(fp, &insn[i], true,  in bpf_jit_fixup_addresses()
123 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)  in bpf_int_jit_compile()  argument
135 	struct bpf_prog *org_fp = fp;  in bpf_int_jit_compile()
142 	if (!fp->jit_requested)  in bpf_int_jit_compile()
151 		fp = tmp_fp;  in bpf_int_jit_compile()
154 	jit_data = fp->aux->jit_data;  in bpf_int_jit_compile()
158 			fp = org_fp;  in bpf_int_jit_compile()
161 		fp->aux->jit_data = jit_data;  in bpf_int_jit_compile()
164 	flen = fp->len;  in bpf_int_jit_compile()
177 		fp = org_fp;  in bpf_int_jit_compile()
185 	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);  in bpf_int_jit_compile()
188 	if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {  in bpf_int_jit_compile()
190 		fp = org_fp;  in bpf_int_jit_compile()
203 		if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {  in bpf_int_jit_compile()
204 			fp = org_fp;  in bpf_int_jit_compile()
216 	addrs[fp->len] = cgctx.idx * 4;  in bpf_int_jit_compile()
219 	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;  in bpf_int_jit_compile()
220 	extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);  in bpf_int_jit_compile()
227 		fp = org_fp;  in bpf_int_jit_compile()
232 		fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len;  in bpf_int_jit_compile()
247 		bpf_jit_fixup_addresses(fp, code_base, &cgctx, addrs);  in bpf_int_jit_compile()
259 		if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {  in bpf_int_jit_compile()
261 			fp = org_fp;  in bpf_int_jit_compile()
285 	fp->bpf_func = (void *)image;  in bpf_int_jit_compile()
286 	fp->jited = 1;  in bpf_int_jit_compile()
287 	fp->jited_len = proglen + FUNCTION_DESCR_SIZE;  in bpf_int_jit_compile()
290 	if (!fp->is_func || extra_pass) {  in bpf_int_jit_compile()
292 		bpf_prog_fill_jited_linfo(fp, addrs);  in bpf_int_jit_compile()
296 		fp->aux->jit_data = NULL;  in bpf_int_jit_compile()
307 		bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);  in bpf_int_jit_compile()
309 	return fp;  in bpf_int_jit_compile()
316 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,  in bpf_add_extable_entry()  argument
328 	if (!fp->aux->extable ||  in bpf_add_extable_entry()
329 	    WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))  in bpf_add_extable_entry()
334 	fixup = (void *)fp->aux->extable -  in bpf_add_extable_entry()
335 		(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +  in bpf_add_extable_entry()
345 	ex = &fp->aux->extable[ctx->exentry_idx];  in bpf_add_extable_entry()