112 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)  in reg_set_seen()  argument
116 	if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15)  in reg_set_seen()
117 		jit->seen_reg[r1] = 1;  in reg_set_seen()
122 	reg_set_seen(jit, b1);					\
125 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
133 	if (jit->prg_buf)					\
134 		*(u16 *) (jit->prg_buf + jit->prg) = op;	\
135 	jit->prg += 2;						\
147 	if (jit->prg_buf)					\
148 		*(u32 *) (jit->prg_buf + jit->prg) = op;	\
149 	jit->prg += 4;						\
196 	if (jit->prg_buf) {					\
197 		*(u32 *) (jit->prg_buf + jit->prg) = op1;	\
198 		*(u16 *) (jit->prg_buf + jit->prg + 4) = op2;	\
200 	jit->prg += 6;						\
228 	int rel = (jit->labels[label] - jit->prg) >> 1;		\
237 	int rel = (jit->labels[label] - jit->prg) >> 1;		\
255 	int rel = (target - jit->prg) / 2;			\
262 	int rel = (target - jit->prg) / 2;			\
281 	ret = jit->lit - jit->base_ip;				\
282 	jit->seen |= SEEN_LITERAL;				\
283 	if (jit->prg_buf)					\
284 		*(u32 *) (jit->prg_buf + jit->lit) = (u32) val;	\
285 	jit->lit += 4;						\
292 	ret = jit->lit - jit->base_ip;				\
293 	jit->seen |= SEEN_LITERAL;				\
294 	if (jit->prg_buf)					\
295 		*(u64 *) (jit->prg_buf + jit->lit) = (u64) val;	\
296 	jit->lit += 8;						\
320 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)  in save_regs()  argument
335 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth)  in restore_regs()  argument
339 	if (jit->seen & SEEN_STACK)  in restore_regs()
353 static int get_start(struct bpf_jit *jit, int start)  in get_start()  argument
358 		if (jit->seen_reg[i])  in get_start()
367 static int get_end(struct bpf_jit *jit, int start)  in get_end()  argument
372 		if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])  in get_end()
375 	return jit->seen_reg[15] ? 15 : 14;  in get_end()
384 static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)  in save_restore_regs()  argument
390 		rs = get_start(jit, re);  in save_restore_regs()
393 		re = get_end(jit, rs + 1);  in save_restore_regs()
395 			save_regs(jit, rs, re);  in save_restore_regs()
397 			restore_regs(jit, rs, re, stack_depth);  in save_restore_regs()
408 static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth)  in bpf_jit_prologue()  argument
410 	if (jit->seen & SEEN_TAIL_CALL) {  in bpf_jit_prologue()
419 	jit->tail_call_start = jit->prg;  in bpf_jit_prologue()
421 	save_restore_regs(jit, REGS_SAVE, stack_depth);  in bpf_jit_prologue()
423 	if (jit->seen & SEEN_LITERAL) {  in bpf_jit_prologue()
426 		jit->base_ip = jit->prg;  in bpf_jit_prologue()
429 	if (jit->seen & SEEN_STACK) {  in bpf_jit_prologue()
430 		if (jit->seen & SEEN_FUNC)  in bpf_jit_prologue()
437 		if (jit->seen & SEEN_FUNC)  in bpf_jit_prologue()
447 static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)  in bpf_jit_epilogue()  argument
450 	if (jit->seen & SEEN_RET0) {  in bpf_jit_epilogue()
451 		jit->ret0_ip = jit->prg;  in bpf_jit_epilogue()
455 	jit->exit_ip = jit->prg;  in bpf_jit_epilogue()
459 	save_restore_regs(jit, REGS_RESTORE, stack_depth);  in bpf_jit_epilogue()
461 		jit->r14_thunk_ip = jit->prg;  in bpf_jit_epilogue()
465 			EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);  in bpf_jit_epilogue()
468 			EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);  in bpf_jit_epilogue()
479 	    (jit->seen & SEEN_FUNC)) {  in bpf_jit_epilogue()
480 		jit->r1_thunk_ip = jit->prg;  in bpf_jit_epilogue()
484 			EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);  in bpf_jit_epilogue()
505 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_insn()  argument
512 	u32 *addrs = jit->addrs;  in bpf_jit_insn()
518 		jit->seen |= SEEN_REG_AX;  in bpf_jit_insn()
921 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
926 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
931 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
936 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
943 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
950 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
957 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
964 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
973 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
979 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
987 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
994 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1000 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1007 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1025 		jit->seen |= SEEN_FUNC;  in bpf_jit_insn()
1031 			EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);  in bpf_jit_insn()
1047 		jit->seen |= SEEN_TAIL_CALL;  in bpf_jit_insn()
1066 		if (jit->seen & SEEN_STACK)  in bpf_jit_insn()
1097 		save_restore_regs(jit, REGS_RESTORE, fp->aux->stack_depth);  in bpf_jit_insn()
1107 		_EMIT4(0x47f01000 + jit->tail_call_start);  in bpf_jit_insn()
1109 		jit->labels[0] = jit->prg;  in bpf_jit_insn()
1113 		if (last && !(jit->seen & SEEN_RET0))  in bpf_jit_insn()
1116 		EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);  in bpf_jit_insn()
1291 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_prog()  argument
1296 	jit->lit = jit->lit_start;  in bpf_jit_prog()
1297 	jit->prg = 0;  in bpf_jit_prog()
1299 	bpf_jit_prologue(jit, fp->aux->stack_depth);  in bpf_jit_prog()
1301 		insn_count = bpf_jit_insn(jit, fp, i, extra_pass);  in bpf_jit_prog()
1305 		jit->addrs[i + insn_count] = jit->prg;  in bpf_jit_prog()
1307 	bpf_jit_epilogue(jit, fp->aux->stack_depth);  in bpf_jit_prog()
1309 	jit->lit_start = jit->prg;  in bpf_jit_prog()
1310 	jit->size = jit->lit;  in bpf_jit_prog()
1311 	jit->size_prg = jit->prg;  in bpf_jit_prog()
1336 	struct bpf_jit jit;  in bpf_int_jit_compile()  local
1364 		jit = jit_data->ctx;  in bpf_int_jit_compile()
1371 	memset(&jit, 0, sizeof(jit));  in bpf_int_jit_compile()
1372 	jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);  in bpf_int_jit_compile()
1373 	if (jit.addrs == NULL) {  in bpf_int_jit_compile()
1383 		if (bpf_jit_prog(&jit, fp, extra_pass)) {  in bpf_int_jit_compile()
1391 	if (jit.size >= BPF_SIZE_MAX) {  in bpf_int_jit_compile()
1396 	header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);  in bpf_int_jit_compile()
1402 	if (bpf_jit_prog(&jit, fp, extra_pass)) {  in bpf_int_jit_compile()
1408 		bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);  in bpf_int_jit_compile()
1409 		print_fn_code(jit.prg_buf, jit.size_prg);  in bpf_int_jit_compile()
1415 		jit_data->ctx = jit;  in bpf_int_jit_compile()
1418 	fp->bpf_func = (void *) jit.prg_buf;  in bpf_int_jit_compile()
1420 	fp->jited_len = jit.size;  in bpf_int_jit_compile()
1423 		bpf_prog_fill_jited_linfo(fp, jit.addrs + 1);  in bpf_int_jit_compile()
1425 		kfree(jit.addrs);  in bpf_int_jit_compile()