111 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)  in reg_set_seen()  argument
115 	if (r1 >= 6 && r1 <= 15 && !jit->seen_reg[r1])  in reg_set_seen()
116 		jit->seen_reg[r1] = 1;  in reg_set_seen()
121 	reg_set_seen(jit, b1);					\
124 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
132 	if (jit->prg_buf)					\
133 		*(u16 *) (jit->prg_buf + jit->prg) = (op);	\
134 	jit->prg += 2;						\
146 	if (jit->prg_buf)					\
147 		*(u32 *) (jit->prg_buf + jit->prg) = (op);	\
148 	jit->prg += 4;						\
195 	int __rel = ((target) - jit->prg) / 2;			\
201 	if (jit->prg_buf) {					\
202 		*(u32 *) (jit->prg_buf + jit->prg) = (op1);	\
203 		*(u16 *) (jit->prg_buf + jit->prg + 4) = (op2);	\
205 	jit->prg += 6;						\
233 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
242 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
251 	int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2;	\
259 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
266 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
290 	ret = jit->lit32;					\
291 	if (jit->prg_buf)					\
292 		*(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\
293 	jit->lit32 += 4;					\
299 	jit->seen |= SEEN_LITERAL;				\
300 	_EMIT_CONST_U32(val) - jit->base_ip;			\
306 	ret = jit->lit64;					\
307 	if (jit->prg_buf)					\
308 		*(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\
309 	jit->lit64 += 8;					\
315 	jit->seen |= SEEN_LITERAL;				\
316 	_EMIT_CONST_U64(val) - jit->base_ip;			\
332 static bool is_first_pass(struct bpf_jit *jit)  in is_first_pass()  argument
334 	return jit->size == 0;  in is_first_pass()
341 static bool is_codegen_pass(struct bpf_jit *jit)  in is_codegen_pass()  argument
343 	return jit->prg_buf;  in is_codegen_pass()
357 static bool can_use_rel(struct bpf_jit *jit, int off)  in can_use_rel()  argument
359 	return is_valid_rel(off - jit->prg);  in can_use_rel()
375 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)  in can_use_ldisp_for_lit32()  argument
377 	return is_valid_ldisp(jit->lit32 - jit->base_ip);  in can_use_ldisp_for_lit32()
384 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)  in can_use_ldisp_for_lit64()  argument
386 	return is_valid_ldisp(jit->lit64 - jit->base_ip);  in can_use_ldisp_for_lit64()
400 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)  in save_regs()  argument
415 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth)  in restore_regs()  argument
419 	if (jit->seen & SEEN_STACK)  in restore_regs()
433 static int get_start(struct bpf_jit *jit, int start)  in get_start()  argument
438 		if (jit->seen_reg[i])  in get_start()
447 static int get_end(struct bpf_jit *jit, int start)  in get_end()  argument
452 		if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])  in get_end()
455 	return jit->seen_reg[15] ? 15 : 14;  in get_end()
464 static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)  in save_restore_regs()  argument
469 	if (is_first_pass(jit)) {  in save_restore_regs()
474 		jit->prg += (last - re + 1) * save_restore_size;  in save_restore_regs()
479 		rs = get_start(jit, re);  in save_restore_regs()
482 		re = get_end(jit, rs + 1);  in save_restore_regs()
484 			save_regs(jit, rs, re);  in save_restore_regs()
486 			restore_regs(jit, rs, re, stack_depth);  in save_restore_regs()
491 static void bpf_skip(struct bpf_jit *jit, int size)  in bpf_skip()  argument
515 static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth)  in bpf_jit_prologue()  argument
517 	if (jit->seen & SEEN_TAIL_CALL) {  in bpf_jit_prologue()
525 		bpf_skip(jit, 6);  in bpf_jit_prologue()
528 	jit->tail_call_start = jit->prg;  in bpf_jit_prologue()
530 	save_restore_regs(jit, REGS_SAVE, stack_depth);  in bpf_jit_prologue()
532 	if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) {  in bpf_jit_prologue()
533 		if (!is_first_pass(jit) &&  in bpf_jit_prologue()
534 		    is_valid_ldisp(jit->size - (jit->prg + 2))) {  in bpf_jit_prologue()
537 			jit->base_ip = jit->prg;  in bpf_jit_prologue()
540 			EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start);  in bpf_jit_prologue()
541 			jit->base_ip = jit->lit32_start;  in bpf_jit_prologue()
545 	if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {  in bpf_jit_prologue()
546 		if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))  in bpf_jit_prologue()
553 		if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))  in bpf_jit_prologue()
563 static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)  in bpf_jit_epilogue()  argument
565 	jit->exit_ip = jit->prg;  in bpf_jit_epilogue()
569 	save_restore_regs(jit, REGS_RESTORE, stack_depth);  in bpf_jit_epilogue()
571 		jit->r14_thunk_ip = jit->prg;  in bpf_jit_epilogue()
574 		EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);  in bpf_jit_epilogue()
582 	    (is_first_pass(jit) || (jit->seen & SEEN_FUNC))) {  in bpf_jit_epilogue()
583 		jit->r1_thunk_ip = jit->prg;  in bpf_jit_epilogue()
586 		EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);  in bpf_jit_epilogue()
617 static int bpf_jit_probe_mem(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_probe_mem()  argument
629 	insn = jit->prg_buf + probe_prg;  in bpf_jit_probe_mem()
638 		if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))  in bpf_jit_probe_mem()
641 		ex = &fp->aux->extable[jit->excnt];  in bpf_jit_probe_mem()
644 		delta = jit->prg_buf + prg - (u8 *)&ex->insn;  in bpf_jit_probe_mem()
653 		delta = jit->prg_buf + nop_prg - (u8 *)&ex->fixup;  in bpf_jit_probe_mem()
660 		jit->excnt++;  in bpf_jit_probe_mem()
671 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_insn()  argument
678 	u32 *addrs = jit->addrs;  in bpf_jit_insn()
688 		probe_prg = jit->prg;  in bpf_jit_insn()
860 		if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {  in bpf_jit_insn()
868 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
893 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
901 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
927 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
936 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
959 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
968 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
993 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
1002 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
1151 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1156 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1161 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1166 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1173 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1180 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1187 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1194 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1243 			EMIT4_PCREL_RIC(0xa7040000, 4, jit->prg - 6);  in bpf_jit_insn()
1259 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1269 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1277 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1284 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1292 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1310 		jit->seen |= SEEN_FUNC;  in bpf_jit_insn()
1315 			EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);  in bpf_jit_insn()
1333 		jit->seen |= SEEN_TAIL_CALL;  in bpf_jit_insn()
1345 		patch_1_clrj = jit->prg;  in bpf_jit_insn()
1347 				 jit->prg);  in bpf_jit_insn()
1354 		if (jit->seen & SEEN_STACK)  in bpf_jit_insn()
1363 		patch_2_clij = jit->prg;  in bpf_jit_insn()
1365 				 2, jit->prg);  in bpf_jit_insn()
1381 		patch_3_brc = jit->prg;  in bpf_jit_insn()
1382 		EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);  in bpf_jit_insn()
1387 		save_restore_regs(jit, REGS_RESTORE, stack_depth);  in bpf_jit_insn()
1397 		_EMIT4(0x47f01000 + jit->tail_call_start);  in bpf_jit_insn()
1399 		if (jit->prg_buf) {  in bpf_jit_insn()
1400 			*(u16 *)(jit->prg_buf + patch_1_clrj + 2) =  in bpf_jit_insn()
1401 				(jit->prg - patch_1_clrj) >> 1;  in bpf_jit_insn()
1402 			*(u16 *)(jit->prg_buf + patch_2_clij + 2) =  in bpf_jit_insn()
1403 				(jit->prg - patch_2_clij) >> 1;  in bpf_jit_insn()
1404 			*(u16 *)(jit->prg_buf + patch_3_brc + 2) =  in bpf_jit_insn()
1405 				(jit->prg - patch_3_brc) >> 1;  in bpf_jit_insn()
1413 		if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip))  in bpf_jit_insn()
1415 			EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip);  in bpf_jit_insn()
1418 			EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip);  in bpf_jit_insn()
1554 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1555 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1572 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1573 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1590 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1591 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1607 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1608 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1632 		nop_prg = jit->prg;  in bpf_jit_insn()
1635 		err = bpf_jit_probe_mem(jit, fp, probe_prg, nop_prg);  in bpf_jit_insn()
1646 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i)  in bpf_is_new_addr_sane()  argument
1649 	if (is_first_pass(jit))  in bpf_is_new_addr_sane()
1653 	if (is_codegen_pass(jit))  in bpf_is_new_addr_sane()
1654 		return jit->addrs[i] == jit->prg;  in bpf_is_new_addr_sane()
1657 	return jit->addrs[i] >= jit->prg;  in bpf_is_new_addr_sane()
1663 static int bpf_set_addr(struct bpf_jit *jit, int i)  in bpf_set_addr()  argument
1667 	if (is_codegen_pass(jit)) {  in bpf_set_addr()
1668 		delta = jit->prg - jit->addrs[i];  in bpf_set_addr()
1670 			bpf_skip(jit, -delta);  in bpf_set_addr()
1672 	if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))  in bpf_set_addr()
1674 	jit->addrs[i] = jit->prg;  in bpf_set_addr()
1681 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_prog()  argument
1686 	jit->lit32 = jit->lit32_start;  in bpf_jit_prog()
1687 	jit->lit64 = jit->lit64_start;  in bpf_jit_prog()
1688 	jit->prg = 0;  in bpf_jit_prog()
1689 	jit->excnt = 0;  in bpf_jit_prog()
1691 	bpf_jit_prologue(jit, stack_depth);  in bpf_jit_prog()
1692 	if (bpf_set_addr(jit, 0) < 0)  in bpf_jit_prog()
1695 		insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth);  in bpf_jit_prog()
1699 		if (bpf_set_addr(jit, i + insn_count) < 0)  in bpf_jit_prog()
1702 	bpf_jit_epilogue(jit, stack_depth);  in bpf_jit_prog()
1704 	lit32_size = jit->lit32 - jit->lit32_start;  in bpf_jit_prog()
1705 	lit64_size = jit->lit64 - jit->lit64_start;  in bpf_jit_prog()
1706 	jit->lit32_start = jit->prg;  in bpf_jit_prog()
1708 		jit->lit32_start = ALIGN(jit->lit32_start, 4);  in bpf_jit_prog()
1709 	jit->lit64_start = jit->lit32_start + lit32_size;  in bpf_jit_prog()
1711 		jit->lit64_start = ALIGN(jit->lit64_start, 8);  in bpf_jit_prog()
1712 	jit->size = jit->lit64_start + lit64_size;  in bpf_jit_prog()
1713 	jit->size_prg = jit->prg;  in bpf_jit_prog()
1716 			 jit->excnt != fp->aux->num_exentries))  in bpf_jit_prog()
1734 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,  in bpf_jit_alloc()  argument
1744 	code_size = roundup(jit->size,  in bpf_jit_alloc()
1748 	header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf,  in bpf_jit_alloc()
1753 		(jit->prg_buf + code_size);  in bpf_jit_alloc()
1768 	struct bpf_jit jit;  in bpf_int_jit_compile()  local
1796 		jit = jit_data->ctx;  in bpf_int_jit_compile()
1803 	memset(&jit, 0, sizeof(jit));  in bpf_int_jit_compile()
1804 	jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);  in bpf_int_jit_compile()
1805 	if (jit.addrs == NULL) {  in bpf_int_jit_compile()
1815 		if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {  in bpf_int_jit_compile()
1823 	header = bpf_jit_alloc(&jit, fp);  in bpf_int_jit_compile()
1829 	if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {  in bpf_int_jit_compile()
1835 		bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);  in bpf_int_jit_compile()
1836 		print_fn_code(jit.prg_buf, jit.size_prg);  in bpf_int_jit_compile()
1842 		jit_data->ctx = jit;  in bpf_int_jit_compile()
1845 	fp->bpf_func = (void *) jit.prg_buf;  in bpf_int_jit_compile()
1847 	fp->jited_len = jit.size;  in bpf_int_jit_compile()
1850 		bpf_prog_fill_jited_linfo(fp, jit.addrs + 1);  in bpf_int_jit_compile()
1852 		kvfree(jit.addrs);  in bpf_int_jit_compile()