203 static u8 add_1reg(u8 byte, u32 dst_reg)  in add_1reg()  argument
205 	return byte + reg2hex[dst_reg];  in add_1reg()
209 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)  in add_2reg()  argument
211 	return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);  in add_2reg()
625 			   u32 dst_reg, const u32 imm32)  in emit_mov_imm32()  argument
636 		b1 = add_1mod(0x48, dst_reg);  in emit_mov_imm32()
639 		EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32);  in emit_mov_imm32()
648 		if (is_ereg(dst_reg))  in emit_mov_imm32()
649 			EMIT1(add_2mod(0x40, dst_reg, dst_reg));  in emit_mov_imm32()
652 		EMIT2(b2, add_2reg(b3, dst_reg, dst_reg));  in emit_mov_imm32()
657 	if (is_ereg(dst_reg))  in emit_mov_imm32()
658 		EMIT1(add_1mod(0x40, dst_reg));  in emit_mov_imm32()
659 	EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);  in emit_mov_imm32()
664 static void emit_mov_imm64(u8 **pprog, u32 dst_reg,  in emit_mov_imm64()  argument
676 		emit_mov_imm32(&prog, false, dst_reg, imm32_lo);  in emit_mov_imm64()
679 		EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));  in emit_mov_imm64()
687 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)  in emit_mov_reg()  argument
693 		EMIT_mov(dst_reg, src_reg);  in emit_mov_reg()
696 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_mov_reg()
697 			EMIT1(add_2mod(0x40, dst_reg, src_reg));  in emit_mov_reg()
698 		EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));  in emit_mov_reg()
704 static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg,  in emit_movsx_reg()  argument
712 			EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbe,  in emit_movsx_reg()
713 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
715 			EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbf,  in emit_movsx_reg()
716 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
718 			EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x63,  in emit_movsx_reg()
719 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
723 			EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe,  in emit_movsx_reg()
724 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
726 			if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_movsx_reg()
727 				EMIT1(add_2mod(0x40, src_reg, dst_reg));  in emit_movsx_reg()
728 			EMIT3(add_2mod(0x0f, src_reg, dst_reg), 0xbf,  in emit_movsx_reg()
729 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
759 static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)  in maybe_emit_mod()  argument
764 		EMIT1(add_2mod(0x48, dst_reg, src_reg));  in maybe_emit_mod()
765 	else if (is_ereg(dst_reg) || is_ereg(src_reg))  in maybe_emit_mod()
766 		EMIT1(add_2mod(0x40, dst_reg, src_reg));  in maybe_emit_mod()
785 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx()  argument
792 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);  in emit_ldx()
796 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);  in emit_ldx()
800 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_ldx()
801 			EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);  in emit_ldx()
807 		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);  in emit_ldx()
810 	emit_insn_suffix(&prog, src_reg, dst_reg, off);  in emit_ldx()
815 static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldsx()  argument
822 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBE);  in emit_ldsx()
826 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBF);  in emit_ldsx()
830 		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x63);  in emit_ldsx()
833 	emit_insn_suffix(&prog, src_reg, dst_reg, off);  in emit_ldsx()
838 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx()  argument
845 		if (is_ereg(dst_reg) || is_ereg_8l(src_reg))  in emit_stx()
847 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);  in emit_stx()
852 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
853 			EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
858 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
859 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
864 		EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);  in emit_stx()
867 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_stx()
872 		       u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)  in emit_atomic()  argument
878 	maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW);  in emit_atomic()
906 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_atomic()
930 		if (insn->dst_reg == BPF_REG_6 || insn->src_reg == BPF_REG_6)  in detect_reg_usage()
932 		if (insn->dst_reg == BPF_REG_7 || insn->src_reg == BPF_REG_7)  in detect_reg_usage()
934 		if (insn->dst_reg == BPF_REG_8 || insn->src_reg == BPF_REG_8)  in detect_reg_usage()
936 		if (insn->dst_reg == BPF_REG_9 || insn->src_reg == BPF_REG_9)  in detect_reg_usage()
1008 static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)  in emit_shiftx()  argument
1011 	bool r = is_ereg(dst_reg);  in emit_shiftx()
1015 	EMIT2(0xf7, add_2reg(0xC0, dst_reg, dst_reg));  in emit_shiftx()
1056 		u32 dst_reg = insn->dst_reg;  in do_jit()  local
1078 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1081 			EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1089 					     dst_reg, src_reg);  in do_jit()
1093 					       dst_reg, src_reg);  in do_jit()
1099 			maybe_emit_1mod(&prog, dst_reg,  in do_jit()
1101 			EMIT2(0xF7, add_1reg(0xD8, dst_reg));  in do_jit()
1114 			maybe_emit_1mod(&prog, dst_reg,  in do_jit()
1145 				EMIT3(0x83, add_1reg(b3, dst_reg), imm32);  in do_jit()
1146 			else if (is_axreg(dst_reg))  in do_jit()
1149 				EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);  in do_jit()
1155 				       dst_reg, imm32);  in do_jit()
1159 			emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm);  in do_jit()
1175 			if (dst_reg != BPF_REG_0)  in do_jit()
1177 			if (dst_reg != BPF_REG_3)  in do_jit()
1193 			if (dst_reg != BPF_REG_0)  in do_jit()
1195 				emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg);  in do_jit()
1219 			    dst_reg != BPF_REG_3)  in do_jit()
1221 				emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3);  in do_jit()
1223 				 dst_reg != BPF_REG_0)  in do_jit()
1225 				emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0);  in do_jit()
1227 			if (dst_reg != BPF_REG_3)  in do_jit()
1229 			if (dst_reg != BPF_REG_0)  in do_jit()
1236 			maybe_emit_mod(&prog, dst_reg, dst_reg,  in do_jit()
1241 				EMIT3(0x6B, add_2reg(0xC0, dst_reg, dst_reg),  in do_jit()
1246 					    add_2reg(0xC0, dst_reg, dst_reg),  in do_jit()
1252 			maybe_emit_mod(&prog, src_reg, dst_reg,  in do_jit()
1256 			EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));  in do_jit()
1266 			maybe_emit_1mod(&prog, dst_reg,  in do_jit()
1271 				EMIT2(0xD1, add_1reg(b3, dst_reg));  in do_jit()
1273 				EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);  in do_jit()
1300 				emit_shiftx(&prog, dst_reg, src_reg, w, op);  in do_jit()
1307 				if (dst_reg == BPF_REG_4) {  in do_jit()
1309 					EMIT_mov(AUX_REG, dst_reg);  in do_jit()
1310 					dst_reg = AUX_REG;  in do_jit()
1319 			maybe_emit_1mod(&prog, dst_reg,  in do_jit()
1323 			EMIT2(0xD3, add_1reg(b3, dst_reg));  in do_jit()
1326 				if (insn->dst_reg == BPF_REG_4)  in do_jit()
1328 					EMIT_mov(insn->dst_reg, AUX_REG);  in do_jit()
1341 				if (is_ereg(dst_reg))  in do_jit()
1343 				EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);  in do_jit()
1346 				if (is_ereg(dst_reg))  in do_jit()
1350 				EMIT1(add_2reg(0xC0, dst_reg, dst_reg));  in do_jit()
1354 				if (is_ereg(dst_reg))  in do_jit()
1358 				EMIT1(add_1reg(0xC8, dst_reg));  in do_jit()
1362 				EMIT3(add_1mod(0x48, dst_reg), 0x0F,  in do_jit()
1363 				      add_1reg(0xC8, dst_reg));  in do_jit()
1375 				if (is_ereg(dst_reg))  in do_jit()
1379 				EMIT1(add_2reg(0xC0, dst_reg, dst_reg));  in do_jit()
1383 				if (is_ereg(dst_reg))  in do_jit()
1385 				EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));  in do_jit()
1400 			if (is_ereg(dst_reg))  in do_jit()
1406 			if (is_ereg(dst_reg))  in do_jit()
1412 			if (is_ereg(dst_reg))  in do_jit()
1418 			EMIT2(add_1mod(0x48, dst_reg), 0xC7);  in do_jit()
1421 				EMIT2(add_1reg(0x40, dst_reg), insn->off);  in do_jit()
1423 				EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);  in do_jit()
1433 			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);  in do_jit()
1490 				emit_mov_imm32(&prog, false, dst_reg, 0);  in do_jit()
1500 				emit_ldsx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);  in do_jit()
1502 				emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);  in do_jit()
1512 				if (insn->off && src_reg != dst_reg) {  in do_jit()
1545 				if (dst_reg > BPF_REG_9) {  in do_jit()
1557 				ex->fixup = (prog - start_of_ldx) | (reg2pt_regs[dst_reg] << 8);  in do_jit()
1568 				u32 real_dst_reg = dst_reg;  in do_jit()
1580 				if (dst_reg == BPF_REG_0)  in do_jit()
1614 			err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,  in do_jit()
1679 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1681 			EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1687 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1689 			EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1695 			maybe_emit_1mod(&prog, dst_reg,  in do_jit()
1697 			EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);  in do_jit()
1722 				maybe_emit_mod(&prog, dst_reg, dst_reg,  in do_jit()
1724 				EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg));  in do_jit()
1729 			maybe_emit_1mod(&prog, dst_reg,  in do_jit()
1733 				EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);  in do_jit()
1735 				EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);  in do_jit()