208 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)  in add_2reg()  argument
210 	return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);  in add_2reg()
676 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)  in emit_mov_reg()  argument
682 		EMIT_mov(dst_reg, src_reg);  in emit_mov_reg()
685 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_mov_reg()
686 			EMIT1(add_2mod(0x40, dst_reg, src_reg));  in emit_mov_reg()
687 		EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));  in emit_mov_reg()
716 static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)  in maybe_emit_mod()  argument
721 		EMIT1(add_2mod(0x48, dst_reg, src_reg));  in maybe_emit_mod()
722 	else if (is_ereg(dst_reg) || is_ereg(src_reg))  in maybe_emit_mod()
723 		EMIT1(add_2mod(0x40, dst_reg, src_reg));  in maybe_emit_mod()
742 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx()  argument
749 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);  in emit_ldx()
753 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);  in emit_ldx()
757 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_ldx()
758 			EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);  in emit_ldx()
764 		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);  in emit_ldx()
767 	emit_insn_suffix(&prog, src_reg, dst_reg, off);  in emit_ldx()
772 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx()  argument
779 		if (is_ereg(dst_reg) || is_ereg_8l(src_reg))  in emit_stx()
781 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);  in emit_stx()
786 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
787 			EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
792 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
793 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
798 		EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);  in emit_stx()
801 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_stx()
806 		       u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)  in emit_atomic()  argument
812 	maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW);  in emit_atomic()
840 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_atomic()
864 		if (insn->dst_reg == BPF_REG_6 || insn->src_reg == BPF_REG_6)  in detect_reg_usage()
866 		if (insn->dst_reg == BPF_REG_7 || insn->src_reg == BPF_REG_7)  in detect_reg_usage()
868 		if (insn->dst_reg == BPF_REG_8 || insn->src_reg == BPF_REG_8)  in detect_reg_usage()
870 		if (insn->dst_reg == BPF_REG_9 || insn->src_reg == BPF_REG_9)  in detect_reg_usage()
932 		u32 src_reg = insn->src_reg;  in do_jit()  local
952 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
955 			EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
962 				     dst_reg, src_reg);  in do_jit()
1050 				if (src_reg == BPF_REG_0 ||  in do_jit()
1051 				    src_reg == BPF_REG_3) {  in do_jit()
1053 					EMIT_mov(AUX_REG, src_reg);  in do_jit()
1054 					src_reg = AUX_REG;  in do_jit()
1059 				src_reg = AUX_REG;  in do_jit()
1073 			maybe_emit_1mod(&prog, src_reg, is64);  in do_jit()
1074 			EMIT2(0xF7, add_1reg(0xF0, src_reg));  in do_jit()
1110 			maybe_emit_mod(&prog, src_reg, dst_reg,  in do_jit()
1114 			EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));  in do_jit()
1148 			if (src_reg != BPF_REG_4) { /* common case */  in do_jit()
1152 				EMIT_mov(BPF_REG_4, src_reg);  in do_jit()
1162 			if (src_reg != BPF_REG_4)  in do_jit()
1268 			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);  in do_jit()
1301 				maybe_emit_mod(&prog, src_reg, AUX_REG, true);  in do_jit()
1302 				EMIT2(0x39, add_2reg(0xC0, src_reg, AUX_REG));  in do_jit()
1308 				emit_mov_reg(&prog, true, AUX_REG, src_reg);  in do_jit()
1330 			emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);  in do_jit()
1382 				u32 real_src_reg = src_reg;  in do_jit()
1393 				if (src_reg == BPF_REG_0)  in do_jit()
1429 			err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,  in do_jit()
1487 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1489 			EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1495 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1497 			EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()