209 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)  in add_2reg()  argument
211 	return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);  in add_2reg()
687 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)  in emit_mov_reg()  argument
693 		EMIT_mov(dst_reg, src_reg);  in emit_mov_reg()
696 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_mov_reg()
697 			EMIT1(add_2mod(0x40, dst_reg, src_reg));  in emit_mov_reg()
698 		EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));  in emit_mov_reg()
705 			   u32 src_reg)  in emit_movsx_reg()  argument
712 			EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbe,  in emit_movsx_reg()
713 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
715 			EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbf,  in emit_movsx_reg()
716 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
718 			EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x63,  in emit_movsx_reg()
719 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
723 			EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe,  in emit_movsx_reg()
724 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
726 			if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_movsx_reg()
727 				EMIT1(add_2mod(0x40, src_reg, dst_reg));  in emit_movsx_reg()
728 			EMIT3(add_2mod(0x0f, src_reg, dst_reg), 0xbf,  in emit_movsx_reg()
729 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
759 static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)  in maybe_emit_mod()  argument
764 		EMIT1(add_2mod(0x48, dst_reg, src_reg));  in maybe_emit_mod()
765 	else if (is_ereg(dst_reg) || is_ereg(src_reg))  in maybe_emit_mod()
766 		EMIT1(add_2mod(0x40, dst_reg, src_reg));  in maybe_emit_mod()
785 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx()  argument
792 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);  in emit_ldx()
796 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);  in emit_ldx()
800 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_ldx()
801 			EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);  in emit_ldx()
807 		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);  in emit_ldx()
810 	emit_insn_suffix(&prog, src_reg, dst_reg, off);  in emit_ldx()
815 static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldsx()  argument
822 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBE);  in emit_ldsx()
826 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBF);  in emit_ldsx()
830 		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x63);  in emit_ldsx()
833 	emit_insn_suffix(&prog, src_reg, dst_reg, off);  in emit_ldsx()
838 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx()  argument
845 		if (is_ereg(dst_reg) || is_ereg_8l(src_reg))  in emit_stx()
847 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);  in emit_stx()
852 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
853 			EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
858 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
859 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
864 		EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);  in emit_stx()
867 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_stx()
872 		       u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)  in emit_atomic()  argument
878 	maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW);  in emit_atomic()
906 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_atomic()
930 		if (insn->dst_reg == BPF_REG_6 || insn->src_reg == BPF_REG_6)  in detect_reg_usage()
932 		if (insn->dst_reg == BPF_REG_7 || insn->src_reg == BPF_REG_7)  in detect_reg_usage()
934 		if (insn->dst_reg == BPF_REG_8 || insn->src_reg == BPF_REG_8)  in detect_reg_usage()
936 		if (insn->dst_reg == BPF_REG_9 || insn->src_reg == BPF_REG_9)  in detect_reg_usage()
1008 static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)  in emit_shiftx()  argument
1014 	emit_3vex(&prog, r, false, r, m, is64, src_reg, false, op);  in emit_shiftx()
1057 		u32 src_reg = insn->src_reg;  in do_jit()  local
1078 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1081 			EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1089 					     dst_reg, src_reg);  in do_jit()
1093 					       dst_reg, src_reg);  in do_jit()
1181 				if (src_reg == BPF_REG_0 ||  in do_jit()
1182 				    src_reg == BPF_REG_3) {  in do_jit()
1184 					EMIT_mov(AUX_REG, src_reg);  in do_jit()
1185 					src_reg = AUX_REG;  in do_jit()
1190 				src_reg = AUX_REG;  in do_jit()
1205 				maybe_emit_1mod(&prog, src_reg, is64);  in do_jit()
1206 				EMIT2(0xF7, add_1reg(0xF0, src_reg));  in do_jit()
1214 				maybe_emit_1mod(&prog, src_reg, is64);  in do_jit()
1215 				EMIT2(0xF7, add_1reg(0xF8, src_reg));  in do_jit()
1252 			maybe_emit_mod(&prog, src_reg, dst_reg,  in do_jit()
1256 			EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));  in do_jit()
1283 			if (boot_cpu_has(X86_FEATURE_BMI2) && src_reg != BPF_REG_4) {  in do_jit()
1300 				emit_shiftx(&prog, dst_reg, src_reg, w, op);  in do_jit()
1305 			if (src_reg != BPF_REG_4) { /* common case */  in do_jit()
1315 				EMIT_mov(BPF_REG_4, src_reg);  in do_jit()
1325 			if (src_reg != BPF_REG_4) {  in do_jit()
1433 			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);  in do_jit()
1477 					maybe_emit_1mod(&prog, src_reg, true);  in do_jit()
1478 					EMIT2_off32(0x81, add_1reg(0xC0, src_reg), insn->off);  in do_jit()
1482 				maybe_emit_mod(&prog, src_reg, AUX_REG, true);  in do_jit()
1483 				EMIT2(0x39, add_2reg(0xC0, src_reg, AUX_REG));  in do_jit()
1500 				emit_ldsx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);  in do_jit()
1502 				emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);  in do_jit()
1512 				if (insn->off && src_reg != dst_reg) {  in do_jit()
1520 					maybe_emit_1mod(&prog, src_reg, true);  in do_jit()
1521 					EMIT2_off32(0x81, add_1reg(0xE8, src_reg), insn->off);  in do_jit()
1567 				u32 real_src_reg = src_reg;  in do_jit()
1578 				if (src_reg == BPF_REG_0)  in do_jit()
1614 			err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,  in do_jit()
1679 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1681 			EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1687 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1689 			EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()