197 		int (*execute)(struct x86_emulate_ctxt *ctxt);
206 	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
245 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)  in reg_read()  argument
247 	if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))  in reg_read()
250 	if (!(ctxt->regs_valid & (1 << nr))) {  in reg_read()
251 		ctxt->regs_valid |= 1 << nr;  in reg_read()
252 		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);  in reg_read()
254 	return ctxt->_regs[nr];  in reg_read()
257 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)  in reg_write()  argument
259 	if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))  in reg_write()
262 	BUILD_BUG_ON(sizeof(ctxt->regs_dirty) * BITS_PER_BYTE < NR_EMULATOR_GPRS);  in reg_write()
263 	BUILD_BUG_ON(sizeof(ctxt->regs_valid) * BITS_PER_BYTE < NR_EMULATOR_GPRS);  in reg_write()
265 	ctxt->regs_valid |= 1 << nr;  in reg_write()
266 	ctxt->regs_dirty |= 1 << nr;  in reg_write()
267 	return &ctxt->_regs[nr];  in reg_write()
270 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)  in reg_rmw()  argument
272 	reg_read(ctxt, nr);  in reg_rmw()
273 	return reg_write(ctxt, nr);  in reg_rmw()
276 static void writeback_registers(struct x86_emulate_ctxt *ctxt)  in writeback_registers()  argument
278 	unsigned long dirty = ctxt->regs_dirty;  in writeback_registers()
282 		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);  in writeback_registers()
285 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)  in invalidate_registers()  argument
287 	ctxt->regs_dirty = 0;  in invalidate_registers()
288 	ctxt->regs_valid = 0;  in invalidate_registers()
321 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
496 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,  in emulator_check_intercept()  argument
502 		.rep_prefix = ctxt->rep_prefix,  in emulator_check_intercept()
503 		.modrm_mod  = ctxt->modrm_mod,  in emulator_check_intercept()
504 		.modrm_reg  = ctxt->modrm_reg,  in emulator_check_intercept()
505 		.modrm_rm   = ctxt->modrm_rm,  in emulator_check_intercept()
506 		.src_val    = ctxt->src.val64,  in emulator_check_intercept()
507 		.dst_val    = ctxt->dst.val64,  in emulator_check_intercept()
508 		.src_bytes  = ctxt->src.bytes,  in emulator_check_intercept()
509 		.dst_bytes  = ctxt->dst.bytes,  in emulator_check_intercept()
510 		.ad_bytes   = ctxt->ad_bytes,  in emulator_check_intercept()
511 		.next_rip   = ctxt->eip,  in emulator_check_intercept()
514 	return ctxt->ops->intercept(ctxt, &info, stage);  in emulator_check_intercept()
541 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)  in ad_mask()  argument
543 	return (1UL << (ctxt->ad_bytes << 3)) - 1;  in ad_mask()
546 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)  in stack_mask()  argument
551 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in stack_mask()
553 	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);  in stack_mask()
557 static int stack_size(struct x86_emulate_ctxt *ctxt)  in stack_size()  argument
559 	return (__fls(stack_mask(ctxt)) + 1) >> 3;  in stack_size()
564 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)  in address_mask()  argument
566 	if (ctxt->ad_bytes == sizeof(unsigned long))  in address_mask()
569 		return reg & ad_mask(ctxt);  in address_mask()
573 register_address(struct x86_emulate_ctxt *ctxt, int reg)  in register_address()  argument
575 	return address_mask(ctxt, reg_read(ctxt, reg));  in register_address()
584 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)  in register_address_increment()  argument
586 	ulong *preg = reg_rmw(ctxt, reg);  in register_address_increment()
588 	assign_register(preg, *preg + inc, ctxt->ad_bytes);  in register_address_increment()
591 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)  in rsp_increment()  argument
593 	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);  in rsp_increment()
603 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)  in seg_base()  argument
605 	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)  in seg_base()
608 	return ctxt->ops->get_cached_segment_base(ctxt, seg);  in seg_base()
611 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,  in emulate_exception()  argument
614 	if (KVM_EMULATOR_BUG_ON(vec > 0x1f, ctxt))  in emulate_exception()
617 	ctxt->exception.vector = vec;  in emulate_exception()
618 	ctxt->exception.error_code = error;  in emulate_exception()
619 	ctxt->exception.error_code_valid = valid;  in emulate_exception()
623 static int emulate_db(struct x86_emulate_ctxt *ctxt)  in emulate_db()  argument
625 	return emulate_exception(ctxt, DB_VECTOR, 0, false);  in emulate_db()
628 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)  in emulate_gp()  argument
630 	return emulate_exception(ctxt, GP_VECTOR, err, true);  in emulate_gp()
633 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)  in emulate_ss()  argument
635 	return emulate_exception(ctxt, SS_VECTOR, err, true);  in emulate_ss()
638 static int emulate_ud(struct x86_emulate_ctxt *ctxt)  in emulate_ud()  argument
640 	return emulate_exception(ctxt, UD_VECTOR, 0, false);  in emulate_ud()
643 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)  in emulate_ts()  argument
645 	return emulate_exception(ctxt, TS_VECTOR, err, true);  in emulate_ts()
648 static int emulate_de(struct x86_emulate_ctxt *ctxt)  in emulate_de()  argument
650 	return emulate_exception(ctxt, DE_VECTOR, 0, false);  in emulate_de()
653 static int emulate_nm(struct x86_emulate_ctxt *ctxt)  in emulate_nm()  argument
655 	return emulate_exception(ctxt, NM_VECTOR, 0, false);  in emulate_nm()
658 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)  in get_segment_selector()  argument
663 	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);  in get_segment_selector()
667 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,  in set_segment_selector()  argument
674 	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);  in set_segment_selector()
675 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);  in set_segment_selector()
678 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)  in ctxt_virt_addr_bits()  argument
680 	return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;  in ctxt_virt_addr_bits()
684 						struct x86_emulate_ctxt *ctxt)  in emul_is_noncanonical_address()  argument
686 	return !__is_canonical_address(la, ctxt_virt_addr_bits(ctxt));  in emul_is_noncanonical_address()
698 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)  in insn_alignment()  argument
700 	u64 alignment = ctxt->d & AlignMask;  in insn_alignment()
717 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,  in __linearize()  argument
730 	la = seg_base(ctxt, addr.seg) + addr.ea;  in __linearize()
735 		va_bits = ctxt_virt_addr_bits(ctxt);  in __linearize()
745 		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,  in __linearize()
750 		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))  in __linearize()
774 	if (la & (insn_alignment(ctxt, size) - 1))  in __linearize()
775 		return emulate_gp(ctxt, 0);  in __linearize()
779 		return emulate_ss(ctxt, 0);  in __linearize()
781 		return emulate_gp(ctxt, 0);  in __linearize()
784 static int linearize(struct x86_emulate_ctxt *ctxt,  in linearize()  argument
790 	return __linearize(ctxt, addr, &max_size, size, write, false,  in linearize()
791 			   ctxt->mode, linear);  in linearize()
794 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)  in assign_eip()  argument
802 	if (ctxt->op_bytes != sizeof(unsigned long))  in assign_eip()
803 		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);  in assign_eip()
804 	rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear);  in assign_eip()
806 		ctxt->_eip = addr.ea;  in assign_eip()
810 static inline int emulator_recalc_and_set_mode(struct x86_emulate_ctxt *ctxt)  in emulator_recalc_and_set_mode()  argument
817 	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in emulator_recalc_and_set_mode()
819 	if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) {  in emulator_recalc_and_set_mode()
823 		ctxt->mode = X86EMUL_MODE_REAL;  in emulator_recalc_and_set_mode()
827 	if (ctxt->eflags & X86_EFLAGS_VM) {  in emulator_recalc_and_set_mode()
831 		ctxt->mode = X86EMUL_MODE_VM86;  in emulator_recalc_and_set_mode()
835 	if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS))  in emulator_recalc_and_set_mode()
841 			ctxt->mode = X86EMUL_MODE_PROT64;  in emulator_recalc_and_set_mode()
844 			ctxt->mode = X86EMUL_MODE_PROT32;  in emulator_recalc_and_set_mode()
846 			ctxt->mode = X86EMUL_MODE_PROT16;  in emulator_recalc_and_set_mode()
850 		ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;  in emulator_recalc_and_set_mode()
856 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)  in assign_eip_near()  argument
858 	return assign_eip(ctxt, dst);  in assign_eip_near()
861 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst)  in assign_eip_far()  argument
863 	int rc = emulator_recalc_and_set_mode(ctxt);  in assign_eip_far()
868 	return assign_eip(ctxt, dst);  in assign_eip_far()
871 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)  in jmp_rel()  argument
873 	return assign_eip_near(ctxt, ctxt->_eip + rel);  in jmp_rel()
876 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,  in linear_read_system()  argument
879 	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);  in linear_read_system()
882 static int linear_write_system(struct x86_emulate_ctxt *ctxt,  in linear_write_system()  argument
886 	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);  in linear_write_system()
889 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,  in segmented_read_std()  argument
897 	rc = linearize(ctxt, addr, size, false, &linear);  in segmented_read_std()
900 	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);  in segmented_read_std()
903 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,  in segmented_write_std()  argument
911 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_write_std()
914 	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);  in segmented_write_std()
921 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)  in __do_insn_fetch_bytes()  argument
926 	int cur_size = ctxt->fetch.end - ctxt->fetch.data;  in __do_insn_fetch_bytes()
928 					   .ea = ctxt->eip + cur_size };  in __do_insn_fetch_bytes()
940 	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,  in __do_insn_fetch_bytes()
955 		return emulate_gp(ctxt, 0);  in __do_insn_fetch_bytes()
957 	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,  in __do_insn_fetch_bytes()
958 			      size, &ctxt->exception);  in __do_insn_fetch_bytes()
961 	ctxt->fetch.end += size;  in __do_insn_fetch_bytes()
965 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,  in do_insn_fetch_bytes()  argument
968 	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;  in do_insn_fetch_bytes()
971 		return __do_insn_fetch_bytes(ctxt, size - done_size);  in do_insn_fetch_bytes()
983 	ctxt->_eip += sizeof(_type);					\
984 	memcpy(&_x, ctxt->fetch.ptr, sizeof(_type));			\
985 	ctxt->fetch.ptr += sizeof(_type);				\
994 	ctxt->_eip += (_size);						\
995 	memcpy(_arr, ctxt->fetch.ptr, _size);				\
996 	ctxt->fetch.ptr += (_size);					\
1004 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,  in decode_register()  argument
1008 	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;  in decode_register()
1011 		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;  in decode_register()
1013 		p = reg_rmw(ctxt, modrm_reg);  in decode_register()
1017 static int read_descriptor(struct x86_emulate_ctxt *ctxt,  in read_descriptor()  argument
1026 	rc = segmented_read_std(ctxt, addr, size, 2);  in read_descriptor()
1030 	rc = segmented_read_std(ctxt, addr, address, op_bytes);  in read_descriptor()
1078 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)  in em_bsf_c()  argument
1081 	if (ctxt->src.val == 0)  in em_bsf_c()
1082 		ctxt->dst.type = OP_NONE;  in em_bsf_c()
1083 	return fastop(ctxt, em_bsf);  in em_bsf_c()
1086 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)  in em_bsr_c()  argument
1089 	if (ctxt->src.val == 0)  in em_bsr_c()
1090 		ctxt->dst.type = OP_NONE;  in em_bsr_c()
1091 	return fastop(ctxt, em_bsr);  in em_bsr_c()
1123 static int em_fninit(struct x86_emulate_ctxt *ctxt)  in em_fninit()  argument
1125 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fninit()
1126 		return emulate_nm(ctxt);  in em_fninit()
1134 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)  in em_fnstcw()  argument
1138 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fnstcw()
1139 		return emulate_nm(ctxt);  in em_fnstcw()
1145 	ctxt->dst.val = fcw;  in em_fnstcw()
1150 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)  in em_fnstsw()  argument
1154 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fnstsw()
1155 		return emulate_nm(ctxt);  in em_fnstsw()
1161 	ctxt->dst.val = fsw;  in em_fnstsw()
1166 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,  in decode_register_operand()  argument
1171 	if (ctxt->d & ModRM)  in decode_register_operand()
1172 		reg = ctxt->modrm_reg;  in decode_register_operand()
1174 		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);  in decode_register_operand()
1176 	if (ctxt->d & Sse) {  in decode_register_operand()
1183 	if (ctxt->d & Mmx) {  in decode_register_operand()
1192 	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_register_operand()
1193 	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);  in decode_register_operand()
1199 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)  in adjust_modrm_seg()  argument
1202 		ctxt->modrm_seg = VCPU_SREG_SS;  in adjust_modrm_seg()
1205 static int decode_modrm(struct x86_emulate_ctxt *ctxt,  in decode_modrm()  argument
1213 	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */  in decode_modrm()
1214 	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */  in decode_modrm()
1215 	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */  in decode_modrm()
1217 	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;  in decode_modrm()
1218 	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;  in decode_modrm()
1219 	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);  in decode_modrm()
1220 	ctxt->modrm_seg = VCPU_SREG_DS;  in decode_modrm()
1222 	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {  in decode_modrm()
1224 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_modrm()
1225 		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,  in decode_modrm()
1226 				ctxt->d & ByteOp);  in decode_modrm()
1227 		if (ctxt->d & Sse) {  in decode_modrm()
1230 			op->addr.xmm = ctxt->modrm_rm;  in decode_modrm()
1231 			kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);  in decode_modrm()
1234 		if (ctxt->d & Mmx) {  in decode_modrm()
1237 			op->addr.mm = ctxt->modrm_rm & 7;  in decode_modrm()
1246 	if (ctxt->ad_bytes == 2) {  in decode_modrm()
1247 		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);  in decode_modrm()
1248 		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);  in decode_modrm()
1249 		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);  in decode_modrm()
1250 		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);  in decode_modrm()
1253 		switch (ctxt->modrm_mod) {  in decode_modrm()
1255 			if (ctxt->modrm_rm == 6)  in decode_modrm()
1256 				modrm_ea += insn_fetch(u16, ctxt);  in decode_modrm()
1259 			modrm_ea += insn_fetch(s8, ctxt);  in decode_modrm()
1262 			modrm_ea += insn_fetch(u16, ctxt);  in decode_modrm()
1265 		switch (ctxt->modrm_rm) {  in decode_modrm()
1285 			if (ctxt->modrm_mod != 0)  in decode_modrm()
1292 		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||  in decode_modrm()
1293 		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))  in decode_modrm()
1294 			ctxt->modrm_seg = VCPU_SREG_SS;  in decode_modrm()
1298 		if ((ctxt->modrm_rm & 7) == 4) {  in decode_modrm()
1299 			sib = insn_fetch(u8, ctxt);  in decode_modrm()
1304 			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)  in decode_modrm()
1305 				modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1307 				modrm_ea += reg_read(ctxt, base_reg);  in decode_modrm()
1308 				adjust_modrm_seg(ctxt, base_reg);  in decode_modrm()
1310 				if ((ctxt->d & IncSP) &&  in decode_modrm()
1312 					modrm_ea += ctxt->op_bytes;  in decode_modrm()
1315 				modrm_ea += reg_read(ctxt, index_reg) << scale;  in decode_modrm()
1316 		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {  in decode_modrm()
1317 			modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1318 			if (ctxt->mode == X86EMUL_MODE_PROT64)  in decode_modrm()
1319 				ctxt->rip_relative = 1;  in decode_modrm()
1321 			base_reg = ctxt->modrm_rm;  in decode_modrm()
1322 			modrm_ea += reg_read(ctxt, base_reg);  in decode_modrm()
1323 			adjust_modrm_seg(ctxt, base_reg);  in decode_modrm()
1325 		switch (ctxt->modrm_mod) {  in decode_modrm()
1327 			modrm_ea += insn_fetch(s8, ctxt);  in decode_modrm()
1330 			modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1335 	if (ctxt->ad_bytes != 8)  in decode_modrm()
1336 		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;  in decode_modrm()
1342 static int decode_abs(struct x86_emulate_ctxt *ctxt,  in decode_abs()  argument
1348 	switch (ctxt->ad_bytes) {  in decode_abs()
1350 		op->addr.mem.ea = insn_fetch(u16, ctxt);  in decode_abs()
1353 		op->addr.mem.ea = insn_fetch(u32, ctxt);  in decode_abs()
1356 		op->addr.mem.ea = insn_fetch(u64, ctxt);  in decode_abs()
1363 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)  in fetch_bit_operand()  argument
1367 	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {  in fetch_bit_operand()
1368 		mask = ~((long)ctxt->dst.bytes * 8 - 1);  in fetch_bit_operand()
1370 		if (ctxt->src.bytes == 2)  in fetch_bit_operand()
1371 			sv = (s16)ctxt->src.val & (s16)mask;  in fetch_bit_operand()
1372 		else if (ctxt->src.bytes == 4)  in fetch_bit_operand()
1373 			sv = (s32)ctxt->src.val & (s32)mask;  in fetch_bit_operand()
1375 			sv = (s64)ctxt->src.val & (s64)mask;  in fetch_bit_operand()
1377 		ctxt->dst.addr.mem.ea = address_mask(ctxt,  in fetch_bit_operand()
1378 					   ctxt->dst.addr.mem.ea + (sv >> 3));  in fetch_bit_operand()
1382 	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;  in fetch_bit_operand()
1385 static int read_emulated(struct x86_emulate_ctxt *ctxt,  in read_emulated()  argument
1389 	struct read_cache *mc = &ctxt->mem_read;  in read_emulated()
1394 	if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))  in read_emulated()
1397 	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,  in read_emulated()
1398 				      &ctxt->exception);  in read_emulated()
1410 static int segmented_read(struct x86_emulate_ctxt *ctxt,  in segmented_read()  argument
1418 	rc = linearize(ctxt, addr, size, false, &linear);  in segmented_read()
1421 	return read_emulated(ctxt, linear, data, size);  in segmented_read()
1424 static int segmented_write(struct x86_emulate_ctxt *ctxt,  in segmented_write()  argument
1432 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_write()
1435 	return ctxt->ops->write_emulated(ctxt, linear, data, size,  in segmented_write()
1436 					 &ctxt->exception);  in segmented_write()
1439 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,  in segmented_cmpxchg()  argument
1447 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_cmpxchg()
1450 	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,  in segmented_cmpxchg()
1451 					   size, &ctxt->exception);  in segmented_cmpxchg()
1454 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,  in pio_in_emulated()  argument
1458 	struct read_cache *rc = &ctxt->io_read;  in pio_in_emulated()
1462 		unsigned int count = ctxt->rep_prefix ?  in pio_in_emulated()
1463 			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;  in pio_in_emulated()
1464 		in_page = (ctxt->eflags & X86_EFLAGS_DF) ?  in pio_in_emulated()
1465 			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :  in pio_in_emulated()
1466 			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));  in pio_in_emulated()
1471 		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))  in pio_in_emulated()
1476 	if (ctxt->rep_prefix && (ctxt->d & String) &&  in pio_in_emulated()
1477 	    !(ctxt->eflags & X86_EFLAGS_DF)) {  in pio_in_emulated()
1478 		ctxt->dst.data = rc->data + rc->pos;  in pio_in_emulated()
1479 		ctxt->dst.type = OP_MEM_STR;  in pio_in_emulated()
1480 		ctxt->dst.count = (rc->end - rc->pos) / size;  in pio_in_emulated()
1489 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,  in read_interrupt_descriptor()  argument
1495 	ctxt->ops->get_idt(ctxt, &dt);  in read_interrupt_descriptor()
1498 		return emulate_gp(ctxt, index << 3 | 0x2);  in read_interrupt_descriptor()
1501 	return linear_read_system(ctxt, addr, desc, sizeof(*desc));  in read_interrupt_descriptor()
1504 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,  in get_descriptor_table_ptr()  argument
1507 	const struct x86_emulate_ops *ops = ctxt->ops;  in get_descriptor_table_ptr()
1515 		if (!ops->get_segment(ctxt, &sel, &desc, &base3,  in get_descriptor_table_ptr()
1522 		ops->get_gdt(ctxt, dt);  in get_descriptor_table_ptr()
1525 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,  in get_descriptor_ptr()  argument
1532 	get_descriptor_table_ptr(ctxt, selector, &dt);  in get_descriptor_ptr()
1535 		return emulate_gp(ctxt, selector & 0xfffc);  in get_descriptor_ptr()
1543 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in get_descriptor_ptr()
1554 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in read_segment_descriptor()  argument
1560 	rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);  in read_segment_descriptor()
1564 	return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));  in read_segment_descriptor()
1568 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in write_segment_descriptor()  argument
1574 	rc = get_descriptor_ptr(ctxt, selector, &addr);  in write_segment_descriptor()
1578 	return linear_write_system(ctxt, addr, desc, sizeof(*desc));  in write_segment_descriptor()
1581 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in __load_segment_descriptor()  argument
1598 	if (ctxt->mode == X86EMUL_MODE_REAL) {  in __load_segment_descriptor()
1601 		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);  in __load_segment_descriptor()
1604 	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {  in __load_segment_descriptor()
1627 			if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)  in __load_segment_descriptor()
1646 	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);  in __load_segment_descriptor()
1709 			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in __load_segment_descriptor()
1747 			ret = write_segment_descriptor(ctxt, selector,  in __load_segment_descriptor()
1752 	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {  in __load_segment_descriptor()
1753 		ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));  in __load_segment_descriptor()
1757 						 ((u64)base3 << 32), ctxt))  in __load_segment_descriptor()
1758 			return emulate_gp(ctxt, err_code);  in __load_segment_descriptor()
1764 		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,  in __load_segment_descriptor()
1765 						  sizeof(seg_desc), &ctxt->exception);  in __load_segment_descriptor()
1770 	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);  in __load_segment_descriptor()
1775 	return emulate_exception(ctxt, err_vec, err_code, true);  in __load_segment_descriptor()
1778 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in load_segment_descriptor()  argument
1781 	u8 cpl = ctxt->ops->cpl(ctxt);  in load_segment_descriptor()
1794 	    ctxt->mode == X86EMUL_MODE_PROT64)  in load_segment_descriptor()
1795 		return emulate_exception(ctxt, GP_VECTOR, 0, true);  in load_segment_descriptor()
1797 	return __load_segment_descriptor(ctxt, selector, seg, cpl,  in load_segment_descriptor()
1806 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)  in writeback()  argument
1813 		if (ctxt->lock_prefix)  in writeback()
1814 			return segmented_cmpxchg(ctxt,  in writeback()
1820 			return segmented_write(ctxt,  in writeback()
1826 		return segmented_write(ctxt,  in writeback()
1846 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)  in push()  argument
1850 	rsp_increment(ctxt, -bytes);  in push()
1851 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);  in push()
1854 	return segmented_write(ctxt, addr, data, bytes);  in push()
1857 static int em_push(struct x86_emulate_ctxt *ctxt)  in em_push()  argument
1860 	ctxt->dst.type = OP_NONE;  in em_push()
1861 	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);  in em_push()
1864 static int emulate_pop(struct x86_emulate_ctxt *ctxt,  in emulate_pop()  argument
1870 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);  in emulate_pop()
1872 	rc = segmented_read(ctxt, addr, dest, len);  in emulate_pop()
1876 	rsp_increment(ctxt, len);  in emulate_pop()
1880 static int em_pop(struct x86_emulate_ctxt *ctxt)  in em_pop()  argument
1882 	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);  in em_pop()
1885 static int emulate_popf(struct x86_emulate_ctxt *ctxt,  in emulate_popf()  argument
1890 	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;  in emulate_popf()
1891 	int cpl = ctxt->ops->cpl(ctxt);  in emulate_popf()
1893 	rc = emulate_pop(ctxt, &val, len);  in emulate_popf()
1902 	switch(ctxt->mode) {  in emulate_popf()
1913 			return emulate_gp(ctxt, 0);  in emulate_popf()
1922 		(ctxt->eflags & ~change_mask) | (val & change_mask);  in emulate_popf()
1927 static int em_popf(struct x86_emulate_ctxt *ctxt)  in em_popf()  argument
1929 	ctxt->dst.type = OP_REG;  in em_popf()
1930 	ctxt->dst.addr.reg = &ctxt->eflags;  in em_popf()
1931 	ctxt->dst.bytes = ctxt->op_bytes;  in em_popf()
1932 	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);  in em_popf()
1935 static int em_enter(struct x86_emulate_ctxt *ctxt)  in em_enter()  argument
1938 	unsigned frame_size = ctxt->src.val;  in em_enter()
1939 	unsigned nesting_level = ctxt->src2.val & 31;  in em_enter()
1945 	rbp = reg_read(ctxt, VCPU_REGS_RBP);  in em_enter()
1946 	rc = push(ctxt, &rbp, stack_size(ctxt));  in em_enter()
1949 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),  in em_enter()
1950 		      stack_mask(ctxt));  in em_enter()
1951 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),  in em_enter()
1952 		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,  in em_enter()
1953 		      stack_mask(ctxt));  in em_enter()
1957 static int em_leave(struct x86_emulate_ctxt *ctxt)  in em_leave()  argument
1959 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),  in em_leave()
1960 		      stack_mask(ctxt));  in em_leave()
1961 	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);  in em_leave()
1964 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)  in em_push_sreg()  argument
1966 	int seg = ctxt->src2.val;  in em_push_sreg()
1968 	ctxt->src.val = get_segment_selector(ctxt, seg);  in em_push_sreg()
1969 	if (ctxt->op_bytes == 4) {  in em_push_sreg()
1970 		rsp_increment(ctxt, -2);  in em_push_sreg()
1971 		ctxt->op_bytes = 2;  in em_push_sreg()
1974 	return em_push(ctxt);  in em_push_sreg()
1977 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)  in em_pop_sreg()  argument
1979 	int seg = ctxt->src2.val;  in em_pop_sreg()
1983 	rc = emulate_pop(ctxt, &selector, 2);  in em_pop_sreg()
1988 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;  in em_pop_sreg()
1989 	if (ctxt->op_bytes > 2)  in em_pop_sreg()
1990 		rsp_increment(ctxt, ctxt->op_bytes - 2);  in em_pop_sreg()
1992 	rc = load_segment_descriptor(ctxt, (u16)selector, seg);  in em_pop_sreg()
1996 static int em_pusha(struct x86_emulate_ctxt *ctxt)  in em_pusha()  argument
1998 	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);  in em_pusha()
2004 		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));  in em_pusha()
2006 		rc = em_push(ctxt);  in em_pusha()
2016 static int em_pushf(struct x86_emulate_ctxt *ctxt)  in em_pushf()  argument
2018 	ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;  in em_pushf()
2019 	return em_push(ctxt);  in em_pushf()
2022 static int em_popa(struct x86_emulate_ctxt *ctxt)  in em_popa()  argument
2030 			rsp_increment(ctxt, ctxt->op_bytes);  in em_popa()
2034 		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);  in em_popa()
2037 		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);  in em_popa()
2043 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)  in __emulate_int_real()  argument
2045 	const struct x86_emulate_ops *ops = ctxt->ops;  in __emulate_int_real()
2053 	ctxt->src.val = ctxt->eflags;  in __emulate_int_real()
2054 	rc = em_push(ctxt);  in __emulate_int_real()
2058 	ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);  in __emulate_int_real()
2060 	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);  in __emulate_int_real()
2061 	rc = em_push(ctxt);  in __emulate_int_real()
2065 	ctxt->src.val = ctxt->_eip;  in __emulate_int_real()
2066 	rc = em_push(ctxt);  in __emulate_int_real()
2070 	ops->get_idt(ctxt, &dt);  in __emulate_int_real()
2075 	rc = linear_read_system(ctxt, cs_addr, &cs, 2);  in __emulate_int_real()
2079 	rc = linear_read_system(ctxt, eip_addr, &eip, 2);  in __emulate_int_real()
2083 	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);  in __emulate_int_real()
2087 	ctxt->_eip = eip;  in __emulate_int_real()
2092 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)  in emulate_int_real()  argument
2096 	invalidate_registers(ctxt);  in emulate_int_real()
2097 	rc = __emulate_int_real(ctxt, irq);  in emulate_int_real()
2099 		writeback_registers(ctxt);  in emulate_int_real()
2103 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)  in emulate_int()  argument
2105 	switch(ctxt->mode) {  in emulate_int()
2107 		return __emulate_int_real(ctxt, irq);  in emulate_int()
2118 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)  in emulate_iret_real()  argument
2135 	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);  in emulate_iret_real()
2141 		return emulate_gp(ctxt, 0);  in emulate_iret_real()
2143 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);  in emulate_iret_real()
2148 	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);  in emulate_iret_real()
2153 	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);  in emulate_iret_real()
2158 	ctxt->_eip = temp_eip;  in emulate_iret_real()
2160 	if (ctxt->op_bytes == 4)  in emulate_iret_real()
2161 		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));  in emulate_iret_real()
2162 	else if (ctxt->op_bytes == 2) {  in emulate_iret_real()
2163 		ctxt->eflags &= ~0xffff;  in emulate_iret_real()
2164 		ctxt->eflags |= temp_eflags;  in emulate_iret_real()
2167 	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */  in emulate_iret_real()
2168 	ctxt->eflags |= X86_EFLAGS_FIXED;  in emulate_iret_real()
2169 	ctxt->ops->set_nmi_mask(ctxt, false);  in emulate_iret_real()
2174 static int em_iret(struct x86_emulate_ctxt *ctxt)  in em_iret()  argument
2176 	switch(ctxt->mode) {  in em_iret()
2178 		return emulate_iret_real(ctxt);  in em_iret()
2189 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)  in em_jmp_far()  argument
2194 	u8 cpl = ctxt->ops->cpl(ctxt);  in em_jmp_far()
2196 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_jmp_far()
2198 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,  in em_jmp_far()
2204 	rc = assign_eip_far(ctxt, ctxt->src.val);  in em_jmp_far()
2212 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)  in em_jmp_abs()  argument
2214 	return assign_eip_near(ctxt, ctxt->src.val);  in em_jmp_abs()
2217 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)  in em_call_near_abs()  argument
2222 	old_eip = ctxt->_eip;  in em_call_near_abs()
2223 	rc = assign_eip_near(ctxt, ctxt->src.val);  in em_call_near_abs()
2226 	ctxt->src.val = old_eip;  in em_call_near_abs()
2227 	rc = em_push(ctxt);  in em_call_near_abs()
2231 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)  in em_cmpxchg8b()  argument
2233 	u64 old = ctxt->dst.orig_val64;  in em_cmpxchg8b()
2235 	if (ctxt->dst.bytes == 16)  in em_cmpxchg8b()
2238 	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||  in em_cmpxchg8b()
2239 	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {  in em_cmpxchg8b()
2240 		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);  in em_cmpxchg8b()
2241 		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);  in em_cmpxchg8b()
2242 		ctxt->eflags &= ~X86_EFLAGS_ZF;  in em_cmpxchg8b()
2244 		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |  in em_cmpxchg8b()
2245 			(u32) reg_read(ctxt, VCPU_REGS_RBX);  in em_cmpxchg8b()
2247 		ctxt->eflags |= X86_EFLAGS_ZF;  in em_cmpxchg8b()
2252 static int em_ret(struct x86_emulate_ctxt *ctxt)  in em_ret()  argument
2257 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret()
2261 	return assign_eip_near(ctxt, eip);  in em_ret()
2264 static int em_ret_far(struct x86_emulate_ctxt *ctxt)  in em_ret_far()  argument
2268 	int cpl = ctxt->ops->cpl(ctxt);  in em_ret_far()
2271 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret_far()
2274 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);  in em_ret_far()
2277 	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,  in em_ret_far()
2282 	rc = assign_eip_far(ctxt, eip);  in em_ret_far()
2290 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)  in em_ret_far_imm()  argument
2294         rc = em_ret_far(ctxt);  in em_ret_far_imm()
2297         rsp_increment(ctxt, ctxt->src.val);  in em_ret_far_imm()
2301 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)  in em_cmpxchg()  argument
2304 	ctxt->dst.orig_val = ctxt->dst.val;  in em_cmpxchg()
2305 	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);  in em_cmpxchg()
2306 	ctxt->src.orig_val = ctxt->src.val;  in em_cmpxchg()
2307 	ctxt->src.val = ctxt->dst.orig_val;  in em_cmpxchg()
2308 	fastop(ctxt, em_cmp);  in em_cmpxchg()
2310 	if (ctxt->eflags & X86_EFLAGS_ZF) {  in em_cmpxchg()
2312 		ctxt->src.type = OP_NONE;  in em_cmpxchg()
2313 		ctxt->dst.val = ctxt->src.orig_val;  in em_cmpxchg()
2316 		ctxt->src.type = OP_REG;  in em_cmpxchg()
2317 		ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in em_cmpxchg()
2318 		ctxt->src.val = ctxt->dst.orig_val;  in em_cmpxchg()
2320 		ctxt->dst.val = ctxt->dst.orig_val;  in em_cmpxchg()
2325 static int em_lseg(struct x86_emulate_ctxt *ctxt)  in em_lseg()  argument
2327 	int seg = ctxt->src2.val;  in em_lseg()
2331 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_lseg()
2333 	rc = load_segment_descriptor(ctxt, sel, seg);  in em_lseg()
2337 	ctxt->dst.val = ctxt->src.val;  in em_lseg()
2341 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)  in emulator_has_longmode()  argument
2344 	return ctxt->ops->guest_has_long_mode(ctxt);  in emulator_has_longmode()
2362 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,  in rsm_load_seg_32()  argument
2379 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);  in rsm_load_seg_32()
2384 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,  in rsm_load_seg_64()  argument
2400 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);  in rsm_load_seg_64()
2405 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,  in rsm_enter_protected_mode()  argument
2418 	bad = ctxt->ops->set_cr(ctxt, 3, cr3);  in rsm_enter_protected_mode()
2427 	bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);  in rsm_enter_protected_mode()
2431 	bad = ctxt->ops->set_cr(ctxt, 0, cr0);  in rsm_enter_protected_mode()
2436 		bad = ctxt->ops->set_cr(ctxt, 4, cr4);  in rsm_enter_protected_mode()
2440 			bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);  in rsm_enter_protected_mode()
2450 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,  in rsm_load_state_32()  argument
2461 	ctxt->eflags =             GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;  in rsm_load_state_32()
2462 	ctxt->_eip =               GET_SMSTATE(u32, smstate, 0x7ff0);  in rsm_load_state_32()
2465 		*reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);  in rsm_load_state_32()
2469 	if (ctxt->ops->set_dr(ctxt, 6, val))  in rsm_load_state_32()
2474 	if (ctxt->ops->set_dr(ctxt, 7, val))  in rsm_load_state_32()
2481 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);  in rsm_load_state_32()
2487 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);  in rsm_load_state_32()
2491 	ctxt->ops->set_gdt(ctxt, &dt);  in rsm_load_state_32()
2495 	ctxt->ops->set_idt(ctxt, &dt);  in rsm_load_state_32()
2498 		int r = rsm_load_seg_32(ctxt, smstate, i);  in rsm_load_state_32()
2505 	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));  in rsm_load_state_32()
2507 	return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);  in rsm_load_state_32()
2511 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,  in rsm_load_state_64()  argument
2522 		*reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);  in rsm_load_state_64()
2524 	ctxt->_eip   = GET_SMSTATE(u64, smstate, 0x7f78);  in rsm_load_state_64()
2525 	ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;  in rsm_load_state_64()
2529 	if (ctxt->ops->set_dr(ctxt, 6, val))  in rsm_load_state_64()
2534 	if (ctxt->ops->set_dr(ctxt, 7, val))  in rsm_load_state_64()
2540 	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));  in rsm_load_state_64()
2543 	if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))  in rsm_load_state_64()
2551 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);  in rsm_load_state_64()
2555 	ctxt->ops->set_idt(ctxt, &dt);  in rsm_load_state_64()
2562 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);  in rsm_load_state_64()
2566 	ctxt->ops->set_gdt(ctxt, &dt);  in rsm_load_state_64()
2568 	r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);  in rsm_load_state_64()
2573 		r = rsm_load_seg_64(ctxt, smstate, i);  in rsm_load_state_64()
2582 static int em_rsm(struct x86_emulate_ctxt *ctxt)  in em_rsm()  argument
2589 	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)  in em_rsm()
2590 		return emulate_ud(ctxt);  in em_rsm()
2592 	smbase = ctxt->ops->get_smbase(ctxt);  in em_rsm()
2594 	ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));  in em_rsm()
2598 	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)  in em_rsm()
2599 		ctxt->ops->set_nmi_mask(ctxt, false);  in em_rsm()
2601 	ctxt->ops->exiting_smm(ctxt);  in em_rsm()
2608 	if (emulator_has_longmode(ctxt)) {  in em_rsm()
2612 		cr4 = ctxt->ops->get_cr(ctxt, 4);  in em_rsm()
2614 			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);  in em_rsm()
2620 		ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);  in em_rsm()
2624 	cr0 = ctxt->ops->get_cr(ctxt, 0);  in em_rsm()
2626 		ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));  in em_rsm()
2628 	if (emulator_has_longmode(ctxt)) {  in em_rsm()
2630 		cr4 = ctxt->ops->get_cr(ctxt, 4);  in em_rsm()
2632 			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);  in em_rsm()
2636 		ctxt->ops->set_msr(ctxt, MSR_EFER, efer);  in em_rsm()
2644 	if (ctxt->ops->leave_smm(ctxt, buf))  in em_rsm()
2648 	if (emulator_has_longmode(ctxt))  in em_rsm()
2649 		ret = rsm_load_state_64(ctxt, buf);  in em_rsm()
2652 		ret = rsm_load_state_32(ctxt, buf);  in em_rsm()
2665 	return emulator_recalc_and_set_mode(ctxt);  in em_rsm()
2668 	ctxt->ops->triple_fault(ctxt);  in em_rsm()
2698 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)  in vendor_intel()  argument
2703 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);  in vendor_intel()
2707 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)  in em_syscall_is_enabled()  argument
2709 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_syscall_is_enabled()
2716 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_syscall_is_enabled()
2721 	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);  in em_syscall_is_enabled()
2742 static int em_syscall(struct x86_emulate_ctxt *ctxt)  in em_syscall()  argument
2744 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_syscall()
2751 	if (ctxt->mode == X86EMUL_MODE_REAL ||  in em_syscall()
2752 	    ctxt->mode == X86EMUL_MODE_VM86)  in em_syscall()
2753 		return emulate_ud(ctxt);  in em_syscall()
2755 	if (!(em_syscall_is_enabled(ctxt)))  in em_syscall()
2756 		return emulate_ud(ctxt);  in em_syscall()
2758 	ops->get_msr(ctxt, MSR_EFER, &efer);  in em_syscall()
2760 		return emulate_ud(ctxt);  in em_syscall()
2763 	ops->get_msr(ctxt, MSR_STAR, &msr_data);  in em_syscall()
2772 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_syscall()
2773 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_syscall()
2775 	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;  in em_syscall()
2778 		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;  in em_syscall()
2780 		ops->get_msr(ctxt,  in em_syscall()
2781 			     ctxt->mode == X86EMUL_MODE_PROT64 ?  in em_syscall()
2783 		ctxt->_eip = msr_data;  in em_syscall()
2785 		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);  in em_syscall()
2786 		ctxt->eflags &= ~msr_data;  in em_syscall()
2787 		ctxt->eflags |= X86_EFLAGS_FIXED;  in em_syscall()
2791 		ops->get_msr(ctxt, MSR_STAR, &msr_data);  in em_syscall()
2792 		ctxt->_eip = (u32)msr_data;  in em_syscall()
2794 		ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);  in em_syscall()
2797 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;  in em_syscall()
2801 static int em_sysenter(struct x86_emulate_ctxt *ctxt)  in em_sysenter()  argument
2803 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_sysenter()
2809 	ops->get_msr(ctxt, MSR_EFER, &efer);  in em_sysenter()
2811 	if (ctxt->mode == X86EMUL_MODE_REAL)  in em_sysenter()
2812 		return emulate_gp(ctxt, 0);  in em_sysenter()
2818 	if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)  in em_sysenter()
2819 	    && !vendor_intel(ctxt))  in em_sysenter()
2820 		return emulate_ud(ctxt);  in em_sysenter()
2823 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_sysenter()
2826 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);  in em_sysenter()
2828 		return emulate_gp(ctxt, 0);  in em_sysenter()
2831 	ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);  in em_sysenter()
2839 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_sysenter()
2840 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_sysenter()
2842 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);  in em_sysenter()
2843 	ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;  in em_sysenter()
2845 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);  in em_sysenter()
2846 	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :  in em_sysenter()
2849 		ctxt->mode = X86EMUL_MODE_PROT64;  in em_sysenter()
2854 static int em_sysexit(struct x86_emulate_ctxt *ctxt)  in em_sysexit()  argument
2856 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_sysexit()
2863 	if (ctxt->mode == X86EMUL_MODE_REAL ||  in em_sysexit()
2864 	    ctxt->mode == X86EMUL_MODE_VM86)  in em_sysexit()
2865 		return emulate_gp(ctxt, 0);  in em_sysexit()
2869 	if ((ctxt->rex_prefix & 0x8) != 0x0)  in em_sysexit()
2874 	rcx = reg_read(ctxt, VCPU_REGS_RCX);  in em_sysexit()
2875 	rdx = reg_read(ctxt, VCPU_REGS_RDX);  in em_sysexit()
2879 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);  in em_sysexit()
2884 			return emulate_gp(ctxt, 0);  in em_sysexit()
2892 			return emulate_gp(ctxt, 0);  in em_sysexit()
2896 		if (emul_is_noncanonical_address(rcx, ctxt) ||  in em_sysexit()
2897 		    emul_is_noncanonical_address(rdx, ctxt))  in em_sysexit()
2898 			return emulate_gp(ctxt, 0);  in em_sysexit()
2904 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_sysexit()
2905 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_sysexit()
2907 	ctxt->_eip = rdx;  in em_sysexit()
2908 	ctxt->mode = usermode;  in em_sysexit()
2909 	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;  in em_sysexit()
2914 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)  in emulator_bad_iopl()  argument
2917 	if (ctxt->mode == X86EMUL_MODE_REAL)  in emulator_bad_iopl()
2919 	if (ctxt->mode == X86EMUL_MODE_VM86)  in emulator_bad_iopl()
2921 	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;  in emulator_bad_iopl()
2922 	return ctxt->ops->cpl(ctxt) > iopl;  in emulator_bad_iopl()
2928 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,  in emulator_io_port_access_allowed()  argument
2931 	const struct x86_emulate_ops *ops = ctxt->ops;  in emulator_io_port_access_allowed()
2947 	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);  in emulator_io_port_access_allowed()
2956 	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);  in emulator_io_port_access_allowed()
2961 	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);  in emulator_io_port_access_allowed()
2969 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,  in emulator_io_permited()  argument
2972 	if (ctxt->perm_ok)  in emulator_io_permited()
2975 	if (emulator_bad_iopl(ctxt))  in emulator_io_permited()
2976 		if (!emulator_io_port_access_allowed(ctxt, port, len))  in emulator_io_permited()
2979 	ctxt->perm_ok = true;  in emulator_io_permited()
2984 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)  in string_registers_quirk()  argument
2991 	if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))  in string_registers_quirk()
2994 	*reg_write(ctxt, VCPU_REGS_RCX) = 0;  in string_registers_quirk()
2996 	switch (ctxt->b) {  in string_registers_quirk()
2999 		*reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;  in string_registers_quirk()
3003 		*reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;  in string_registers_quirk()
3008 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,  in save_state_to_tss16()  argument
3011 	tss->ip = ctxt->_eip;  in save_state_to_tss16()
3012 	tss->flag = ctxt->eflags;  in save_state_to_tss16()
3013 	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);  in save_state_to_tss16()
3014 	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);  in save_state_to_tss16()
3015 	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);  in save_state_to_tss16()
3016 	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);  in save_state_to_tss16()
3017 	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);  in save_state_to_tss16()
3018 	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);  in save_state_to_tss16()
3019 	tss->si = reg_read(ctxt, VCPU_REGS_RSI);  in save_state_to_tss16()
3020 	tss->di = reg_read(ctxt, VCPU_REGS_RDI);  in save_state_to_tss16()
3022 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);  in save_state_to_tss16()
3023 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);  in save_state_to_tss16()
3024 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);  in save_state_to_tss16()
3025 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);  in save_state_to_tss16()
3026 	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);  in save_state_to_tss16()
3029 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,  in load_state_from_tss16()  argument
3035 	ctxt->_eip = tss->ip;  in load_state_from_tss16()
3036 	ctxt->eflags = tss->flag | 2;  in load_state_from_tss16()
3037 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;  in load_state_from_tss16()
3038 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;  in load_state_from_tss16()
3039 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;  in load_state_from_tss16()
3040 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;  in load_state_from_tss16()
3041 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;  in load_state_from_tss16()
3042 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;  in load_state_from_tss16()
3043 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;  in load_state_from_tss16()
3044 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;  in load_state_from_tss16()
3050 	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);  in load_state_from_tss16()
3051 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);  in load_state_from_tss16()
3052 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);  in load_state_from_tss16()
3053 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);  in load_state_from_tss16()
3054 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);  in load_state_from_tss16()
3062 	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,  in load_state_from_tss16()
3066 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,  in load_state_from_tss16()
3070 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,  in load_state_from_tss16()
3074 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,  in load_state_from_tss16()
3078 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,  in load_state_from_tss16()
3086 static int task_switch_16(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,  in task_switch_16()  argument
3093 	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
3097 	save_state_to_tss16(ctxt, &tss_seg);  in task_switch_16()
3099 	ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
3103 	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
3110 		ret = linear_write_system(ctxt, new_tss_base,  in task_switch_16()
3117 	return load_state_from_tss16(ctxt, &tss_seg);  in task_switch_16()
3120 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,  in save_state_to_tss32()  argument
3124 	tss->eip = ctxt->_eip;  in save_state_to_tss32()
3125 	tss->eflags = ctxt->eflags;  in save_state_to_tss32()
3126 	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);  in save_state_to_tss32()
3127 	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);  in save_state_to_tss32()
3128 	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);  in save_state_to_tss32()
3129 	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);  in save_state_to_tss32()
3130 	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);  in save_state_to_tss32()
3131 	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);  in save_state_to_tss32()
3132 	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);  in save_state_to_tss32()
3133 	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);  in save_state_to_tss32()
3135 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);  in save_state_to_tss32()
3136 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);  in save_state_to_tss32()
3137 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);  in save_state_to_tss32()
3138 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);  in save_state_to_tss32()
3139 	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);  in save_state_to_tss32()
3140 	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);  in save_state_to_tss32()
3143 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,  in load_state_from_tss32()  argument
3149 	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))  in load_state_from_tss32()
3150 		return emulate_gp(ctxt, 0);  in load_state_from_tss32()
3151 	ctxt->_eip = tss->eip;  in load_state_from_tss32()
3152 	ctxt->eflags = tss->eflags | 2;  in load_state_from_tss32()
3155 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;  in load_state_from_tss32()
3156 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;  in load_state_from_tss32()
3157 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;  in load_state_from_tss32()
3158 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;  in load_state_from_tss32()
3159 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;  in load_state_from_tss32()
3160 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;  in load_state_from_tss32()
3161 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;  in load_state_from_tss32()
3162 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;  in load_state_from_tss32()
3169 	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);  in load_state_from_tss32()
3170 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);  in load_state_from_tss32()
3171 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);  in load_state_from_tss32()
3172 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);  in load_state_from_tss32()
3173 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);  in load_state_from_tss32()
3174 	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);  in load_state_from_tss32()
3175 	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);  in load_state_from_tss32()
3182 	if (ctxt->eflags & X86_EFLAGS_VM) {  in load_state_from_tss32()
3183 		ctxt->mode = X86EMUL_MODE_VM86;  in load_state_from_tss32()
3186 		ctxt->mode = X86EMUL_MODE_PROT32;  in load_state_from_tss32()
3194 	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,  in load_state_from_tss32()
3198 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,  in load_state_from_tss32()
3202 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,  in load_state_from_tss32()
3206 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,  in load_state_from_tss32()
3210 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,  in load_state_from_tss32()
3214 	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,  in load_state_from_tss32()
3218 	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,  in load_state_from_tss32()
3224 static int task_switch_32(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,  in task_switch_32()  argument
3233 	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_32()
3237 	save_state_to_tss32(ctxt, &tss_seg);  in task_switch_32()
3240 	ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,  in task_switch_32()
3245 	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_32()
3252 		ret = linear_write_system(ctxt, new_tss_base,  in task_switch_32()
3259 	return load_state_from_tss32(ctxt, &tss_seg);  in task_switch_32()
3262 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,  in emulator_do_task_switch()  argument
3266 	const struct x86_emulate_ops *ops = ctxt->ops;  in emulator_do_task_switch()
3269 	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);  in emulator_do_task_switch()
3271 		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);  in emulator_do_task_switch()
3277 	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);  in emulator_do_task_switch()
3280 	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);  in emulator_do_task_switch()
3300 			ret = read_interrupt_descriptor(ctxt, idt_index,  in emulator_do_task_switch()
3306 			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)  in emulator_do_task_switch()
3307 				return emulate_gp(ctxt, (idt_index << 3) | 0x2);  in emulator_do_task_switch()
3315 		return emulate_ts(ctxt, tss_selector & 0xfffc);  in emulator_do_task_switch()
3320 		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);  in emulator_do_task_switch()
3324 		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;  in emulator_do_task_switch()
3332 		ret = task_switch_32(ctxt, old_tss_sel, old_tss_base, &next_tss_desc);  in emulator_do_task_switch()
3334 		ret = task_switch_16(ctxt, old_tss_sel,  in emulator_do_task_switch()
3340 		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;  in emulator_do_task_switch()
3344 		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);  in emulator_do_task_switch()
3347 	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);  in emulator_do_task_switch()
3348 	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);  in emulator_do_task_switch()
3351 		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;  in emulator_do_task_switch()
3352 		ctxt->lock_prefix = 0;  in emulator_do_task_switch()
3353 		ctxt->src.val = (unsigned long) error_code;  in emulator_do_task_switch()
3354 		ret = em_push(ctxt);  in emulator_do_task_switch()
3357 	ops->get_dr(ctxt, 7, &dr7);  in emulator_do_task_switch()
3358 	ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));  in emulator_do_task_switch()
3363 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,  in emulator_task_switch()  argument
3369 	invalidate_registers(ctxt);  in emulator_task_switch()
3370 	ctxt->_eip = ctxt->eip;  in emulator_task_switch()
3371 	ctxt->dst.type = OP_NONE;  in emulator_task_switch()
3373 	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,  in emulator_task_switch()
3377 		ctxt->eip = ctxt->_eip;  in emulator_task_switch()
3378 		writeback_registers(ctxt);  in emulator_task_switch()
3384 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,  in string_addr_inc()  argument
3387 	int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;  in string_addr_inc()
3389 	register_address_increment(ctxt, reg, df * op->bytes);  in string_addr_inc()
3390 	op->addr.mem.ea = register_address(ctxt, reg);  in string_addr_inc()
3393 static int em_das(struct x86_emulate_ctxt *ctxt)  in em_das()  argument
3398 	cf = ctxt->eflags & X86_EFLAGS_CF;  in em_das()
3399 	al = ctxt->dst.val;  in em_das()
3404 	af = ctxt->eflags & X86_EFLAGS_AF;  in em_das()
3417 	ctxt->dst.val = al;  in em_das()
3419 	ctxt->src.type = OP_IMM;  in em_das()
3420 	ctxt->src.val = 0;  in em_das()
3421 	ctxt->src.bytes = 1;  in em_das()
3422 	fastop(ctxt, em_or);  in em_das()
3423 	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);  in em_das()
3425 		ctxt->eflags |= X86_EFLAGS_CF;  in em_das()
3427 		ctxt->eflags |= X86_EFLAGS_AF;  in em_das()
3431 static int em_aam(struct x86_emulate_ctxt *ctxt)  in em_aam()  argument
3435 	if (ctxt->src.val == 0)  in em_aam()
3436 		return emulate_de(ctxt);  in em_aam()
3438 	al = ctxt->dst.val & 0xff;  in em_aam()
3439 	ah = al / ctxt->src.val;  in em_aam()
3440 	al %= ctxt->src.val;  in em_aam()
3442 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);  in em_aam()
3445 	ctxt->src.type = OP_IMM;  in em_aam()
3446 	ctxt->src.val = 0;  in em_aam()
3447 	ctxt->src.bytes = 1;  in em_aam()
3448 	fastop(ctxt, em_or);  in em_aam()
3453 static int em_aad(struct x86_emulate_ctxt *ctxt)  in em_aad()  argument
3455 	u8 al = ctxt->dst.val & 0xff;  in em_aad()
3456 	u8 ah = (ctxt->dst.val >> 8) & 0xff;  in em_aad()
3458 	al = (al + (ah * ctxt->src.val)) & 0xff;  in em_aad()
3460 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;  in em_aad()
3463 	ctxt->src.type = OP_IMM;  in em_aad()
3464 	ctxt->src.val = 0;  in em_aad()
3465 	ctxt->src.bytes = 1;  in em_aad()
3466 	fastop(ctxt, em_or);  in em_aad()
3471 static int em_call(struct x86_emulate_ctxt *ctxt)  in em_call()  argument
3474 	long rel = ctxt->src.val;  in em_call()
3476 	ctxt->src.val = (unsigned long)ctxt->_eip;  in em_call()
3477 	rc = jmp_rel(ctxt, rel);  in em_call()
3480 	return em_push(ctxt);  in em_call()
3483 static int em_call_far(struct x86_emulate_ctxt *ctxt)  in em_call_far()  argument
3489 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_call_far()
3490 	int cpl = ctxt->ops->cpl(ctxt);  in em_call_far()
3491 	enum x86emul_mode prev_mode = ctxt->mode;  in em_call_far()
3493 	old_eip = ctxt->_eip;  in em_call_far()
3494 	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);  in em_call_far()
3496 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_call_far()
3497 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,  in em_call_far()
3502 	rc = assign_eip_far(ctxt, ctxt->src.val);  in em_call_far()
3506 	ctxt->src.val = old_cs;  in em_call_far()
3507 	rc = em_push(ctxt);  in em_call_far()
3511 	ctxt->src.val = old_eip;  in em_call_far()
3512 	rc = em_push(ctxt);  in em_call_far()
3521 	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);  in em_call_far()
3522 	ctxt->mode = prev_mode;  in em_call_far()
3527 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)  in em_ret_near_imm()  argument
3532 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret_near_imm()
3535 	rc = assign_eip_near(ctxt, eip);  in em_ret_near_imm()
3538 	rsp_increment(ctxt, ctxt->src.val);  in em_ret_near_imm()
3542 static int em_xchg(struct x86_emulate_ctxt *ctxt)  in em_xchg()  argument
3545 	ctxt->src.val = ctxt->dst.val;  in em_xchg()
3546 	write_register_operand(&ctxt->src);  in em_xchg()
3549 	ctxt->dst.val = ctxt->src.orig_val;  in em_xchg()
3550 	ctxt->lock_prefix = 1;  in em_xchg()
3554 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)  in em_imul_3op()  argument
3556 	ctxt->dst.val = ctxt->src2.val;  in em_imul_3op()
3557 	return fastop(ctxt, em_imul);  in em_imul_3op()
3560 static int em_cwd(struct x86_emulate_ctxt *ctxt)  in em_cwd()  argument
3562 	ctxt->dst.type = OP_REG;  in em_cwd()
3563 	ctxt->dst.bytes = ctxt->src.bytes;  in em_cwd()
3564 	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in em_cwd()
3565 	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);  in em_cwd()
3570 static int em_rdpid(struct x86_emulate_ctxt *ctxt)  in em_rdpid()  argument
3574 	if (!ctxt->ops->guest_has_rdpid(ctxt))  in em_rdpid()
3575 		return emulate_ud(ctxt);  in em_rdpid()
3577 	ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);  in em_rdpid()
3578 	ctxt->dst.val = tsc_aux;  in em_rdpid()
3582 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)  in em_rdtsc()  argument
3586 	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);  in em_rdtsc()
3587 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;  in em_rdtsc()
3588 	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;  in em_rdtsc()
3592 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)  in em_rdpmc()  argument
3596 	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))  in em_rdpmc()
3597 		return emulate_gp(ctxt, 0);  in em_rdpmc()
3598 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;  in em_rdpmc()
3599 	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;  in em_rdpmc()
3603 static int em_mov(struct x86_emulate_ctxt *ctxt)  in em_mov()  argument
3605 	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));  in em_mov()
3609 static int em_movbe(struct x86_emulate_ctxt *ctxt)  in em_movbe()  argument
3613 	if (!ctxt->ops->guest_has_movbe(ctxt))  in em_movbe()
3614 		return emulate_ud(ctxt);  in em_movbe()
3616 	switch (ctxt->op_bytes) {  in em_movbe()
3626 		tmp = (u16)ctxt->src.val;  in em_movbe()
3627 		ctxt->dst.val &= ~0xffffUL;  in em_movbe()
3628 		ctxt->dst.val |= (unsigned long)swab16(tmp);  in em_movbe()
3631 		ctxt->dst.val = swab32((u32)ctxt->src.val);  in em_movbe()
3634 		ctxt->dst.val = swab64(ctxt->src.val);  in em_movbe()
3642 static int em_cr_write(struct x86_emulate_ctxt *ctxt)  in em_cr_write()  argument
3644 	int cr_num = ctxt->modrm_reg;  in em_cr_write()
3647 	if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val))  in em_cr_write()
3648 		return emulate_gp(ctxt, 0);  in em_cr_write()
3651 	ctxt->dst.type = OP_NONE;  in em_cr_write()
3658 		r = emulator_recalc_and_set_mode(ctxt);  in em_cr_write()
3666 static int em_dr_write(struct x86_emulate_ctxt *ctxt)  in em_dr_write()  argument
3670 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_dr_write()
3671 		val = ctxt->src.val & ~0ULL;  in em_dr_write()
3673 		val = ctxt->src.val & ~0U;  in em_dr_write()
3676 	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)  in em_dr_write()
3677 		return emulate_gp(ctxt, 0);  in em_dr_write()
3680 	ctxt->dst.type = OP_NONE;  in em_dr_write()
3684 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)  in em_wrmsr()  argument
3686 	u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);  in em_wrmsr()
3690 	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)  in em_wrmsr()
3691 		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);  in em_wrmsr()
3692 	r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data);  in em_wrmsr()
3695 		return emulate_gp(ctxt, 0);  in em_wrmsr()
3700 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)  in em_rdmsr()  argument
3702 	u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);  in em_rdmsr()
3706 	r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data);  in em_rdmsr()
3709 		return emulate_gp(ctxt, 0);  in em_rdmsr()
3712 		*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;  in em_rdmsr()
3713 		*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;  in em_rdmsr()
3718 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)  in em_store_sreg()  argument
3721 	    (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in em_store_sreg()
3722 	    ctxt->ops->cpl(ctxt) > 0)  in em_store_sreg()
3723 		return emulate_gp(ctxt, 0);  in em_store_sreg()
3725 	ctxt->dst.val = get_segment_selector(ctxt, segment);  in em_store_sreg()
3726 	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)  in em_store_sreg()
3727 		ctxt->dst.bytes = 2;  in em_store_sreg()
3731 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)  in em_mov_rm_sreg()  argument
3733 	if (ctxt->modrm_reg > VCPU_SREG_GS)  in em_mov_rm_sreg()
3734 		return emulate_ud(ctxt);  in em_mov_rm_sreg()
3736 	return em_store_sreg(ctxt, ctxt->modrm_reg);  in em_mov_rm_sreg()
3739 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)  in em_mov_sreg_rm()  argument
3741 	u16 sel = ctxt->src.val;  in em_mov_sreg_rm()
3743 	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)  in em_mov_sreg_rm()
3744 		return emulate_ud(ctxt);  in em_mov_sreg_rm()
3746 	if (ctxt->modrm_reg == VCPU_SREG_SS)  in em_mov_sreg_rm()
3747 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;  in em_mov_sreg_rm()
3750 	ctxt->dst.type = OP_NONE;  in em_mov_sreg_rm()
3751 	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);  in em_mov_sreg_rm()
3754 static int em_sldt(struct x86_emulate_ctxt *ctxt)  in em_sldt()  argument
3756 	return em_store_sreg(ctxt, VCPU_SREG_LDTR);  in em_sldt()
3759 static int em_lldt(struct x86_emulate_ctxt *ctxt)  in em_lldt()  argument
3761 	u16 sel = ctxt->src.val;  in em_lldt()
3764 	ctxt->dst.type = OP_NONE;  in em_lldt()
3765 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);  in em_lldt()
3768 static int em_str(struct x86_emulate_ctxt *ctxt)  in em_str()  argument
3770 	return em_store_sreg(ctxt, VCPU_SREG_TR);  in em_str()
3773 static int em_ltr(struct x86_emulate_ctxt *ctxt)  in em_ltr()  argument
3775 	u16 sel = ctxt->src.val;  in em_ltr()
3778 	ctxt->dst.type = OP_NONE;  in em_ltr()
3779 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);  in em_ltr()
3782 static int em_invlpg(struct x86_emulate_ctxt *ctxt)  in em_invlpg()  argument
3787 	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);  in em_invlpg()
3789 		ctxt->ops->invlpg(ctxt, linear);  in em_invlpg()
3791 	ctxt->dst.type = OP_NONE;  in em_invlpg()
3795 static int em_clts(struct x86_emulate_ctxt *ctxt)  in em_clts()  argument
3799 	cr0 = ctxt->ops->get_cr(ctxt, 0);  in em_clts()
3801 	ctxt->ops->set_cr(ctxt, 0, cr0);  in em_clts()
3805 static int em_hypercall(struct x86_emulate_ctxt *ctxt)  in em_hypercall()  argument
3807 	int rc = ctxt->ops->fix_hypercall(ctxt);  in em_hypercall()
3813 	ctxt->_eip = ctxt->eip;  in em_hypercall()
3815 	ctxt->dst.type = OP_NONE;  in em_hypercall()
3819 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,  in emulate_store_desc_ptr()  argument
3820 				  void (*get)(struct x86_emulate_ctxt *ctxt,  in emulate_store_desc_ptr()  argument
3825 	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in emulate_store_desc_ptr()
3826 	    ctxt->ops->cpl(ctxt) > 0)  in emulate_store_desc_ptr()
3827 		return emulate_gp(ctxt, 0);  in emulate_store_desc_ptr()
3829 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in emulate_store_desc_ptr()
3830 		ctxt->op_bytes = 8;  in emulate_store_desc_ptr()
3831 	get(ctxt, &desc_ptr);  in emulate_store_desc_ptr()
3832 	if (ctxt->op_bytes == 2) {  in emulate_store_desc_ptr()
3833 		ctxt->op_bytes = 4;  in emulate_store_desc_ptr()
3837 	ctxt->dst.type = OP_NONE;  in emulate_store_desc_ptr()
3838 	return segmented_write_std(ctxt, ctxt->dst.addr.mem,  in emulate_store_desc_ptr()
3839 				   &desc_ptr, 2 + ctxt->op_bytes);  in emulate_store_desc_ptr()
3842 static int em_sgdt(struct x86_emulate_ctxt *ctxt)  in em_sgdt()  argument
3844 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);  in em_sgdt()
3847 static int em_sidt(struct x86_emulate_ctxt *ctxt)  in em_sidt()  argument
3849 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);  in em_sidt()
3852 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)  in em_lgdt_lidt()  argument
3857 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_lgdt_lidt()
3858 		ctxt->op_bytes = 8;  in em_lgdt_lidt()
3859 	rc = read_descriptor(ctxt, ctxt->src.addr.mem,  in em_lgdt_lidt()
3861 			     ctxt->op_bytes);  in em_lgdt_lidt()
3864 	if (ctxt->mode == X86EMUL_MODE_PROT64 &&  in em_lgdt_lidt()
3865 	    emul_is_noncanonical_address(desc_ptr.address, ctxt))  in em_lgdt_lidt()
3866 		return emulate_gp(ctxt, 0);  in em_lgdt_lidt()
3868 		ctxt->ops->set_gdt(ctxt, &desc_ptr);  in em_lgdt_lidt()
3870 		ctxt->ops->set_idt(ctxt, &desc_ptr);  in em_lgdt_lidt()
3872 	ctxt->dst.type = OP_NONE;  in em_lgdt_lidt()
3876 static int em_lgdt(struct x86_emulate_ctxt *ctxt)  in em_lgdt()  argument
3878 	return em_lgdt_lidt(ctxt, true);  in em_lgdt()
3881 static int em_lidt(struct x86_emulate_ctxt *ctxt)  in em_lidt()  argument
3883 	return em_lgdt_lidt(ctxt, false);  in em_lidt()
3886 static int em_smsw(struct x86_emulate_ctxt *ctxt)  in em_smsw()  argument
3888 	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in em_smsw()
3889 	    ctxt->ops->cpl(ctxt) > 0)  in em_smsw()
3890 		return emulate_gp(ctxt, 0);  in em_smsw()
3892 	if (ctxt->dst.type == OP_MEM)  in em_smsw()
3893 		ctxt->dst.bytes = 2;  in em_smsw()
3894 	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);  in em_smsw()
3898 static int em_lmsw(struct x86_emulate_ctxt *ctxt)  in em_lmsw()  argument
3900 	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)  in em_lmsw()
3901 			  | (ctxt->src.val & 0x0f));  in em_lmsw()
3902 	ctxt->dst.type = OP_NONE;  in em_lmsw()
3906 static int em_loop(struct x86_emulate_ctxt *ctxt)  in em_loop()  argument
3910 	register_address_increment(ctxt, VCPU_REGS_RCX, -1);  in em_loop()
3911 	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&  in em_loop()
3912 	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))  in em_loop()
3913 		rc = jmp_rel(ctxt, ctxt->src.val);  in em_loop()
3918 static int em_jcxz(struct x86_emulate_ctxt *ctxt)  in em_jcxz()  argument
3922 	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)  in em_jcxz()
3923 		rc = jmp_rel(ctxt, ctxt->src.val);  in em_jcxz()
3928 static int em_in(struct x86_emulate_ctxt *ctxt)  in em_in()  argument
3930 	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,  in em_in()
3931 			     &ctxt->dst.val))  in em_in()
3937 static int em_out(struct x86_emulate_ctxt *ctxt)  in em_out()  argument
3939 	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,  in em_out()
3940 				    &ctxt->src.val, 1);  in em_out()
3942 	ctxt->dst.type = OP_NONE;  in em_out()
3946 static int em_cli(struct x86_emulate_ctxt *ctxt)  in em_cli()  argument
3948 	if (emulator_bad_iopl(ctxt))  in em_cli()
3949 		return emulate_gp(ctxt, 0);  in em_cli()
3951 	ctxt->eflags &= ~X86_EFLAGS_IF;  in em_cli()
3955 static int em_sti(struct x86_emulate_ctxt *ctxt)  in em_sti()  argument
3957 	if (emulator_bad_iopl(ctxt))  in em_sti()
3958 		return emulate_gp(ctxt, 0);  in em_sti()
3960 	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;  in em_sti()
3961 	ctxt->eflags |= X86_EFLAGS_IF;  in em_sti()
3965 static int em_cpuid(struct x86_emulate_ctxt *ctxt)  in em_cpuid()  argument
3970 	ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);  in em_cpuid()
3972 	    ctxt->ops->cpl(ctxt)) {  in em_cpuid()
3973 		return emulate_gp(ctxt, 0);  in em_cpuid()
3976 	eax = reg_read(ctxt, VCPU_REGS_RAX);  in em_cpuid()
3977 	ecx = reg_read(ctxt, VCPU_REGS_RCX);  in em_cpuid()
3978 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);  in em_cpuid()
3979 	*reg_write(ctxt, VCPU_REGS_RAX) = eax;  in em_cpuid()
3980 	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;  in em_cpuid()
3981 	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;  in em_cpuid()
3982 	*reg_write(ctxt, VCPU_REGS_RDX) = edx;  in em_cpuid()
3986 static int em_sahf(struct x86_emulate_ctxt *ctxt)  in em_sahf()  argument
3992 	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;  in em_sahf()
3994 	ctxt->eflags &= ~0xffUL;  in em_sahf()
3995 	ctxt->eflags |= flags | X86_EFLAGS_FIXED;  in em_sahf()
3999 static int em_lahf(struct x86_emulate_ctxt *ctxt)  in em_lahf()  argument
4001 	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;  in em_lahf()
4002 	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;  in em_lahf()
4006 static int em_bswap(struct x86_emulate_ctxt *ctxt)  in em_bswap()  argument
4008 	switch (ctxt->op_bytes) {  in em_bswap()
4011 		asm("bswap %0" : "+r"(ctxt->dst.val));  in em_bswap()
4015 		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));  in em_bswap()
4021 static int em_clflush(struct x86_emulate_ctxt *ctxt)  in em_clflush()  argument
4027 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)  in em_clflushopt()  argument
4033 static int em_movsxd(struct x86_emulate_ctxt *ctxt)  in em_movsxd()  argument
4035 	ctxt->dst.val = (s32) ctxt->src.val;  in em_movsxd()
4039 static int check_fxsr(struct x86_emulate_ctxt *ctxt)  in check_fxsr()  argument
4041 	if (!ctxt->ops->guest_has_fxsr(ctxt))  in check_fxsr()
4042 		return emulate_ud(ctxt);  in check_fxsr()
4044 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in check_fxsr()
4045 		return emulate_nm(ctxt);  in check_fxsr()
4051 	if (ctxt->mode >= X86EMUL_MODE_PROT64)  in check_fxsr()
4066 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)  in fxstate_size()  argument
4069 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in fxstate_size()
4072 	cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;  in fxstate_size()
4094 static int em_fxsave(struct x86_emulate_ctxt *ctxt)  in em_fxsave()  argument
4099 	rc = check_fxsr(ctxt);  in em_fxsave()
4112 	return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,  in em_fxsave()
4113 		                   fxstate_size(ctxt));  in em_fxsave()
4136 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)  in em_fxrstor()  argument
4142 	rc = check_fxsr(ctxt);  in em_fxrstor()
4146 	size = fxstate_size(ctxt);  in em_fxrstor()
4147 	rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);  in em_fxrstor()
4160 		rc = emulate_gp(ctxt, 0);  in em_fxrstor()
4173 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)  in em_xsetbv()  argument
4177 	if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE))  in em_xsetbv()
4178 		return emulate_ud(ctxt);  in em_xsetbv()
4180 	eax = reg_read(ctxt, VCPU_REGS_RAX);  in em_xsetbv()
4181 	edx = reg_read(ctxt, VCPU_REGS_RDX);  in em_xsetbv()
4182 	ecx = reg_read(ctxt, VCPU_REGS_RCX);  in em_xsetbv()
4184 	if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))  in em_xsetbv()
4185 		return emulate_gp(ctxt, 0);  in em_xsetbv()
4202 static int check_cr_access(struct x86_emulate_ctxt *ctxt)  in check_cr_access()  argument
4204 	if (!valid_cr(ctxt->modrm_reg))  in check_cr_access()
4205 		return emulate_ud(ctxt);  in check_cr_access()
4210 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)  in check_dr7_gd()  argument
4214 	ctxt->ops->get_dr(ctxt, 7, &dr7);  in check_dr7_gd()
4219 static int check_dr_read(struct x86_emulate_ctxt *ctxt)  in check_dr_read()  argument
4221 	int dr = ctxt->modrm_reg;  in check_dr_read()
4225 		return emulate_ud(ctxt);  in check_dr_read()
4227 	cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_dr_read()
4229 		return emulate_ud(ctxt);  in check_dr_read()
4231 	if (check_dr7_gd(ctxt)) {  in check_dr_read()
4234 		ctxt->ops->get_dr(ctxt, 6, &dr6);  in check_dr_read()
4237 		ctxt->ops->set_dr(ctxt, 6, dr6);  in check_dr_read()
4238 		return emulate_db(ctxt);  in check_dr_read()
4244 static int check_dr_write(struct x86_emulate_ctxt *ctxt)  in check_dr_write()  argument
4246 	u64 new_val = ctxt->src.val64;  in check_dr_write()
4247 	int dr = ctxt->modrm_reg;  in check_dr_write()
4250 		return emulate_gp(ctxt, 0);  in check_dr_write()
4252 	return check_dr_read(ctxt);  in check_dr_write()
4255 static int check_svme(struct x86_emulate_ctxt *ctxt)  in check_svme()  argument
4259 	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in check_svme()
4262 		return emulate_ud(ctxt);  in check_svme()
4267 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)  in check_svme_pa()  argument
4269 	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);  in check_svme_pa()
4273 		return emulate_gp(ctxt, 0);  in check_svme_pa()
4275 	return check_svme(ctxt);  in check_svme_pa()
4278 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)  in check_rdtsc()  argument
4280 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_rdtsc()
4282 	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))  in check_rdtsc()
4283 		return emulate_gp(ctxt, 0);  in check_rdtsc()
4288 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)  in check_rdpmc()  argument
4290 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_rdpmc()
4291 	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);  in check_rdpmc()
4305 	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||  in check_rdpmc()
4306 	    ctxt->ops->check_pmc(ctxt, rcx))  in check_rdpmc()
4307 		return emulate_gp(ctxt, 0);  in check_rdpmc()
4312 static int check_perm_in(struct x86_emulate_ctxt *ctxt)  in check_perm_in()  argument
4314 	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);  in check_perm_in()
4315 	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))  in check_perm_in()
4316 		return emulate_gp(ctxt, 0);  in check_perm_in()
4321 static int check_perm_out(struct x86_emulate_ctxt *ctxt)  in check_perm_out()  argument
4323 	ctxt->src.bytes = min(ctxt->src.bytes, 4u);  in check_perm_out()
4324 	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))  in check_perm_out()
4325 		return emulate_gp(ctxt, 0);  in check_perm_out()
4888 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)  in imm_size()  argument
4892 	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in imm_size()
4898 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,  in decode_imm()  argument
4905 	op->addr.mem.ea = ctxt->_eip;  in decode_imm()
4909 		op->val = insn_fetch(s8, ctxt);  in decode_imm()
4912 		op->val = insn_fetch(s16, ctxt);  in decode_imm()
4915 		op->val = insn_fetch(s32, ctxt);  in decode_imm()
4918 		op->val = insn_fetch(s64, ctxt);  in decode_imm()
4938 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,  in decode_operand()  argument
4945 		decode_register_operand(ctxt, op);  in decode_operand()
4948 		rc = decode_imm(ctxt, op, 1, false);  in decode_operand()
4951 		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4953 		*op = ctxt->memop;  in decode_operand()
4954 		ctxt->memopp = op;  in decode_operand()
4955 		if (ctxt->d & BitOp)  in decode_operand()
4956 			fetch_bit_operand(ctxt);  in decode_operand()
4960 		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;  in decode_operand()
4964 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4965 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in decode_operand()
4971 		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;  in decode_operand()
4972 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in decode_operand()
4977 		if (ctxt->d & ByteOp) {  in decode_operand()
4982 		op->bytes = ctxt->op_bytes;  in decode_operand()
4983 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in decode_operand()
4989 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4991 			register_address(ctxt, VCPU_REGS_RDI);  in decode_operand()
4999 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in decode_operand()
5005 		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;  in decode_operand()
5008 		rc = decode_imm(ctxt, op, 1, true);  in decode_operand()
5016 		rc = decode_imm(ctxt, op, imm_size(ctxt), true);  in decode_operand()
5019 		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);  in decode_operand()
5022 		ctxt->memop.bytes = 1;  in decode_operand()
5023 		if (ctxt->memop.type == OP_REG) {  in decode_operand()
5024 			ctxt->memop.addr.reg = decode_register(ctxt,  in decode_operand()
5025 					ctxt->modrm_rm, true);  in decode_operand()
5026 			fetch_register_operand(&ctxt->memop);  in decode_operand()
5030 		ctxt->memop.bytes = 2;  in decode_operand()
5033 		ctxt->memop.bytes = 4;  in decode_operand()
5036 		rc = decode_imm(ctxt, op, 2, false);  in decode_operand()
5039 		rc = decode_imm(ctxt, op, imm_size(ctxt), false);  in decode_operand()
5043 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
5045 			register_address(ctxt, VCPU_REGS_RSI);  in decode_operand()
5046 		op->addr.mem.seg = ctxt->seg_override;  in decode_operand()
5052 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
5054 			address_mask(ctxt,  in decode_operand()
5055 				reg_read(ctxt, VCPU_REGS_RBX) +  in decode_operand()
5056 				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));  in decode_operand()
5057 		op->addr.mem.seg = ctxt->seg_override;  in decode_operand()
5062 		op->addr.mem.ea = ctxt->_eip;  in decode_operand()
5063 		op->bytes = ctxt->op_bytes + 2;  in decode_operand()
5064 		insn_fetch_arr(op->valptr, op->bytes, ctxt);  in decode_operand()
5067 		ctxt->memop.bytes = ctxt->op_bytes + 2;  in decode_operand()
5104 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type)  in x86_decode_insn()  argument
5107 	int mode = ctxt->mode;  in x86_decode_insn()
5115 	ctxt->memop.type = OP_NONE;  in x86_decode_insn()
5116 	ctxt->memopp = NULL;  in x86_decode_insn()
5117 	ctxt->_eip = ctxt->eip;  in x86_decode_insn()
5118 	ctxt->fetch.ptr = ctxt->fetch.data;  in x86_decode_insn()
5119 	ctxt->fetch.end = ctxt->fetch.data + insn_len;  in x86_decode_insn()
5120 	ctxt->opcode_len = 1;  in x86_decode_insn()
5121 	ctxt->intercept = x86_intercept_none;  in x86_decode_insn()
5123 		memcpy(ctxt->fetch.data, insn, insn_len);  in x86_decode_insn()
5125 		rc = __do_insn_fetch_bytes(ctxt, 1);  in x86_decode_insn()
5134 		ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);  in x86_decode_insn()
5154 	ctxt->op_bytes = def_op_bytes;  in x86_decode_insn()
5155 	ctxt->ad_bytes = def_ad_bytes;  in x86_decode_insn()
5159 		switch (ctxt->b = insn_fetch(u8, ctxt)) {  in x86_decode_insn()
5163 			ctxt->op_bytes = def_op_bytes ^ 6;  in x86_decode_insn()
5168 				ctxt->ad_bytes = def_ad_bytes ^ 12;  in x86_decode_insn()
5171 				ctxt->ad_bytes = def_ad_bytes ^ 6;  in x86_decode_insn()
5175 			ctxt->seg_override = VCPU_SREG_ES;  in x86_decode_insn()
5179 			ctxt->seg_override = VCPU_SREG_CS;  in x86_decode_insn()
5183 			ctxt->seg_override = VCPU_SREG_SS;  in x86_decode_insn()
5187 			ctxt->seg_override = VCPU_SREG_DS;  in x86_decode_insn()
5191 			ctxt->seg_override = VCPU_SREG_FS;  in x86_decode_insn()
5195 			ctxt->seg_override = VCPU_SREG_GS;  in x86_decode_insn()
5200 			ctxt->rex_prefix = ctxt->b;  in x86_decode_insn()
5203 			ctxt->lock_prefix = 1;  in x86_decode_insn()
5207 			ctxt->rep_prefix = ctxt->b;  in x86_decode_insn()
5215 		ctxt->rex_prefix = 0;  in x86_decode_insn()
5221 	if (ctxt->rex_prefix & 8)  in x86_decode_insn()
5222 		ctxt->op_bytes = 8;	/* REX.W */  in x86_decode_insn()
5225 	opcode = opcode_table[ctxt->b];  in x86_decode_insn()
5227 	if (ctxt->b == 0x0f) {  in x86_decode_insn()
5228 		ctxt->opcode_len = 2;  in x86_decode_insn()
5229 		ctxt->b = insn_fetch(u8, ctxt);  in x86_decode_insn()
5230 		opcode = twobyte_table[ctxt->b];  in x86_decode_insn()
5233 		if (ctxt->b == 0x38) {  in x86_decode_insn()
5234 			ctxt->opcode_len = 3;  in x86_decode_insn()
5235 			ctxt->b = insn_fetch(u8, ctxt);  in x86_decode_insn()
5236 			opcode = opcode_map_0f_38[ctxt->b];  in x86_decode_insn()
5239 	ctxt->d = opcode.flags;  in x86_decode_insn()
5241 	if (ctxt->d & ModRM)  in x86_decode_insn()
5242 		ctxt->modrm = insn_fetch(u8, ctxt);  in x86_decode_insn()
5245 	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&  in x86_decode_insn()
5246 	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {  in x86_decode_insn()
5247 		ctxt->d = NotImpl;  in x86_decode_insn()
5250 	while (ctxt->d & GroupMask) {  in x86_decode_insn()
5251 		switch (ctxt->d & GroupMask) {  in x86_decode_insn()
5253 			goffset = (ctxt->modrm >> 3) & 7;  in x86_decode_insn()
5257 			goffset = (ctxt->modrm >> 3) & 7;  in x86_decode_insn()
5258 			if ((ctxt->modrm >> 6) == 3)  in x86_decode_insn()
5264 			goffset = ctxt->modrm & 7;  in x86_decode_insn()
5268 			if (ctxt->rep_prefix && op_prefix)  in x86_decode_insn()
5270 			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;  in x86_decode_insn()
5279 			if (ctxt->modrm > 0xbf) {  in x86_decode_insn()
5282 					ctxt->modrm - 0xc0, size);  in x86_decode_insn()
5286 				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];  in x86_decode_insn()
5290 			if ((ctxt->modrm >> 6) == 3)  in x86_decode_insn()
5296 			if (ctxt->mode == X86EMUL_MODE_PROT64)  in x86_decode_insn()
5305 		ctxt->d &= ~(u64)GroupMask;  in x86_decode_insn()
5306 		ctxt->d |= opcode.flags;  in x86_decode_insn()
5309 	ctxt->is_branch = opcode.flags & IsBranch;  in x86_decode_insn()
5312 	if (ctxt->d == 0)  in x86_decode_insn()
5315 	ctxt->execute = opcode.u.execute;  in x86_decode_insn()
5318 	    likely(!(ctxt->d & EmulateOnUD)))  in x86_decode_insn()
5321 	if (unlikely(ctxt->d &  in x86_decode_insn()
5328 		ctxt->check_perm = opcode.check_perm;  in x86_decode_insn()
5329 		ctxt->intercept = opcode.intercept;  in x86_decode_insn()
5331 		if (ctxt->d & NotImpl)  in x86_decode_insn()
5335 			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))  in x86_decode_insn()
5336 				ctxt->op_bytes = 8;  in x86_decode_insn()
5337 			else if (ctxt->d & NearBranch)  in x86_decode_insn()
5338 				ctxt->op_bytes = 8;  in x86_decode_insn()
5341 		if (ctxt->d & Op3264) {  in x86_decode_insn()
5343 				ctxt->op_bytes = 8;  in x86_decode_insn()
5345 				ctxt->op_bytes = 4;  in x86_decode_insn()
5348 		if ((ctxt->d & No16) && ctxt->op_bytes == 2)  in x86_decode_insn()
5349 			ctxt->op_bytes = 4;  in x86_decode_insn()
5351 		if (ctxt->d & Sse)  in x86_decode_insn()
5352 			ctxt->op_bytes = 16;  in x86_decode_insn()
5353 		else if (ctxt->d & Mmx)  in x86_decode_insn()
5354 			ctxt->op_bytes = 8;  in x86_decode_insn()
5358 	if (ctxt->d & ModRM) {  in x86_decode_insn()
5359 		rc = decode_modrm(ctxt, &ctxt->memop);  in x86_decode_insn()
5362 			ctxt->seg_override = ctxt->modrm_seg;  in x86_decode_insn()
5364 	} else if (ctxt->d & MemAbs)  in x86_decode_insn()
5365 		rc = decode_abs(ctxt, &ctxt->memop);  in x86_decode_insn()
5370 		ctxt->seg_override = VCPU_SREG_DS;  in x86_decode_insn()
5372 	ctxt->memop.addr.mem.seg = ctxt->seg_override;  in x86_decode_insn()
5378 	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);  in x86_decode_insn()
5386 	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);  in x86_decode_insn()
5391 	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);  in x86_decode_insn()
5393 	if (ctxt->rip_relative && likely(ctxt->memopp))  in x86_decode_insn()
5394 		ctxt->memopp->addr.mem.ea = address_mask(ctxt,  in x86_decode_insn()
5395 					ctxt->memopp->addr.mem.ea + ctxt->_eip);  in x86_decode_insn()
5399 		ctxt->have_exception = true;  in x86_decode_insn()
5403 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)  in x86_page_table_writing_insn()  argument
5405 	return ctxt->d & PageTable;  in x86_page_table_writing_insn()
5408 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)  in string_insn_completed()  argument
5417 	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||  in string_insn_completed()
5418 	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))  in string_insn_completed()
5419 	    && (((ctxt->rep_prefix == REPE_PREFIX) &&  in string_insn_completed()
5420 		 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))  in string_insn_completed()
5421 		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&  in string_insn_completed()
5422 		    ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))  in string_insn_completed()
5428 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)  in flush_pending_x87_faults()  argument
5437 		return emulate_exception(ctxt, MF_VECTOR, 0, false);  in flush_pending_x87_faults()
5448 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)  in fastop()  argument
5450 	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;  in fastop()
5452 	if (!(ctxt->d & ByteOp))  in fastop()
5453 		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;  in fastop()
5456 	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),  in fastop()
5458 	    : "c"(ctxt->src2.val));  in fastop()
5460 	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);  in fastop()
5462 		return emulate_de(ctxt);  in fastop()
5466 void init_decode_cache(struct x86_emulate_ctxt *ctxt)  in init_decode_cache()  argument
5469 	ctxt->rip_relative = false;  in init_decode_cache()
5470 	ctxt->rex_prefix = 0;  in init_decode_cache()
5471 	ctxt->lock_prefix = 0;  in init_decode_cache()
5472 	ctxt->rep_prefix = 0;  in init_decode_cache()
5473 	ctxt->regs_valid = 0;  in init_decode_cache()
5474 	ctxt->regs_dirty = 0;  in init_decode_cache()
5476 	ctxt->io_read.pos = 0;  in init_decode_cache()
5477 	ctxt->io_read.end = 0;  in init_decode_cache()
5478 	ctxt->mem_read.end = 0;  in init_decode_cache()
5481 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)  in x86_emulate_insn()  argument
5483 	const struct x86_emulate_ops *ops = ctxt->ops;  in x86_emulate_insn()
5485 	int saved_dst_type = ctxt->dst.type;  in x86_emulate_insn()
5488 	ctxt->mem_read.pos = 0;  in x86_emulate_insn()
5491 	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {  in x86_emulate_insn()
5492 		rc = emulate_ud(ctxt);  in x86_emulate_insn()
5496 	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {  in x86_emulate_insn()
5497 		rc = emulate_ud(ctxt);  in x86_emulate_insn()
5501 	emul_flags = ctxt->ops->get_hflags(ctxt);  in x86_emulate_insn()
5502 	if (unlikely(ctxt->d &  in x86_emulate_insn()
5504 		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||  in x86_emulate_insn()
5505 				(ctxt->d & Undefined)) {  in x86_emulate_insn()
5506 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5510 		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))  in x86_emulate_insn()
5511 		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {  in x86_emulate_insn()
5512 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5516 		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {  in x86_emulate_insn()
5517 			rc = emulate_nm(ctxt);  in x86_emulate_insn()
5521 		if (ctxt->d & Mmx) {  in x86_emulate_insn()
5522 			rc = flush_pending_x87_faults(ctxt);  in x86_emulate_insn()
5529 			fetch_possible_mmx_operand(&ctxt->src);  in x86_emulate_insn()
5530 			fetch_possible_mmx_operand(&ctxt->src2);  in x86_emulate_insn()
5531 			if (!(ctxt->d & Mov))  in x86_emulate_insn()
5532 				fetch_possible_mmx_operand(&ctxt->dst);  in x86_emulate_insn()
5535 		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {  in x86_emulate_insn()
5536 			rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5543 		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {  in x86_emulate_insn()
5544 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5549 		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {  in x86_emulate_insn()
5550 			if (ctxt->d & PrivUD)  in x86_emulate_insn()
5551 				rc = emulate_ud(ctxt);  in x86_emulate_insn()
5553 				rc = emulate_gp(ctxt, 0);  in x86_emulate_insn()
5558 		if (ctxt->d & CheckPerm) {  in x86_emulate_insn()
5559 			rc = ctxt->check_perm(ctxt);  in x86_emulate_insn()
5564 		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {  in x86_emulate_insn()
5565 			rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5571 		if (ctxt->rep_prefix && (ctxt->d & String)) {  in x86_emulate_insn()
5573 			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {  in x86_emulate_insn()
5574 				string_registers_quirk(ctxt);  in x86_emulate_insn()
5575 				ctxt->eip = ctxt->_eip;  in x86_emulate_insn()
5576 				ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5582 	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {  in x86_emulate_insn()
5583 		rc = segmented_read(ctxt, ctxt->src.addr.mem,  in x86_emulate_insn()
5584 				    ctxt->src.valptr, ctxt->src.bytes);  in x86_emulate_insn()
5587 		ctxt->src.orig_val64 = ctxt->src.val64;  in x86_emulate_insn()
5590 	if (ctxt->src2.type == OP_MEM) {  in x86_emulate_insn()
5591 		rc = segmented_read(ctxt, ctxt->src2.addr.mem,  in x86_emulate_insn()
5592 				    &ctxt->src2.val, ctxt->src2.bytes);  in x86_emulate_insn()
5597 	if ((ctxt->d & DstMask) == ImplicitOps)  in x86_emulate_insn()
5601 	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {  in x86_emulate_insn()
5603 		rc = segmented_read(ctxt, ctxt->dst.addr.mem,  in x86_emulate_insn()
5604 				   &ctxt->dst.val, ctxt->dst.bytes);  in x86_emulate_insn()
5606 			if (!(ctxt->d & NoWrite) &&  in x86_emulate_insn()
5608 			    ctxt->exception.vector == PF_VECTOR)  in x86_emulate_insn()
5609 				ctxt->exception.error_code |= PFERR_WRITE_MASK;  in x86_emulate_insn()
5614 	ctxt->dst.orig_val64 = ctxt->dst.val64;  in x86_emulate_insn()
5618 	if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {  in x86_emulate_insn()
5619 		rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5625 	if (ctxt->rep_prefix && (ctxt->d & String))  in x86_emulate_insn()
5626 		ctxt->eflags |= X86_EFLAGS_RF;  in x86_emulate_insn()
5628 		ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5630 	if (ctxt->execute) {  in x86_emulate_insn()
5631 		if (ctxt->d & Fastop)  in x86_emulate_insn()
5632 			rc = fastop(ctxt, ctxt->fop);  in x86_emulate_insn()
5634 			rc = ctxt->execute(ctxt);  in x86_emulate_insn()
5640 	if (ctxt->opcode_len == 2)  in x86_emulate_insn()
5642 	else if (ctxt->opcode_len == 3)  in x86_emulate_insn()
5645 	switch (ctxt->b) {  in x86_emulate_insn()
5647 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5648 			rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5651 		ctxt->dst.val = ctxt->src.addr.mem.ea;  in x86_emulate_insn()
5654 		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))  in x86_emulate_insn()
5655 			ctxt->dst.type = OP_NONE;  in x86_emulate_insn()
5657 			rc = em_xchg(ctxt);  in x86_emulate_insn()
5660 		switch (ctxt->op_bytes) {  in x86_emulate_insn()
5661 		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;  in x86_emulate_insn()
5662 		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;  in x86_emulate_insn()
5663 		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;  in x86_emulate_insn()
5667 		rc = emulate_int(ctxt, 3);  in x86_emulate_insn()
5670 		rc = emulate_int(ctxt, ctxt->src.val);  in x86_emulate_insn()
5673 		if (ctxt->eflags & X86_EFLAGS_OF)  in x86_emulate_insn()
5674 			rc = emulate_int(ctxt, 4);  in x86_emulate_insn()
5678 		rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5679 		ctxt->dst.type = OP_NONE; /* Disable writeback. */  in x86_emulate_insn()
5682 		ctxt->ops->halt(ctxt);  in x86_emulate_insn()
5686 		ctxt->eflags ^= X86_EFLAGS_CF;  in x86_emulate_insn()
5689 		ctxt->eflags &= ~X86_EFLAGS_CF;  in x86_emulate_insn()
5692 		ctxt->eflags |= X86_EFLAGS_CF;  in x86_emulate_insn()
5695 		ctxt->eflags &= ~X86_EFLAGS_DF;  in x86_emulate_insn()
5698 		ctxt->eflags |= X86_EFLAGS_DF;  in x86_emulate_insn()
5708 	if (ctxt->d & SrcWrite) {  in x86_emulate_insn()
5709 		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);  in x86_emulate_insn()
5710 		rc = writeback(ctxt, &ctxt->src);  in x86_emulate_insn()
5714 	if (!(ctxt->d & NoWrite)) {  in x86_emulate_insn()
5715 		rc = writeback(ctxt, &ctxt->dst);  in x86_emulate_insn()
5724 	ctxt->dst.type = saved_dst_type;  in x86_emulate_insn()
5726 	if ((ctxt->d & SrcMask) == SrcSI)  in x86_emulate_insn()
5727 		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);  in x86_emulate_insn()
5729 	if ((ctxt->d & DstMask) == DstDI)  in x86_emulate_insn()
5730 		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);  in x86_emulate_insn()
5732 	if (ctxt->rep_prefix && (ctxt->d & String)) {  in x86_emulate_insn()
5734 		struct read_cache *r = &ctxt->io_read;  in x86_emulate_insn()
5735 		if ((ctxt->d & SrcMask) == SrcSI)  in x86_emulate_insn()
5736 			count = ctxt->src.count;  in x86_emulate_insn()
5738 			count = ctxt->dst.count;  in x86_emulate_insn()
5739 		register_address_increment(ctxt, VCPU_REGS_RCX, -count);  in x86_emulate_insn()
5741 		if (!string_insn_completed(ctxt)) {  in x86_emulate_insn()
5746 			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&  in x86_emulate_insn()
5753 				ctxt->mem_read.end = 0;  in x86_emulate_insn()
5754 				writeback_registers(ctxt);  in x86_emulate_insn()
5759 		ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5762 	ctxt->eip = ctxt->_eip;  in x86_emulate_insn()
5763 	if (ctxt->mode != X86EMUL_MODE_PROT64)  in x86_emulate_insn()
5764 		ctxt->eip = (u32)ctxt->_eip;  in x86_emulate_insn()
5768 		if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt))  in x86_emulate_insn()
5770 		ctxt->have_exception = true;  in x86_emulate_insn()
5776 		writeback_registers(ctxt);  in x86_emulate_insn()
5781 	switch (ctxt->b) {  in x86_emulate_insn()
5783 		(ctxt->ops->wbinvd)(ctxt);  in x86_emulate_insn()
5791 		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);  in x86_emulate_insn()
5794 		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);  in x86_emulate_insn()
5797 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5798 			ctxt->dst.val = ctxt->src.val;  in x86_emulate_insn()
5799 		else if (ctxt->op_bytes != 4)  in x86_emulate_insn()
5800 			ctxt->dst.type = OP_NONE; /* no writeback */  in x86_emulate_insn()
5803 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5804 			rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5807 		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);  in x86_emulate_insn()
5810 		ctxt->dst.bytes = ctxt->op_bytes;  in x86_emulate_insn()
5811 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val  in x86_emulate_insn()
5812 						       : (u16) ctxt->src.val;  in x86_emulate_insn()
5815 		ctxt->dst.bytes = ctxt->op_bytes;  in x86_emulate_insn()
5816 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :  in x86_emulate_insn()
5817 							(s16) ctxt->src.val;  in x86_emulate_insn()
5834 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)  in emulator_invalidate_register_cache()  argument
5836 	invalidate_registers(ctxt);  in emulator_invalidate_register_cache()
5839 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)  in emulator_writeback_register_cache()  argument
5841 	writeback_registers(ctxt);  in emulator_writeback_register_cache()
5844 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)  in emulator_can_use_gpa()  argument
5846 	if (ctxt->rep_prefix && (ctxt->d & String))  in emulator_can_use_gpa()
5849 	if (ctxt->d & TwoMemOp)  in emulator_can_use_gpa()