216 		int (*execute)(struct x86_emulate_ctxt *ctxt);
225 	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
264 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)  in reg_read()  argument
266 	if (!(ctxt->regs_valid & (1 << nr))) {  in reg_read()
267 		ctxt->regs_valid |= 1 << nr;  in reg_read()
268 		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);  in reg_read()
270 	return ctxt->_regs[nr];  in reg_read()
273 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)  in reg_write()  argument
275 	ctxt->regs_valid |= 1 << nr;  in reg_write()
276 	ctxt->regs_dirty |= 1 << nr;  in reg_write()
277 	return &ctxt->_regs[nr];  in reg_write()
280 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)  in reg_rmw()  argument
282 	reg_read(ctxt, nr);  in reg_rmw()
283 	return reg_write(ctxt, nr);  in reg_rmw()
286 static void writeback_registers(struct x86_emulate_ctxt *ctxt)  in writeback_registers()  argument
290 	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)  in writeback_registers()
291 		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);  in writeback_registers()
294 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)  in invalidate_registers()  argument
296 	ctxt->regs_dirty = 0;  in invalidate_registers()
297 	ctxt->regs_valid = 0;  in invalidate_registers()
313 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
493 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,  in emulator_check_intercept()  argument
499 		.rep_prefix = ctxt->rep_prefix,  in emulator_check_intercept()
500 		.modrm_mod  = ctxt->modrm_mod,  in emulator_check_intercept()
501 		.modrm_reg  = ctxt->modrm_reg,  in emulator_check_intercept()
502 		.modrm_rm   = ctxt->modrm_rm,  in emulator_check_intercept()
503 		.src_val    = ctxt->src.val64,  in emulator_check_intercept()
504 		.dst_val    = ctxt->dst.val64,  in emulator_check_intercept()
505 		.src_bytes  = ctxt->src.bytes,  in emulator_check_intercept()
506 		.dst_bytes  = ctxt->dst.bytes,  in emulator_check_intercept()
507 		.ad_bytes   = ctxt->ad_bytes,  in emulator_check_intercept()
508 		.next_rip   = ctxt->eip,  in emulator_check_intercept()
511 	return ctxt->ops->intercept(ctxt, &info, stage);  in emulator_check_intercept()
538 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)  in ad_mask()  argument
540 	return (1UL << (ctxt->ad_bytes << 3)) - 1;  in ad_mask()
543 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)  in stack_mask()  argument
548 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in stack_mask()
550 	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);  in stack_mask()
554 static int stack_size(struct x86_emulate_ctxt *ctxt)  in stack_size()  argument
556 	return (__fls(stack_mask(ctxt)) + 1) >> 3;  in stack_size()
561 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)  in address_mask()  argument
563 	if (ctxt->ad_bytes == sizeof(unsigned long))  in address_mask()
566 		return reg & ad_mask(ctxt);  in address_mask()
570 register_address(struct x86_emulate_ctxt *ctxt, int reg)  in register_address()  argument
572 	return address_mask(ctxt, reg_read(ctxt, reg));  in register_address()
581 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)  in register_address_increment()  argument
583 	ulong *preg = reg_rmw(ctxt, reg);  in register_address_increment()
585 	assign_register(preg, *preg + inc, ctxt->ad_bytes);  in register_address_increment()
588 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)  in rsp_increment()  argument
590 	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);  in rsp_increment()
600 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)  in seg_base()  argument
602 	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)  in seg_base()
605 	return ctxt->ops->get_cached_segment_base(ctxt, seg);  in seg_base()
608 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,  in emulate_exception()  argument
612 	ctxt->exception.vector = vec;  in emulate_exception()
613 	ctxt->exception.error_code = error;  in emulate_exception()
614 	ctxt->exception.error_code_valid = valid;  in emulate_exception()
618 static int emulate_db(struct x86_emulate_ctxt *ctxt)  in emulate_db()  argument
620 	return emulate_exception(ctxt, DB_VECTOR, 0, false);  in emulate_db()
623 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)  in emulate_gp()  argument
625 	return emulate_exception(ctxt, GP_VECTOR, err, true);  in emulate_gp()
628 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)  in emulate_ss()  argument
630 	return emulate_exception(ctxt, SS_VECTOR, err, true);  in emulate_ss()
633 static int emulate_ud(struct x86_emulate_ctxt *ctxt)  in emulate_ud()  argument
635 	return emulate_exception(ctxt, UD_VECTOR, 0, false);  in emulate_ud()
638 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)  in emulate_ts()  argument
640 	return emulate_exception(ctxt, TS_VECTOR, err, true);  in emulate_ts()
643 static int emulate_de(struct x86_emulate_ctxt *ctxt)  in emulate_de()  argument
645 	return emulate_exception(ctxt, DE_VECTOR, 0, false);  in emulate_de()
648 static int emulate_nm(struct x86_emulate_ctxt *ctxt)  in emulate_nm()  argument
650 	return emulate_exception(ctxt, NM_VECTOR, 0, false);  in emulate_nm()
653 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)  in get_segment_selector()  argument
658 	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);  in get_segment_selector()
662 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,  in set_segment_selector()  argument
669 	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);  in set_segment_selector()
670 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);  in set_segment_selector()
682 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)  in insn_alignment()  argument
684 	u64 alignment = ctxt->d & AlignMask;  in insn_alignment()
701 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,  in __linearize()  argument
714 	la = seg_base(ctxt, addr.seg) + addr.ea;  in __linearize()
719 		va_bits = ctxt_virt_addr_bits(ctxt);  in __linearize()
729 		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,  in __linearize()
734 		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))  in __linearize()
758 	if (la & (insn_alignment(ctxt, size) - 1))  in __linearize()
759 		return emulate_gp(ctxt, 0);  in __linearize()
763 		return emulate_ss(ctxt, 0);  in __linearize()
765 		return emulate_gp(ctxt, 0);  in __linearize()
768 static int linearize(struct x86_emulate_ctxt *ctxt,  in linearize()  argument
774 	return __linearize(ctxt, addr, &max_size, size, write, false,  in linearize()
775 			   ctxt->mode, linear);  in linearize()
778 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,  in assign_eip()  argument
787 	if (ctxt->op_bytes != sizeof(unsigned long))  in assign_eip()
788 		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);  in assign_eip()
789 	rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);  in assign_eip()
791 		ctxt->_eip = addr.ea;  in assign_eip()
795 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)  in assign_eip_near()  argument
797 	return assign_eip(ctxt, dst, ctxt->mode);  in assign_eip_near()
800 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,  in assign_eip_far()  argument
803 	enum x86emul_mode mode = ctxt->mode;  in assign_eip_far()
807 	if (ctxt->mode >= X86EMUL_MODE_PROT16) {  in assign_eip_far()
811 			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in assign_eip_far()
820 	rc = assign_eip(ctxt, dst, mode);  in assign_eip_far()
822 		ctxt->mode = mode;  in assign_eip_far()
826 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)  in jmp_rel()  argument
828 	return assign_eip_near(ctxt, ctxt->_eip + rel);  in jmp_rel()
831 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,  in linear_read_system()  argument
834 	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);  in linear_read_system()
837 static int linear_write_system(struct x86_emulate_ctxt *ctxt,  in linear_write_system()  argument
841 	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);  in linear_write_system()
844 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,  in segmented_read_std()  argument
852 	rc = linearize(ctxt, addr, size, false, &linear);  in segmented_read_std()
855 	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);  in segmented_read_std()
858 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,  in segmented_write_std()  argument
866 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_write_std()
869 	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);  in segmented_write_std()
876 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)  in __do_insn_fetch_bytes()  argument
881 	int cur_size = ctxt->fetch.end - ctxt->fetch.data;  in __do_insn_fetch_bytes()
883 					   .ea = ctxt->eip + cur_size };  in __do_insn_fetch_bytes()
895 	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,  in __do_insn_fetch_bytes()
910 		return emulate_gp(ctxt, 0);  in __do_insn_fetch_bytes()
912 	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,  in __do_insn_fetch_bytes()
913 			      size, &ctxt->exception);  in __do_insn_fetch_bytes()
916 	ctxt->fetch.end += size;  in __do_insn_fetch_bytes()
920 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,  in do_insn_fetch_bytes()  argument
923 	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;  in do_insn_fetch_bytes()
926 		return __do_insn_fetch_bytes(ctxt, size - done_size);  in do_insn_fetch_bytes()
938 	ctxt->_eip += sizeof(_type);					\
939 	memcpy(&_x, ctxt->fetch.ptr, sizeof(_type));			\
940 	ctxt->fetch.ptr += sizeof(_type);				\
949 	ctxt->_eip += (_size);						\
950 	memcpy(_arr, ctxt->fetch.ptr, _size);				\
951 	ctxt->fetch.ptr += (_size);					\
959 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,  in decode_register()  argument
963 	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;  in decode_register()
966 		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;  in decode_register()
968 		p = reg_rmw(ctxt, modrm_reg);  in decode_register()
972 static int read_descriptor(struct x86_emulate_ctxt *ctxt,  in read_descriptor()  argument
981 	rc = segmented_read_std(ctxt, addr, size, 2);  in read_descriptor()
985 	rc = segmented_read_std(ctxt, addr, address, op_bytes);  in read_descriptor()
1033 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)  in em_bsf_c()  argument
1036 	if (ctxt->src.val == 0)  in em_bsf_c()
1037 		ctxt->dst.type = OP_NONE;  in em_bsf_c()
1038 	return fastop(ctxt, em_bsf);  in em_bsf_c()
1041 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)  in em_bsr_c()  argument
1044 	if (ctxt->src.val == 0)  in em_bsr_c()
1045 		ctxt->dst.type = OP_NONE;  in em_bsr_c()
1046 	return fastop(ctxt, em_bsr);  in em_bsr_c()
1078 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)  in read_sse_reg()  argument
1103 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,  in write_sse_reg()  argument
1129 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)  in read_mmx_reg()  argument
1144 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)  in write_mmx_reg()  argument
1159 static int em_fninit(struct x86_emulate_ctxt *ctxt)  in em_fninit()  argument
1161 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fninit()
1162 		return emulate_nm(ctxt);  in em_fninit()
1168 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)  in em_fnstcw()  argument
1172 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fnstcw()
1173 		return emulate_nm(ctxt);  in em_fnstcw()
1177 	ctxt->dst.val = fcw;  in em_fnstcw()
1182 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)  in em_fnstsw()  argument
1186 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fnstsw()
1187 		return emulate_nm(ctxt);  in em_fnstsw()
1191 	ctxt->dst.val = fsw;  in em_fnstsw()
1196 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,  in decode_register_operand()  argument
1199 	unsigned reg = ctxt->modrm_reg;  in decode_register_operand()
1201 	if (!(ctxt->d & ModRM))  in decode_register_operand()
1202 		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);  in decode_register_operand()
1204 	if (ctxt->d & Sse) {  in decode_register_operand()
1208 		read_sse_reg(ctxt, &op->vec_val, reg);  in decode_register_operand()
1211 	if (ctxt->d & Mmx) {  in decode_register_operand()
1220 	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_register_operand()
1221 	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);  in decode_register_operand()
1227 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)  in adjust_modrm_seg()  argument
1230 		ctxt->modrm_seg = VCPU_SREG_SS;  in adjust_modrm_seg()
1233 static int decode_modrm(struct x86_emulate_ctxt *ctxt,  in decode_modrm()  argument
1241 	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */  in decode_modrm()
1242 	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */  in decode_modrm()
1243 	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */  in decode_modrm()
1245 	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;  in decode_modrm()
1246 	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;  in decode_modrm()
1247 	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);  in decode_modrm()
1248 	ctxt->modrm_seg = VCPU_SREG_DS;  in decode_modrm()
1250 	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {  in decode_modrm()
1252 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_modrm()
1253 		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,  in decode_modrm()
1254 				ctxt->d & ByteOp);  in decode_modrm()
1255 		if (ctxt->d & Sse) {  in decode_modrm()
1258 			op->addr.xmm = ctxt->modrm_rm;  in decode_modrm()
1259 			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);  in decode_modrm()
1262 		if (ctxt->d & Mmx) {  in decode_modrm()
1265 			op->addr.mm = ctxt->modrm_rm & 7;  in decode_modrm()
1274 	if (ctxt->ad_bytes == 2) {  in decode_modrm()
1275 		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);  in decode_modrm()
1276 		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);  in decode_modrm()
1277 		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);  in decode_modrm()
1278 		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);  in decode_modrm()
1281 		switch (ctxt->modrm_mod) {  in decode_modrm()
1283 			if (ctxt->modrm_rm == 6)  in decode_modrm()
1284 				modrm_ea += insn_fetch(u16, ctxt);  in decode_modrm()
1287 			modrm_ea += insn_fetch(s8, ctxt);  in decode_modrm()
1290 			modrm_ea += insn_fetch(u16, ctxt);  in decode_modrm()
1293 		switch (ctxt->modrm_rm) {  in decode_modrm()
1313 			if (ctxt->modrm_mod != 0)  in decode_modrm()
1320 		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||  in decode_modrm()
1321 		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))  in decode_modrm()
1322 			ctxt->modrm_seg = VCPU_SREG_SS;  in decode_modrm()
1326 		if ((ctxt->modrm_rm & 7) == 4) {  in decode_modrm()
1327 			sib = insn_fetch(u8, ctxt);  in decode_modrm()
1332 			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)  in decode_modrm()
1333 				modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1335 				modrm_ea += reg_read(ctxt, base_reg);  in decode_modrm()
1336 				adjust_modrm_seg(ctxt, base_reg);  in decode_modrm()
1338 				if ((ctxt->d & IncSP) &&  in decode_modrm()
1340 					modrm_ea += ctxt->op_bytes;  in decode_modrm()
1343 				modrm_ea += reg_read(ctxt, index_reg) << scale;  in decode_modrm()
1344 		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {  in decode_modrm()
1345 			modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1346 			if (ctxt->mode == X86EMUL_MODE_PROT64)  in decode_modrm()
1347 				ctxt->rip_relative = 1;  in decode_modrm()
1349 			base_reg = ctxt->modrm_rm;  in decode_modrm()
1350 			modrm_ea += reg_read(ctxt, base_reg);  in decode_modrm()
1351 			adjust_modrm_seg(ctxt, base_reg);  in decode_modrm()
1353 		switch (ctxt->modrm_mod) {  in decode_modrm()
1355 			modrm_ea += insn_fetch(s8, ctxt);  in decode_modrm()
1358 			modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1363 	if (ctxt->ad_bytes != 8)  in decode_modrm()
1364 		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;  in decode_modrm()
1370 static int decode_abs(struct x86_emulate_ctxt *ctxt,  in decode_abs()  argument
1376 	switch (ctxt->ad_bytes) {  in decode_abs()
1378 		op->addr.mem.ea = insn_fetch(u16, ctxt);  in decode_abs()
1381 		op->addr.mem.ea = insn_fetch(u32, ctxt);  in decode_abs()
1384 		op->addr.mem.ea = insn_fetch(u64, ctxt);  in decode_abs()
1391 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)  in fetch_bit_operand()  argument
1395 	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {  in fetch_bit_operand()
1396 		mask = ~((long)ctxt->dst.bytes * 8 - 1);  in fetch_bit_operand()
1398 		if (ctxt->src.bytes == 2)  in fetch_bit_operand()
1399 			sv = (s16)ctxt->src.val & (s16)mask;  in fetch_bit_operand()
1400 		else if (ctxt->src.bytes == 4)  in fetch_bit_operand()
1401 			sv = (s32)ctxt->src.val & (s32)mask;  in fetch_bit_operand()
1403 			sv = (s64)ctxt->src.val & (s64)mask;  in fetch_bit_operand()
1405 		ctxt->dst.addr.mem.ea = address_mask(ctxt,  in fetch_bit_operand()
1406 					   ctxt->dst.addr.mem.ea + (sv >> 3));  in fetch_bit_operand()
1410 	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;  in fetch_bit_operand()
1413 static int read_emulated(struct x86_emulate_ctxt *ctxt,  in read_emulated()  argument
1417 	struct read_cache *mc = &ctxt->mem_read;  in read_emulated()
1424 	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,  in read_emulated()
1425 				      &ctxt->exception);  in read_emulated()
1437 static int segmented_read(struct x86_emulate_ctxt *ctxt,  in segmented_read()  argument
1445 	rc = linearize(ctxt, addr, size, false, &linear);  in segmented_read()
1448 	return read_emulated(ctxt, linear, data, size);  in segmented_read()
1451 static int segmented_write(struct x86_emulate_ctxt *ctxt,  in segmented_write()  argument
1459 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_write()
1462 	return ctxt->ops->write_emulated(ctxt, linear, data, size,  in segmented_write()
1463 					 &ctxt->exception);  in segmented_write()
1466 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,  in segmented_cmpxchg()  argument
1474 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_cmpxchg()
1477 	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,  in segmented_cmpxchg()
1478 					   size, &ctxt->exception);  in segmented_cmpxchg()
1481 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,  in pio_in_emulated()  argument
1485 	struct read_cache *rc = &ctxt->io_read;  in pio_in_emulated()
1489 		unsigned int count = ctxt->rep_prefix ?  in pio_in_emulated()
1490 			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;  in pio_in_emulated()
1491 		in_page = (ctxt->eflags & X86_EFLAGS_DF) ?  in pio_in_emulated()
1492 			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :  in pio_in_emulated()
1493 			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));  in pio_in_emulated()
1498 		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))  in pio_in_emulated()
1503 	if (ctxt->rep_prefix && (ctxt->d & String) &&  in pio_in_emulated()
1504 	    !(ctxt->eflags & X86_EFLAGS_DF)) {  in pio_in_emulated()
1505 		ctxt->dst.data = rc->data + rc->pos;  in pio_in_emulated()
1506 		ctxt->dst.type = OP_MEM_STR;  in pio_in_emulated()
1507 		ctxt->dst.count = (rc->end - rc->pos) / size;  in pio_in_emulated()
1516 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,  in read_interrupt_descriptor()  argument
1522 	ctxt->ops->get_idt(ctxt, &dt);  in read_interrupt_descriptor()
1525 		return emulate_gp(ctxt, index << 3 | 0x2);  in read_interrupt_descriptor()
1528 	return linear_read_system(ctxt, addr, desc, sizeof(*desc));  in read_interrupt_descriptor()
1531 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,  in get_descriptor_table_ptr()  argument
1534 	const struct x86_emulate_ops *ops = ctxt->ops;  in get_descriptor_table_ptr()
1542 		if (!ops->get_segment(ctxt, &sel, &desc, &base3,  in get_descriptor_table_ptr()
1549 		ops->get_gdt(ctxt, dt);  in get_descriptor_table_ptr()
1552 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,  in get_descriptor_ptr()  argument
1559 	get_descriptor_table_ptr(ctxt, selector, &dt);  in get_descriptor_ptr()
1562 		return emulate_gp(ctxt, selector & 0xfffc);  in get_descriptor_ptr()
1570 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in get_descriptor_ptr()
1581 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in read_segment_descriptor()  argument
1587 	rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);  in read_segment_descriptor()
1591 	return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));  in read_segment_descriptor()
1595 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in write_segment_descriptor()  argument
1601 	rc = get_descriptor_ptr(ctxt, selector, &addr);  in write_segment_descriptor()
1605 	return linear_write_system(ctxt, addr, desc, sizeof(*desc));  in write_segment_descriptor()
1608 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in __load_segment_descriptor()  argument
1625 	if (ctxt->mode == X86EMUL_MODE_REAL) {  in __load_segment_descriptor()
1628 		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);  in __load_segment_descriptor()
1631 	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {  in __load_segment_descriptor()
1654 			if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)  in __load_segment_descriptor()
1673 	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);  in __load_segment_descriptor()
1721 			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in __load_segment_descriptor()
1734 		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,  in __load_segment_descriptor()
1735 						  sizeof(seg_desc), &ctxt->exception);  in __load_segment_descriptor()
1760 			ret = write_segment_descriptor(ctxt, selector,  in __load_segment_descriptor()
1765 	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {  in __load_segment_descriptor()
1766 		ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));  in __load_segment_descriptor()
1770 				((u64)base3 << 32), ctxt))  in __load_segment_descriptor()
1771 			return emulate_gp(ctxt, 0);  in __load_segment_descriptor()
1774 	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);  in __load_segment_descriptor()
1779 	return emulate_exception(ctxt, err_vec, err_code, true);  in __load_segment_descriptor()
1782 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in load_segment_descriptor()  argument
1785 	u8 cpl = ctxt->ops->cpl(ctxt);  in load_segment_descriptor()
1798 	    ctxt->mode == X86EMUL_MODE_PROT64)  in load_segment_descriptor()
1799 		return emulate_exception(ctxt, GP_VECTOR, 0, true);  in load_segment_descriptor()
1801 	return __load_segment_descriptor(ctxt, selector, seg, cpl,  in load_segment_descriptor()
1810 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)  in writeback()  argument
1817 		if (ctxt->lock_prefix)  in writeback()
1818 			return segmented_cmpxchg(ctxt,  in writeback()
1824 			return segmented_write(ctxt,  in writeback()
1830 		return segmented_write(ctxt,  in writeback()
1836 		write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);  in writeback()
1839 		write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);  in writeback()
1850 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)  in push()  argument
1854 	rsp_increment(ctxt, -bytes);  in push()
1855 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);  in push()
1858 	return segmented_write(ctxt, addr, data, bytes);  in push()
1861 static int em_push(struct x86_emulate_ctxt *ctxt)  in em_push()  argument
1864 	ctxt->dst.type = OP_NONE;  in em_push()
1865 	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);  in em_push()
1868 static int emulate_pop(struct x86_emulate_ctxt *ctxt,  in emulate_pop()  argument
1874 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);  in emulate_pop()
1876 	rc = segmented_read(ctxt, addr, dest, len);  in emulate_pop()
1880 	rsp_increment(ctxt, len);  in emulate_pop()
1884 static int em_pop(struct x86_emulate_ctxt *ctxt)  in em_pop()  argument
1886 	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);  in em_pop()
1889 static int emulate_popf(struct x86_emulate_ctxt *ctxt,  in emulate_popf()  argument
1894 	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;  in emulate_popf()
1895 	int cpl = ctxt->ops->cpl(ctxt);  in emulate_popf()
1897 	rc = emulate_pop(ctxt, &val, len);  in emulate_popf()
1906 	switch(ctxt->mode) {  in emulate_popf()
1917 			return emulate_gp(ctxt, 0);  in emulate_popf()
1926 		(ctxt->eflags & ~change_mask) | (val & change_mask);  in emulate_popf()
1931 static int em_popf(struct x86_emulate_ctxt *ctxt)  in em_popf()  argument
1933 	ctxt->dst.type = OP_REG;  in em_popf()
1934 	ctxt->dst.addr.reg = &ctxt->eflags;  in em_popf()
1935 	ctxt->dst.bytes = ctxt->op_bytes;  in em_popf()
1936 	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);  in em_popf()
1939 static int em_enter(struct x86_emulate_ctxt *ctxt)  in em_enter()  argument
1942 	unsigned frame_size = ctxt->src.val;  in em_enter()
1943 	unsigned nesting_level = ctxt->src2.val & 31;  in em_enter()
1949 	rbp = reg_read(ctxt, VCPU_REGS_RBP);  in em_enter()
1950 	rc = push(ctxt, &rbp, stack_size(ctxt));  in em_enter()
1953 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),  in em_enter()
1954 		      stack_mask(ctxt));  in em_enter()
1955 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),  in em_enter()
1956 		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,  in em_enter()
1957 		      stack_mask(ctxt));  in em_enter()
1961 static int em_leave(struct x86_emulate_ctxt *ctxt)  in em_leave()  argument
1963 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),  in em_leave()
1964 		      stack_mask(ctxt));  in em_leave()
1965 	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);  in em_leave()
1968 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)  in em_push_sreg()  argument
1970 	int seg = ctxt->src2.val;  in em_push_sreg()
1972 	ctxt->src.val = get_segment_selector(ctxt, seg);  in em_push_sreg()
1973 	if (ctxt->op_bytes == 4) {  in em_push_sreg()
1974 		rsp_increment(ctxt, -2);  in em_push_sreg()
1975 		ctxt->op_bytes = 2;  in em_push_sreg()
1978 	return em_push(ctxt);  in em_push_sreg()
1981 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)  in em_pop_sreg()  argument
1983 	int seg = ctxt->src2.val;  in em_pop_sreg()
1987 	rc = emulate_pop(ctxt, &selector, 2);  in em_pop_sreg()
1991 	if (ctxt->modrm_reg == VCPU_SREG_SS)  in em_pop_sreg()
1992 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;  in em_pop_sreg()
1993 	if (ctxt->op_bytes > 2)  in em_pop_sreg()
1994 		rsp_increment(ctxt, ctxt->op_bytes - 2);  in em_pop_sreg()
1996 	rc = load_segment_descriptor(ctxt, (u16)selector, seg);  in em_pop_sreg()
2000 static int em_pusha(struct x86_emulate_ctxt *ctxt)  in em_pusha()  argument
2002 	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);  in em_pusha()
2008 		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));  in em_pusha()
2010 		rc = em_push(ctxt);  in em_pusha()
2020 static int em_pushf(struct x86_emulate_ctxt *ctxt)  in em_pushf()  argument
2022 	ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;  in em_pushf()
2023 	return em_push(ctxt);  in em_pushf()
2026 static int em_popa(struct x86_emulate_ctxt *ctxt)  in em_popa()  argument
2034 			rsp_increment(ctxt, ctxt->op_bytes);  in em_popa()
2038 		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);  in em_popa()
2041 		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);  in em_popa()
2047 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)  in __emulate_int_real()  argument
2049 	const struct x86_emulate_ops *ops = ctxt->ops;  in __emulate_int_real()
2057 	ctxt->src.val = ctxt->eflags;  in __emulate_int_real()
2058 	rc = em_push(ctxt);  in __emulate_int_real()
2062 	ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);  in __emulate_int_real()
2064 	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);  in __emulate_int_real()
2065 	rc = em_push(ctxt);  in __emulate_int_real()
2069 	ctxt->src.val = ctxt->_eip;  in __emulate_int_real()
2070 	rc = em_push(ctxt);  in __emulate_int_real()
2074 	ops->get_idt(ctxt, &dt);  in __emulate_int_real()
2079 	rc = linear_read_system(ctxt, cs_addr, &cs, 2);  in __emulate_int_real()
2083 	rc = linear_read_system(ctxt, eip_addr, &eip, 2);  in __emulate_int_real()
2087 	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);  in __emulate_int_real()
2091 	ctxt->_eip = eip;  in __emulate_int_real()
2096 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)  in emulate_int_real()  argument
2100 	invalidate_registers(ctxt);  in emulate_int_real()
2101 	rc = __emulate_int_real(ctxt, irq);  in emulate_int_real()
2103 		writeback_registers(ctxt);  in emulate_int_real()
2107 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)  in emulate_int()  argument
2109 	switch(ctxt->mode) {  in emulate_int()
2111 		return __emulate_int_real(ctxt, irq);  in emulate_int()
2122 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)  in emulate_iret_real()  argument
2139 	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);  in emulate_iret_real()
2145 		return emulate_gp(ctxt, 0);  in emulate_iret_real()
2147 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);  in emulate_iret_real()
2152 	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);  in emulate_iret_real()
2157 	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);  in emulate_iret_real()
2162 	ctxt->_eip = temp_eip;  in emulate_iret_real()
2164 	if (ctxt->op_bytes == 4)  in emulate_iret_real()
2165 		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));  in emulate_iret_real()
2166 	else if (ctxt->op_bytes == 2) {  in emulate_iret_real()
2167 		ctxt->eflags &= ~0xffff;  in emulate_iret_real()
2168 		ctxt->eflags |= temp_eflags;  in emulate_iret_real()
2171 	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */  in emulate_iret_real()
2172 	ctxt->eflags |= X86_EFLAGS_FIXED;  in emulate_iret_real()
2173 	ctxt->ops->set_nmi_mask(ctxt, false);  in emulate_iret_real()
2178 static int em_iret(struct x86_emulate_ctxt *ctxt)  in em_iret()  argument
2180 	switch(ctxt->mode) {  in em_iret()
2182 		return emulate_iret_real(ctxt);  in em_iret()
2193 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)  in em_jmp_far()  argument
2198 	u8 cpl = ctxt->ops->cpl(ctxt);  in em_jmp_far()
2200 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_jmp_far()
2202 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,  in em_jmp_far()
2208 	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);  in em_jmp_far()
2216 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)  in em_jmp_abs()  argument
2218 	return assign_eip_near(ctxt, ctxt->src.val);  in em_jmp_abs()
2221 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)  in em_call_near_abs()  argument
2226 	old_eip = ctxt->_eip;  in em_call_near_abs()
2227 	rc = assign_eip_near(ctxt, ctxt->src.val);  in em_call_near_abs()
2230 	ctxt->src.val = old_eip;  in em_call_near_abs()
2231 	rc = em_push(ctxt);  in em_call_near_abs()
2235 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)  in em_cmpxchg8b()  argument
2237 	u64 old = ctxt->dst.orig_val64;  in em_cmpxchg8b()
2239 	if (ctxt->dst.bytes == 16)  in em_cmpxchg8b()
2242 	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||  in em_cmpxchg8b()
2243 	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {  in em_cmpxchg8b()
2244 		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);  in em_cmpxchg8b()
2245 		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);  in em_cmpxchg8b()
2246 		ctxt->eflags &= ~X86_EFLAGS_ZF;  in em_cmpxchg8b()
2248 		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |  in em_cmpxchg8b()
2249 			(u32) reg_read(ctxt, VCPU_REGS_RBX);  in em_cmpxchg8b()
2251 		ctxt->eflags |= X86_EFLAGS_ZF;  in em_cmpxchg8b()
2256 static int em_ret(struct x86_emulate_ctxt *ctxt)  in em_ret()  argument
2261 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret()
2265 	return assign_eip_near(ctxt, eip);  in em_ret()
2268 static int em_ret_far(struct x86_emulate_ctxt *ctxt)  in em_ret_far()  argument
2272 	int cpl = ctxt->ops->cpl(ctxt);  in em_ret_far()
2275 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret_far()
2278 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);  in em_ret_far()
2282 	if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)  in em_ret_far()
2284 	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,  in em_ret_far()
2289 	rc = assign_eip_far(ctxt, eip, &new_desc);  in em_ret_far()
2297 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)  in em_ret_far_imm()  argument
2301         rc = em_ret_far(ctxt);  in em_ret_far_imm()
2304         rsp_increment(ctxt, ctxt->src.val);  in em_ret_far_imm()
2308 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)  in em_cmpxchg()  argument
2311 	ctxt->dst.orig_val = ctxt->dst.val;  in em_cmpxchg()
2312 	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);  in em_cmpxchg()
2313 	ctxt->src.orig_val = ctxt->src.val;  in em_cmpxchg()
2314 	ctxt->src.val = ctxt->dst.orig_val;  in em_cmpxchg()
2315 	fastop(ctxt, em_cmp);  in em_cmpxchg()
2317 	if (ctxt->eflags & X86_EFLAGS_ZF) {  in em_cmpxchg()
2319 		ctxt->src.type = OP_NONE;  in em_cmpxchg()
2320 		ctxt->dst.val = ctxt->src.orig_val;  in em_cmpxchg()
2323 		ctxt->src.type = OP_REG;  in em_cmpxchg()
2324 		ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in em_cmpxchg()
2325 		ctxt->src.val = ctxt->dst.orig_val;  in em_cmpxchg()
2327 		ctxt->dst.val = ctxt->dst.orig_val;  in em_cmpxchg()
2332 static int em_lseg(struct x86_emulate_ctxt *ctxt)  in em_lseg()  argument
2334 	int seg = ctxt->src2.val;  in em_lseg()
2338 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_lseg()
2340 	rc = load_segment_descriptor(ctxt, sel, seg);  in em_lseg()
2344 	ctxt->dst.val = ctxt->src.val;  in em_lseg()
2348 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)  in emulator_has_longmode()  argument
2355 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);  in emulator_has_longmode()
2374 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,  in rsm_load_seg_32()  argument
2391 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);  in rsm_load_seg_32()
2396 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,  in rsm_load_seg_64()  argument
2412 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);  in rsm_load_seg_64()
2417 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,  in rsm_enter_protected_mode()  argument
2430 	bad = ctxt->ops->set_cr(ctxt, 3, cr3);  in rsm_enter_protected_mode()
2439 	bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);  in rsm_enter_protected_mode()
2443 	bad = ctxt->ops->set_cr(ctxt, 0, cr0);  in rsm_enter_protected_mode()
2448 		bad = ctxt->ops->set_cr(ctxt, 4, cr4);  in rsm_enter_protected_mode()
2452 			bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);  in rsm_enter_protected_mode()
2462 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,  in rsm_load_state_32()  argument
2473 	ctxt->eflags =             GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;  in rsm_load_state_32()
2474 	ctxt->_eip =               GET_SMSTATE(u32, smstate, 0x7ff0);  in rsm_load_state_32()
2477 		*reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);  in rsm_load_state_32()
2480 	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);  in rsm_load_state_32()
2482 	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);  in rsm_load_state_32()
2488 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);  in rsm_load_state_32()
2494 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);  in rsm_load_state_32()
2498 	ctxt->ops->set_gdt(ctxt, &dt);  in rsm_load_state_32()
2502 	ctxt->ops->set_idt(ctxt, &dt);  in rsm_load_state_32()
2505 		int r = rsm_load_seg_32(ctxt, smstate, i);  in rsm_load_state_32()
2512 	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));  in rsm_load_state_32()
2514 	return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);  in rsm_load_state_32()
2518 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,  in rsm_load_state_64()  argument
2529 		*reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);  in rsm_load_state_64()
2531 	ctxt->_eip   = GET_SMSTATE(u64, smstate, 0x7f78);  in rsm_load_state_64()
2532 	ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;  in rsm_load_state_64()
2535 	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);  in rsm_load_state_64()
2537 	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);  in rsm_load_state_64()
2542 	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));  in rsm_load_state_64()
2544 	ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);  in rsm_load_state_64()
2551 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);  in rsm_load_state_64()
2555 	ctxt->ops->set_idt(ctxt, &dt);  in rsm_load_state_64()
2562 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);  in rsm_load_state_64()
2566 	ctxt->ops->set_gdt(ctxt, &dt);  in rsm_load_state_64()
2568 	r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);  in rsm_load_state_64()
2573 		r = rsm_load_seg_64(ctxt, smstate, i);  in rsm_load_state_64()
2582 static int em_rsm(struct x86_emulate_ctxt *ctxt)  in em_rsm()  argument
2589 	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)  in em_rsm()
2590 		return emulate_ud(ctxt);  in em_rsm()
2592 	smbase = ctxt->ops->get_smbase(ctxt);  in em_rsm()
2594 	ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));  in em_rsm()
2598 	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)  in em_rsm()
2599 		ctxt->ops->set_nmi_mask(ctxt, false);  in em_rsm()
2601 	ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &  in em_rsm()
2609 	if (emulator_has_longmode(ctxt)) {  in em_rsm()
2613 		cr4 = ctxt->ops->get_cr(ctxt, 4);  in em_rsm()
2615 			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);  in em_rsm()
2621 		ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);  in em_rsm()
2625 	cr0 = ctxt->ops->get_cr(ctxt, 0);  in em_rsm()
2627 		ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));  in em_rsm()
2629 	if (emulator_has_longmode(ctxt)) {  in em_rsm()
2631 		cr4 = ctxt->ops->get_cr(ctxt, 4);  in em_rsm()
2633 			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);  in em_rsm()
2637 		ctxt->ops->set_msr(ctxt, MSR_EFER, efer);  in em_rsm()
2645 	if (ctxt->ops->pre_leave_smm(ctxt, buf))  in em_rsm()
2649 	if (emulator_has_longmode(ctxt))  in em_rsm()
2650 		ret = rsm_load_state_64(ctxt, buf);  in em_rsm()
2653 		ret = rsm_load_state_32(ctxt, buf);  in em_rsm()
2660 	ctxt->ops->post_leave_smm(ctxt);  in em_rsm()
2666 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,  in setup_syscalls_segments()  argument
2692 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)  in vendor_intel()  argument
2697 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);  in vendor_intel()
2703 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)  in em_syscall_is_enabled()  argument
2705 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_syscall_is_enabled()
2712 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_syscall_is_enabled()
2717 	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);  in em_syscall_is_enabled()
2756 static int em_syscall(struct x86_emulate_ctxt *ctxt)  in em_syscall()  argument
2758 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_syscall()
2765 	if (ctxt->mode == X86EMUL_MODE_REAL ||  in em_syscall()
2766 	    ctxt->mode == X86EMUL_MODE_VM86)  in em_syscall()
2767 		return emulate_ud(ctxt);  in em_syscall()
2769 	if (!(em_syscall_is_enabled(ctxt)))  in em_syscall()
2770 		return emulate_ud(ctxt);  in em_syscall()
2772 	ops->get_msr(ctxt, MSR_EFER, &efer);  in em_syscall()
2773 	setup_syscalls_segments(ctxt, &cs, &ss);  in em_syscall()
2776 		return emulate_ud(ctxt);  in em_syscall()
2778 	ops->get_msr(ctxt, MSR_STAR, &msr_data);  in em_syscall()
2787 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_syscall()
2788 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_syscall()
2790 	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;  in em_syscall()
2793 		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;  in em_syscall()
2795 		ops->get_msr(ctxt,  in em_syscall()
2796 			     ctxt->mode == X86EMUL_MODE_PROT64 ?  in em_syscall()
2798 		ctxt->_eip = msr_data;  in em_syscall()
2800 		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);  in em_syscall()
2801 		ctxt->eflags &= ~msr_data;  in em_syscall()
2802 		ctxt->eflags |= X86_EFLAGS_FIXED;  in em_syscall()
2806 		ops->get_msr(ctxt, MSR_STAR, &msr_data);  in em_syscall()
2807 		ctxt->_eip = (u32)msr_data;  in em_syscall()
2809 		ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);  in em_syscall()
2812 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;  in em_syscall()
2816 static int em_sysenter(struct x86_emulate_ctxt *ctxt)  in em_sysenter()  argument
2818 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_sysenter()
2824 	ops->get_msr(ctxt, MSR_EFER, &efer);  in em_sysenter()
2826 	if (ctxt->mode == X86EMUL_MODE_REAL)  in em_sysenter()
2827 		return emulate_gp(ctxt, 0);  in em_sysenter()
2833 	if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)  in em_sysenter()
2834 	    && !vendor_intel(ctxt))  in em_sysenter()
2835 		return emulate_ud(ctxt);  in em_sysenter()
2838 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_sysenter()
2841 	setup_syscalls_segments(ctxt, &cs, &ss);  in em_sysenter()
2843 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);  in em_sysenter()
2845 		return emulate_gp(ctxt, 0);  in em_sysenter()
2847 	ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);  in em_sysenter()
2855 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_sysenter()
2856 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_sysenter()
2858 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);  in em_sysenter()
2859 	ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;  in em_sysenter()
2861 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);  in em_sysenter()
2862 	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :  in em_sysenter()
2868 static int em_sysexit(struct x86_emulate_ctxt *ctxt)  in em_sysexit()  argument
2870 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_sysexit()
2877 	if (ctxt->mode == X86EMUL_MODE_REAL ||  in em_sysexit()
2878 	    ctxt->mode == X86EMUL_MODE_VM86)  in em_sysexit()
2879 		return emulate_gp(ctxt, 0);  in em_sysexit()
2881 	setup_syscalls_segments(ctxt, &cs, &ss);  in em_sysexit()
2883 	if ((ctxt->rex_prefix & 0x8) != 0x0)  in em_sysexit()
2888 	rcx = reg_read(ctxt, VCPU_REGS_RCX);  in em_sysexit()
2889 	rdx = reg_read(ctxt, VCPU_REGS_RDX);  in em_sysexit()
2893 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);  in em_sysexit()
2898 			return emulate_gp(ctxt, 0);  in em_sysexit()
2906 			return emulate_gp(ctxt, 0);  in em_sysexit()
2910 		if (emul_is_noncanonical_address(rcx, ctxt) ||  in em_sysexit()
2911 		    emul_is_noncanonical_address(rdx, ctxt))  in em_sysexit()
2912 			return emulate_gp(ctxt, 0);  in em_sysexit()
2918 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_sysexit()
2919 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_sysexit()
2921 	ctxt->_eip = rdx;  in em_sysexit()
2922 	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;  in em_sysexit()
2927 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)  in emulator_bad_iopl()  argument
2930 	if (ctxt->mode == X86EMUL_MODE_REAL)  in emulator_bad_iopl()
2932 	if (ctxt->mode == X86EMUL_MODE_VM86)  in emulator_bad_iopl()
2934 	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;  in emulator_bad_iopl()
2935 	return ctxt->ops->cpl(ctxt) > iopl;  in emulator_bad_iopl()
2941 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,  in emulator_io_port_access_allowed()  argument
2944 	const struct x86_emulate_ops *ops = ctxt->ops;  in emulator_io_port_access_allowed()
2960 	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);  in emulator_io_port_access_allowed()
2969 	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);  in emulator_io_port_access_allowed()
2974 	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);  in emulator_io_port_access_allowed()
2982 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,  in emulator_io_permited()  argument
2985 	if (ctxt->perm_ok)  in emulator_io_permited()
2988 	if (emulator_bad_iopl(ctxt))  in emulator_io_permited()
2989 		if (!emulator_io_port_access_allowed(ctxt, port, len))  in emulator_io_permited()
2992 	ctxt->perm_ok = true;  in emulator_io_permited()
2997 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)  in string_registers_quirk()  argument
3004 	if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))  in string_registers_quirk()
3007 	*reg_write(ctxt, VCPU_REGS_RCX) = 0;  in string_registers_quirk()
3009 	switch (ctxt->b) {  in string_registers_quirk()
3012 		*reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;  in string_registers_quirk()
3016 		*reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;  in string_registers_quirk()
3021 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,  in save_state_to_tss16()  argument
3024 	tss->ip = ctxt->_eip;  in save_state_to_tss16()
3025 	tss->flag = ctxt->eflags;  in save_state_to_tss16()
3026 	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);  in save_state_to_tss16()
3027 	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);  in save_state_to_tss16()
3028 	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);  in save_state_to_tss16()
3029 	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);  in save_state_to_tss16()
3030 	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);  in save_state_to_tss16()
3031 	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);  in save_state_to_tss16()
3032 	tss->si = reg_read(ctxt, VCPU_REGS_RSI);  in save_state_to_tss16()
3033 	tss->di = reg_read(ctxt, VCPU_REGS_RDI);  in save_state_to_tss16()
3035 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);  in save_state_to_tss16()
3036 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);  in save_state_to_tss16()
3037 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);  in save_state_to_tss16()
3038 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);  in save_state_to_tss16()
3039 	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);  in save_state_to_tss16()
3042 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,  in load_state_from_tss16()  argument
3048 	ctxt->_eip = tss->ip;  in load_state_from_tss16()
3049 	ctxt->eflags = tss->flag | 2;  in load_state_from_tss16()
3050 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;  in load_state_from_tss16()
3051 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;  in load_state_from_tss16()
3052 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;  in load_state_from_tss16()
3053 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;  in load_state_from_tss16()
3054 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;  in load_state_from_tss16()
3055 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;  in load_state_from_tss16()
3056 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;  in load_state_from_tss16()
3057 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;  in load_state_from_tss16()
3063 	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);  in load_state_from_tss16()
3064 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);  in load_state_from_tss16()
3065 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);  in load_state_from_tss16()
3066 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);  in load_state_from_tss16()
3067 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);  in load_state_from_tss16()
3075 	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,  in load_state_from_tss16()
3079 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,  in load_state_from_tss16()
3083 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,  in load_state_from_tss16()
3087 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,  in load_state_from_tss16()
3091 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,  in load_state_from_tss16()
3099 static int task_switch_16(struct x86_emulate_ctxt *ctxt,  in task_switch_16()  argument
3107 	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
3111 	save_state_to_tss16(ctxt, &tss_seg);  in task_switch_16()
3113 	ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
3117 	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
3124 		ret = linear_write_system(ctxt, new_tss_base,  in task_switch_16()
3131 	return load_state_from_tss16(ctxt, &tss_seg);  in task_switch_16()
3134 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,  in save_state_to_tss32()  argument
3138 	tss->eip = ctxt->_eip;  in save_state_to_tss32()
3139 	tss->eflags = ctxt->eflags;  in save_state_to_tss32()
3140 	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);  in save_state_to_tss32()
3141 	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);  in save_state_to_tss32()
3142 	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);  in save_state_to_tss32()
3143 	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);  in save_state_to_tss32()
3144 	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);  in save_state_to_tss32()
3145 	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);  in save_state_to_tss32()
3146 	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);  in save_state_to_tss32()
3147 	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);  in save_state_to_tss32()
3149 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);  in save_state_to_tss32()
3150 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);  in save_state_to_tss32()
3151 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);  in save_state_to_tss32()
3152 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);  in save_state_to_tss32()
3153 	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);  in save_state_to_tss32()
3154 	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);  in save_state_to_tss32()
3157 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,  in load_state_from_tss32()  argument
3163 	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))  in load_state_from_tss32()
3164 		return emulate_gp(ctxt, 0);  in load_state_from_tss32()
3165 	ctxt->_eip = tss->eip;  in load_state_from_tss32()
3166 	ctxt->eflags = tss->eflags | 2;  in load_state_from_tss32()
3169 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;  in load_state_from_tss32()
3170 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;  in load_state_from_tss32()
3171 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;  in load_state_from_tss32()
3172 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;  in load_state_from_tss32()
3173 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;  in load_state_from_tss32()
3174 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;  in load_state_from_tss32()
3175 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;  in load_state_from_tss32()
3176 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;  in load_state_from_tss32()
3183 	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);  in load_state_from_tss32()
3184 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);  in load_state_from_tss32()
3185 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);  in load_state_from_tss32()
3186 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);  in load_state_from_tss32()
3187 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);  in load_state_from_tss32()
3188 	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);  in load_state_from_tss32()
3189 	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);  in load_state_from_tss32()
3196 	if (ctxt->eflags & X86_EFLAGS_VM) {  in load_state_from_tss32()
3197 		ctxt->mode = X86EMUL_MODE_VM86;  in load_state_from_tss32()
3200 		ctxt->mode = X86EMUL_MODE_PROT32;  in load_state_from_tss32()
3208 	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,  in load_state_from_tss32()
3212 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,  in load_state_from_tss32()
3216 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,  in load_state_from_tss32()
3220 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,  in load_state_from_tss32()
3224 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,  in load_state_from_tss32()
3228 	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,  in load_state_from_tss32()
3232 	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,  in load_state_from_tss32()
3238 static int task_switch_32(struct x86_emulate_ctxt *ctxt,  in task_switch_32()  argument
3248 	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_32()
3252 	save_state_to_tss32(ctxt, &tss_seg);  in task_switch_32()
3255 	ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,  in task_switch_32()
3260 	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_32()
3267 		ret = linear_write_system(ctxt, new_tss_base,  in task_switch_32()
3274 	return load_state_from_tss32(ctxt, &tss_seg);  in task_switch_32()
3277 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,  in emulator_do_task_switch()  argument
3281 	const struct x86_emulate_ops *ops = ctxt->ops;  in emulator_do_task_switch()
3284 	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);  in emulator_do_task_switch()
3286 		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);  in emulator_do_task_switch()
3292 	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);  in emulator_do_task_switch()
3295 	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);  in emulator_do_task_switch()
3315 			ret = read_interrupt_descriptor(ctxt, idt_index,  in emulator_do_task_switch()
3321 			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)  in emulator_do_task_switch()
3322 				return emulate_gp(ctxt, (idt_index << 3) | 0x2);  in emulator_do_task_switch()
3330 		return emulate_ts(ctxt, tss_selector & 0xfffc);  in emulator_do_task_switch()
3335 		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);  in emulator_do_task_switch()
3339 		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;  in emulator_do_task_switch()
3347 		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,  in emulator_do_task_switch()
3350 		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,  in emulator_do_task_switch()
3356 		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;  in emulator_do_task_switch()
3360 		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);  in emulator_do_task_switch()
3363 	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);  in emulator_do_task_switch()
3364 	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);  in emulator_do_task_switch()
3367 		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;  in emulator_do_task_switch()
3368 		ctxt->lock_prefix = 0;  in emulator_do_task_switch()
3369 		ctxt->src.val = (unsigned long) error_code;  in emulator_do_task_switch()
3370 		ret = em_push(ctxt);  in emulator_do_task_switch()
3373 	ops->get_dr(ctxt, 7, &dr7);  in emulator_do_task_switch()
3374 	ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));  in emulator_do_task_switch()
3379 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,  in emulator_task_switch()  argument
3385 	invalidate_registers(ctxt);  in emulator_task_switch()
3386 	ctxt->_eip = ctxt->eip;  in emulator_task_switch()
3387 	ctxt->dst.type = OP_NONE;  in emulator_task_switch()
3389 	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,  in emulator_task_switch()
3393 		ctxt->eip = ctxt->_eip;  in emulator_task_switch()
3394 		writeback_registers(ctxt);  in emulator_task_switch()
3400 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,  in string_addr_inc()  argument
3403 	int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;  in string_addr_inc()
3405 	register_address_increment(ctxt, reg, df * op->bytes);  in string_addr_inc()
3406 	op->addr.mem.ea = register_address(ctxt, reg);  in string_addr_inc()
3409 static int em_das(struct x86_emulate_ctxt *ctxt)  in em_das()  argument
3414 	cf = ctxt->eflags & X86_EFLAGS_CF;  in em_das()
3415 	al = ctxt->dst.val;  in em_das()
3420 	af = ctxt->eflags & X86_EFLAGS_AF;  in em_das()
3433 	ctxt->dst.val = al;  in em_das()
3435 	ctxt->src.type = OP_IMM;  in em_das()
3436 	ctxt->src.val = 0;  in em_das()
3437 	ctxt->src.bytes = 1;  in em_das()
3438 	fastop(ctxt, em_or);  in em_das()
3439 	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);  in em_das()
3441 		ctxt->eflags |= X86_EFLAGS_CF;  in em_das()
3443 		ctxt->eflags |= X86_EFLAGS_AF;  in em_das()
3447 static int em_aam(struct x86_emulate_ctxt *ctxt)  in em_aam()  argument
3451 	if (ctxt->src.val == 0)  in em_aam()
3452 		return emulate_de(ctxt);  in em_aam()
3454 	al = ctxt->dst.val & 0xff;  in em_aam()
3455 	ah = al / ctxt->src.val;  in em_aam()
3456 	al %= ctxt->src.val;  in em_aam()
3458 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);  in em_aam()
3461 	ctxt->src.type = OP_IMM;  in em_aam()
3462 	ctxt->src.val = 0;  in em_aam()
3463 	ctxt->src.bytes = 1;  in em_aam()
3464 	fastop(ctxt, em_or);  in em_aam()
3469 static int em_aad(struct x86_emulate_ctxt *ctxt)  in em_aad()  argument
3471 	u8 al = ctxt->dst.val & 0xff;  in em_aad()
3472 	u8 ah = (ctxt->dst.val >> 8) & 0xff;  in em_aad()
3474 	al = (al + (ah * ctxt->src.val)) & 0xff;  in em_aad()
3476 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;  in em_aad()
3479 	ctxt->src.type = OP_IMM;  in em_aad()
3480 	ctxt->src.val = 0;  in em_aad()
3481 	ctxt->src.bytes = 1;  in em_aad()
3482 	fastop(ctxt, em_or);  in em_aad()
3487 static int em_call(struct x86_emulate_ctxt *ctxt)  in em_call()  argument
3490 	long rel = ctxt->src.val;  in em_call()
3492 	ctxt->src.val = (unsigned long)ctxt->_eip;  in em_call()
3493 	rc = jmp_rel(ctxt, rel);  in em_call()
3496 	return em_push(ctxt);  in em_call()
3499 static int em_call_far(struct x86_emulate_ctxt *ctxt)  in em_call_far()  argument
3505 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_call_far()
3506 	int cpl = ctxt->ops->cpl(ctxt);  in em_call_far()
3507 	enum x86emul_mode prev_mode = ctxt->mode;  in em_call_far()
3509 	old_eip = ctxt->_eip;  in em_call_far()
3510 	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);  in em_call_far()
3512 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_call_far()
3513 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,  in em_call_far()
3518 	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);  in em_call_far()
3522 	ctxt->src.val = old_cs;  in em_call_far()
3523 	rc = em_push(ctxt);  in em_call_far()
3527 	ctxt->src.val = old_eip;  in em_call_far()
3528 	rc = em_push(ctxt);  in em_call_far()
3537 	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);  in em_call_far()
3538 	ctxt->mode = prev_mode;  in em_call_far()
3543 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)  in em_ret_near_imm()  argument
3548 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret_near_imm()
3551 	rc = assign_eip_near(ctxt, eip);  in em_ret_near_imm()
3554 	rsp_increment(ctxt, ctxt->src.val);  in em_ret_near_imm()
3558 static int em_xchg(struct x86_emulate_ctxt *ctxt)  in em_xchg()  argument
3561 	ctxt->src.val = ctxt->dst.val;  in em_xchg()
3562 	write_register_operand(&ctxt->src);  in em_xchg()
3565 	ctxt->dst.val = ctxt->src.orig_val;  in em_xchg()
3566 	ctxt->lock_prefix = 1;  in em_xchg()
3570 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)  in em_imul_3op()  argument
3572 	ctxt->dst.val = ctxt->src2.val;  in em_imul_3op()
3573 	return fastop(ctxt, em_imul);  in em_imul_3op()
3576 static int em_cwd(struct x86_emulate_ctxt *ctxt)  in em_cwd()  argument
3578 	ctxt->dst.type = OP_REG;  in em_cwd()
3579 	ctxt->dst.bytes = ctxt->src.bytes;  in em_cwd()
3580 	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in em_cwd()
3581 	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);  in em_cwd()
3586 static int em_rdpid(struct x86_emulate_ctxt *ctxt)  in em_rdpid()  argument
3590 	if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))  in em_rdpid()
3591 		return emulate_gp(ctxt, 0);  in em_rdpid()
3592 	ctxt->dst.val = tsc_aux;  in em_rdpid()
3596 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)  in em_rdtsc()  argument
3600 	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);  in em_rdtsc()
3601 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;  in em_rdtsc()
3602 	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;  in em_rdtsc()
3606 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)  in em_rdpmc()  argument
3610 	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))  in em_rdpmc()
3611 		return emulate_gp(ctxt, 0);  in em_rdpmc()
3612 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;  in em_rdpmc()
3613 	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;  in em_rdpmc()
3617 static int em_mov(struct x86_emulate_ctxt *ctxt)  in em_mov()  argument
3619 	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));  in em_mov()
3625 static int em_movbe(struct x86_emulate_ctxt *ctxt)  in em_movbe()  argument
3633 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);  in em_movbe()
3635 		return emulate_ud(ctxt);  in em_movbe()
3637 	switch (ctxt->op_bytes) {  in em_movbe()
3647 		tmp = (u16)ctxt->src.val;  in em_movbe()
3648 		ctxt->dst.val &= ~0xffffUL;  in em_movbe()
3649 		ctxt->dst.val |= (unsigned long)swab16(tmp);  in em_movbe()
3652 		ctxt->dst.val = swab32((u32)ctxt->src.val);  in em_movbe()
3655 		ctxt->dst.val = swab64(ctxt->src.val);  in em_movbe()
3663 static int em_cr_write(struct x86_emulate_ctxt *ctxt)  in em_cr_write()  argument
3665 	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))  in em_cr_write()
3666 		return emulate_gp(ctxt, 0);  in em_cr_write()
3669 	ctxt->dst.type = OP_NONE;  in em_cr_write()
3673 static int em_dr_write(struct x86_emulate_ctxt *ctxt)  in em_dr_write()  argument
3677 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_dr_write()
3678 		val = ctxt->src.val & ~0ULL;  in em_dr_write()
3680 		val = ctxt->src.val & ~0U;  in em_dr_write()
3683 	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)  in em_dr_write()
3684 		return emulate_gp(ctxt, 0);  in em_dr_write()
3687 	ctxt->dst.type = OP_NONE;  in em_dr_write()
3691 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)  in em_wrmsr()  argument
3695 	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)  in em_wrmsr()
3696 		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);  in em_wrmsr()
3697 	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))  in em_wrmsr()
3698 		return emulate_gp(ctxt, 0);  in em_wrmsr()
3703 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)  in em_rdmsr()  argument
3707 	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))  in em_rdmsr()
3708 		return emulate_gp(ctxt, 0);  in em_rdmsr()
3710 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;  in em_rdmsr()
3711 	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;  in em_rdmsr()
3715 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)  in em_store_sreg()  argument
3718 	    (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in em_store_sreg()
3719 	    ctxt->ops->cpl(ctxt) > 0)  in em_store_sreg()
3720 		return emulate_gp(ctxt, 0);  in em_store_sreg()
3722 	ctxt->dst.val = get_segment_selector(ctxt, segment);  in em_store_sreg()
3723 	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)  in em_store_sreg()
3724 		ctxt->dst.bytes = 2;  in em_store_sreg()
3728 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)  in em_mov_rm_sreg()  argument
3730 	if (ctxt->modrm_reg > VCPU_SREG_GS)  in em_mov_rm_sreg()
3731 		return emulate_ud(ctxt);  in em_mov_rm_sreg()
3733 	return em_store_sreg(ctxt, ctxt->modrm_reg);  in em_mov_rm_sreg()
3736 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)  in em_mov_sreg_rm()  argument
3738 	u16 sel = ctxt->src.val;  in em_mov_sreg_rm()
3740 	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)  in em_mov_sreg_rm()
3741 		return emulate_ud(ctxt);  in em_mov_sreg_rm()
3743 	if (ctxt->modrm_reg == VCPU_SREG_SS)  in em_mov_sreg_rm()
3744 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;  in em_mov_sreg_rm()
3747 	ctxt->dst.type = OP_NONE;  in em_mov_sreg_rm()
3748 	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);  in em_mov_sreg_rm()
3751 static int em_sldt(struct x86_emulate_ctxt *ctxt)  in em_sldt()  argument
3753 	return em_store_sreg(ctxt, VCPU_SREG_LDTR);  in em_sldt()
3756 static int em_lldt(struct x86_emulate_ctxt *ctxt)  in em_lldt()  argument
3758 	u16 sel = ctxt->src.val;  in em_lldt()
3761 	ctxt->dst.type = OP_NONE;  in em_lldt()
3762 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);  in em_lldt()
3765 static int em_str(struct x86_emulate_ctxt *ctxt)  in em_str()  argument
3767 	return em_store_sreg(ctxt, VCPU_SREG_TR);  in em_str()
3770 static int em_ltr(struct x86_emulate_ctxt *ctxt)  in em_ltr()  argument
3772 	u16 sel = ctxt->src.val;  in em_ltr()
3775 	ctxt->dst.type = OP_NONE;  in em_ltr()
3776 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);  in em_ltr()
3779 static int em_invlpg(struct x86_emulate_ctxt *ctxt)  in em_invlpg()  argument
3784 	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);  in em_invlpg()
3786 		ctxt->ops->invlpg(ctxt, linear);  in em_invlpg()
3788 	ctxt->dst.type = OP_NONE;  in em_invlpg()
3792 static int em_clts(struct x86_emulate_ctxt *ctxt)  in em_clts()  argument
3796 	cr0 = ctxt->ops->get_cr(ctxt, 0);  in em_clts()
3798 	ctxt->ops->set_cr(ctxt, 0, cr0);  in em_clts()
3802 static int em_hypercall(struct x86_emulate_ctxt *ctxt)  in em_hypercall()  argument
3804 	int rc = ctxt->ops->fix_hypercall(ctxt);  in em_hypercall()
3810 	ctxt->_eip = ctxt->eip;  in em_hypercall()
3812 	ctxt->dst.type = OP_NONE;  in em_hypercall()
3816 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,  in emulate_store_desc_ptr()  argument
3817 				  void (*get)(struct x86_emulate_ctxt *ctxt,  in emulate_store_desc_ptr()  argument
3822 	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in emulate_store_desc_ptr()
3823 	    ctxt->ops->cpl(ctxt) > 0)  in emulate_store_desc_ptr()
3824 		return emulate_gp(ctxt, 0);  in emulate_store_desc_ptr()
3826 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in emulate_store_desc_ptr()
3827 		ctxt->op_bytes = 8;  in emulate_store_desc_ptr()
3828 	get(ctxt, &desc_ptr);  in emulate_store_desc_ptr()
3829 	if (ctxt->op_bytes == 2) {  in emulate_store_desc_ptr()
3830 		ctxt->op_bytes = 4;  in emulate_store_desc_ptr()
3834 	ctxt->dst.type = OP_NONE;  in emulate_store_desc_ptr()
3835 	return segmented_write_std(ctxt, ctxt->dst.addr.mem,  in emulate_store_desc_ptr()
3836 				   &desc_ptr, 2 + ctxt->op_bytes);  in emulate_store_desc_ptr()
3839 static int em_sgdt(struct x86_emulate_ctxt *ctxt)  in em_sgdt()  argument
3841 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);  in em_sgdt()
3844 static int em_sidt(struct x86_emulate_ctxt *ctxt)  in em_sidt()  argument
3846 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);  in em_sidt()
3849 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)  in em_lgdt_lidt()  argument
3854 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_lgdt_lidt()
3855 		ctxt->op_bytes = 8;  in em_lgdt_lidt()
3856 	rc = read_descriptor(ctxt, ctxt->src.addr.mem,  in em_lgdt_lidt()
3858 			     ctxt->op_bytes);  in em_lgdt_lidt()
3861 	if (ctxt->mode == X86EMUL_MODE_PROT64 &&  in em_lgdt_lidt()
3862 	    emul_is_noncanonical_address(desc_ptr.address, ctxt))  in em_lgdt_lidt()
3863 		return emulate_gp(ctxt, 0);  in em_lgdt_lidt()
3865 		ctxt->ops->set_gdt(ctxt, &desc_ptr);  in em_lgdt_lidt()
3867 		ctxt->ops->set_idt(ctxt, &desc_ptr);  in em_lgdt_lidt()
3869 	ctxt->dst.type = OP_NONE;  in em_lgdt_lidt()
3873 static int em_lgdt(struct x86_emulate_ctxt *ctxt)  in em_lgdt()  argument
3875 	return em_lgdt_lidt(ctxt, true);  in em_lgdt()
3878 static int em_lidt(struct x86_emulate_ctxt *ctxt)  in em_lidt()  argument
3880 	return em_lgdt_lidt(ctxt, false);  in em_lidt()
3883 static int em_smsw(struct x86_emulate_ctxt *ctxt)  in em_smsw()  argument
3885 	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in em_smsw()
3886 	    ctxt->ops->cpl(ctxt) > 0)  in em_smsw()
3887 		return emulate_gp(ctxt, 0);  in em_smsw()
3889 	if (ctxt->dst.type == OP_MEM)  in em_smsw()
3890 		ctxt->dst.bytes = 2;  in em_smsw()
3891 	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);  in em_smsw()
3895 static int em_lmsw(struct x86_emulate_ctxt *ctxt)  in em_lmsw()  argument
3897 	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)  in em_lmsw()
3898 			  | (ctxt->src.val & 0x0f));  in em_lmsw()
3899 	ctxt->dst.type = OP_NONE;  in em_lmsw()
3903 static int em_loop(struct x86_emulate_ctxt *ctxt)  in em_loop()  argument
3907 	register_address_increment(ctxt, VCPU_REGS_RCX, -1);  in em_loop()
3908 	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&  in em_loop()
3909 	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))  in em_loop()
3910 		rc = jmp_rel(ctxt, ctxt->src.val);  in em_loop()
3915 static int em_jcxz(struct x86_emulate_ctxt *ctxt)  in em_jcxz()  argument
3919 	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)  in em_jcxz()
3920 		rc = jmp_rel(ctxt, ctxt->src.val);  in em_jcxz()
3925 static int em_in(struct x86_emulate_ctxt *ctxt)  in em_in()  argument
3927 	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,  in em_in()
3928 			     &ctxt->dst.val))  in em_in()
3934 static int em_out(struct x86_emulate_ctxt *ctxt)  in em_out()  argument
3936 	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,  in em_out()
3937 				    &ctxt->src.val, 1);  in em_out()
3939 	ctxt->dst.type = OP_NONE;  in em_out()
3943 static int em_cli(struct x86_emulate_ctxt *ctxt)  in em_cli()  argument
3945 	if (emulator_bad_iopl(ctxt))  in em_cli()
3946 		return emulate_gp(ctxt, 0);  in em_cli()
3948 	ctxt->eflags &= ~X86_EFLAGS_IF;  in em_cli()
3952 static int em_sti(struct x86_emulate_ctxt *ctxt)  in em_sti()  argument
3954 	if (emulator_bad_iopl(ctxt))  in em_sti()
3955 		return emulate_gp(ctxt, 0);  in em_sti()
3957 	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;  in em_sti()
3958 	ctxt->eflags |= X86_EFLAGS_IF;  in em_sti()
3962 static int em_cpuid(struct x86_emulate_ctxt *ctxt)  in em_cpuid()  argument
3967 	ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);  in em_cpuid()
3969 	    ctxt->ops->cpl(ctxt)) {  in em_cpuid()
3970 		return emulate_gp(ctxt, 0);  in em_cpuid()
3973 	eax = reg_read(ctxt, VCPU_REGS_RAX);  in em_cpuid()
3974 	ecx = reg_read(ctxt, VCPU_REGS_RCX);  in em_cpuid()
3975 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);  in em_cpuid()
3976 	*reg_write(ctxt, VCPU_REGS_RAX) = eax;  in em_cpuid()
3977 	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;  in em_cpuid()
3978 	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;  in em_cpuid()
3979 	*reg_write(ctxt, VCPU_REGS_RDX) = edx;  in em_cpuid()
3983 static int em_sahf(struct x86_emulate_ctxt *ctxt)  in em_sahf()  argument
3989 	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;  in em_sahf()
3991 	ctxt->eflags &= ~0xffUL;  in em_sahf()
3992 	ctxt->eflags |= flags | X86_EFLAGS_FIXED;  in em_sahf()
3996 static int em_lahf(struct x86_emulate_ctxt *ctxt)  in em_lahf()  argument
3998 	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;  in em_lahf()
3999 	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;  in em_lahf()
4003 static int em_bswap(struct x86_emulate_ctxt *ctxt)  in em_bswap()  argument
4005 	switch (ctxt->op_bytes) {  in em_bswap()
4008 		asm("bswap %0" : "+r"(ctxt->dst.val));  in em_bswap()
4012 		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));  in em_bswap()
4018 static int em_clflush(struct x86_emulate_ctxt *ctxt)  in em_clflush()  argument
4024 static int em_movsxd(struct x86_emulate_ctxt *ctxt)  in em_movsxd()  argument
4026 	ctxt->dst.val = (s32) ctxt->src.val;  in em_movsxd()
4030 static int check_fxsr(struct x86_emulate_ctxt *ctxt)  in check_fxsr()  argument
4034 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);  in check_fxsr()
4036 		return emulate_ud(ctxt);  in check_fxsr()
4038 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in check_fxsr()
4039 		return emulate_nm(ctxt);  in check_fxsr()
4045 	if (ctxt->mode >= X86EMUL_MODE_PROT64)  in check_fxsr()
4060 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)  in fxstate_size()  argument
4063 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in fxstate_size()
4066 	cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;  in fxstate_size()
4088 static int em_fxsave(struct x86_emulate_ctxt *ctxt)  in em_fxsave()  argument
4093 	rc = check_fxsr(ctxt);  in em_fxsave()
4102 	return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,  in em_fxsave()
4103 		                   fxstate_size(ctxt));  in em_fxsave()
4126 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)  in em_fxrstor()  argument
4132 	rc = check_fxsr(ctxt);  in em_fxrstor()
4136 	size = fxstate_size(ctxt);  in em_fxrstor()
4137 	rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);  in em_fxrstor()
4148 		rc = emulate_gp(ctxt, 0);  in em_fxrstor()
4159 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)  in em_xsetbv()  argument
4163 	eax = reg_read(ctxt, VCPU_REGS_RAX);  in em_xsetbv()
4164 	edx = reg_read(ctxt, VCPU_REGS_RDX);  in em_xsetbv()
4165 	ecx = reg_read(ctxt, VCPU_REGS_RCX);  in em_xsetbv()
4167 	if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))  in em_xsetbv()
4168 		return emulate_gp(ctxt, 0);  in em_xsetbv()
4185 static int check_cr_read(struct x86_emulate_ctxt *ctxt)  in check_cr_read()  argument
4187 	if (!valid_cr(ctxt->modrm_reg))  in check_cr_read()
4188 		return emulate_ud(ctxt);  in check_cr_read()
4193 static int check_cr_write(struct x86_emulate_ctxt *ctxt)  in check_cr_write()  argument
4195 	u64 new_val = ctxt->src.val64;  in check_cr_write()
4196 	int cr = ctxt->modrm_reg;  in check_cr_write()
4208 		return emulate_ud(ctxt);  in check_cr_write()
4211 		return emulate_gp(ctxt, 0);  in check_cr_write()
4218 			return emulate_gp(ctxt, 0);  in check_cr_write()
4220 		cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_cr_write()
4221 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in check_cr_write()
4225 			return emulate_gp(ctxt, 0);  in check_cr_write()
4232 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in check_cr_write()
4239 			if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,  in check_cr_write()
4245 			if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)  in check_cr_write()
4250 			return emulate_gp(ctxt, 0);  in check_cr_write()
4255 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in check_cr_write()
4258 			return emulate_gp(ctxt, 0);  in check_cr_write()
4267 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)  in check_dr7_gd()  argument
4271 	ctxt->ops->get_dr(ctxt, 7, &dr7);  in check_dr7_gd()
4277 static int check_dr_read(struct x86_emulate_ctxt *ctxt)  in check_dr_read()  argument
4279 	int dr = ctxt->modrm_reg;  in check_dr_read()
4283 		return emulate_ud(ctxt);  in check_dr_read()
4285 	cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_dr_read()
4287 		return emulate_ud(ctxt);  in check_dr_read()
4289 	if (check_dr7_gd(ctxt)) {  in check_dr_read()
4292 		ctxt->ops->get_dr(ctxt, 6, &dr6);  in check_dr_read()
4295 		ctxt->ops->set_dr(ctxt, 6, dr6);  in check_dr_read()
4296 		return emulate_db(ctxt);  in check_dr_read()
4302 static int check_dr_write(struct x86_emulate_ctxt *ctxt)  in check_dr_write()  argument
4304 	u64 new_val = ctxt->src.val64;  in check_dr_write()
4305 	int dr = ctxt->modrm_reg;  in check_dr_write()
4308 		return emulate_gp(ctxt, 0);  in check_dr_write()
4310 	return check_dr_read(ctxt);  in check_dr_write()
4313 static int check_svme(struct x86_emulate_ctxt *ctxt)  in check_svme()  argument
4317 	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in check_svme()
4320 		return emulate_ud(ctxt);  in check_svme()
4325 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)  in check_svme_pa()  argument
4327 	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);  in check_svme_pa()
4331 		return emulate_gp(ctxt, 0);  in check_svme_pa()
4333 	return check_svme(ctxt);  in check_svme_pa()
4336 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)  in check_rdtsc()  argument
4338 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_rdtsc()
4340 	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))  in check_rdtsc()
4341 		return emulate_ud(ctxt);  in check_rdtsc()
4346 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)  in check_rdpmc()  argument
4348 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_rdpmc()
4349 	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);  in check_rdpmc()
4358 	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||  in check_rdpmc()
4359 	    ctxt->ops->check_pmc(ctxt, rcx))  in check_rdpmc()
4360 		return emulate_gp(ctxt, 0);  in check_rdpmc()
4365 static int check_perm_in(struct x86_emulate_ctxt *ctxt)  in check_perm_in()  argument
4367 	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);  in check_perm_in()
4368 	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))  in check_perm_in()
4369 		return emulate_gp(ctxt, 0);  in check_perm_in()
4374 static int check_perm_out(struct x86_emulate_ctxt *ctxt)  in check_perm_out()  argument
4376 	ctxt->src.bytes = min(ctxt->src.bytes, 4u);  in check_perm_out()
4377 	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))  in check_perm_out()
4378 		return emulate_gp(ctxt, 0);  in check_perm_out()
4930 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)  in imm_size()  argument
4934 	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in imm_size()
4940 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,  in decode_imm()  argument
4947 	op->addr.mem.ea = ctxt->_eip;  in decode_imm()
4951 		op->val = insn_fetch(s8, ctxt);  in decode_imm()
4954 		op->val = insn_fetch(s16, ctxt);  in decode_imm()
4957 		op->val = insn_fetch(s32, ctxt);  in decode_imm()
4960 		op->val = insn_fetch(s64, ctxt);  in decode_imm()
4980 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,  in decode_operand()  argument
4987 		decode_register_operand(ctxt, op);  in decode_operand()
4990 		rc = decode_imm(ctxt, op, 1, false);  in decode_operand()
4993 		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4995 		*op = ctxt->memop;  in decode_operand()
4996 		ctxt->memopp = op;  in decode_operand()
4997 		if (ctxt->d & BitOp)  in decode_operand()
4998 			fetch_bit_operand(ctxt);  in decode_operand()
5002 		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;  in decode_operand()
5006 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
5007 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in decode_operand()
5013 		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;  in decode_operand()
5014 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in decode_operand()
5019 		if (ctxt->d & ByteOp) {  in decode_operand()
5024 		op->bytes = ctxt->op_bytes;  in decode_operand()
5025 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in decode_operand()
5031 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
5033 			register_address(ctxt, VCPU_REGS_RDI);  in decode_operand()
5041 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in decode_operand()
5047 		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;  in decode_operand()
5050 		rc = decode_imm(ctxt, op, 1, true);  in decode_operand()
5058 		rc = decode_imm(ctxt, op, imm_size(ctxt), true);  in decode_operand()
5061 		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);  in decode_operand()
5064 		ctxt->memop.bytes = 1;  in decode_operand()
5065 		if (ctxt->memop.type == OP_REG) {  in decode_operand()
5066 			ctxt->memop.addr.reg = decode_register(ctxt,  in decode_operand()
5067 					ctxt->modrm_rm, true);  in decode_operand()
5068 			fetch_register_operand(&ctxt->memop);  in decode_operand()
5072 		ctxt->memop.bytes = 2;  in decode_operand()
5075 		ctxt->memop.bytes = 4;  in decode_operand()
5078 		rc = decode_imm(ctxt, op, 2, false);  in decode_operand()
5081 		rc = decode_imm(ctxt, op, imm_size(ctxt), false);  in decode_operand()
5085 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
5087 			register_address(ctxt, VCPU_REGS_RSI);  in decode_operand()
5088 		op->addr.mem.seg = ctxt->seg_override;  in decode_operand()
5094 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
5096 			address_mask(ctxt,  in decode_operand()
5097 				reg_read(ctxt, VCPU_REGS_RBX) +  in decode_operand()
5098 				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));  in decode_operand()
5099 		op->addr.mem.seg = ctxt->seg_override;  in decode_operand()
5104 		op->addr.mem.ea = ctxt->_eip;  in decode_operand()
5105 		op->bytes = ctxt->op_bytes + 2;  in decode_operand()
5106 		insn_fetch_arr(op->valptr, op->bytes, ctxt);  in decode_operand()
5109 		ctxt->memop.bytes = ctxt->op_bytes + 2;  in decode_operand()
5146 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)  in x86_decode_insn()  argument
5149 	int mode = ctxt->mode;  in x86_decode_insn()
5157 	ctxt->memop.type = OP_NONE;  in x86_decode_insn()
5158 	ctxt->memopp = NULL;  in x86_decode_insn()
5159 	ctxt->_eip = ctxt->eip;  in x86_decode_insn()
5160 	ctxt->fetch.ptr = ctxt->fetch.data;  in x86_decode_insn()
5161 	ctxt->fetch.end = ctxt->fetch.data + insn_len;  in x86_decode_insn()
5162 	ctxt->opcode_len = 1;  in x86_decode_insn()
5164 		memcpy(ctxt->fetch.data, insn, insn_len);  in x86_decode_insn()
5166 		rc = __do_insn_fetch_bytes(ctxt, 1);  in x86_decode_insn()
5175 		ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);  in x86_decode_insn()
5195 	ctxt->op_bytes = def_op_bytes;  in x86_decode_insn()
5196 	ctxt->ad_bytes = def_ad_bytes;  in x86_decode_insn()
5200 		switch (ctxt->b = insn_fetch(u8, ctxt)) {  in x86_decode_insn()
5204 			ctxt->op_bytes = def_op_bytes ^ 6;  in x86_decode_insn()
5209 				ctxt->ad_bytes = def_ad_bytes ^ 12;  in x86_decode_insn()
5212 				ctxt->ad_bytes = def_ad_bytes ^ 6;  in x86_decode_insn()
5219 			ctxt->seg_override = (ctxt->b >> 3) & 3;  in x86_decode_insn()
5224 			ctxt->seg_override = ctxt->b & 7;  in x86_decode_insn()
5229 			ctxt->rex_prefix = ctxt->b;  in x86_decode_insn()
5232 			ctxt->lock_prefix = 1;  in x86_decode_insn()
5236 			ctxt->rep_prefix = ctxt->b;  in x86_decode_insn()
5244 		ctxt->rex_prefix = 0;  in x86_decode_insn()
5250 	if (ctxt->rex_prefix & 8)  in x86_decode_insn()
5251 		ctxt->op_bytes = 8;	/* REX.W */  in x86_decode_insn()
5254 	opcode = opcode_table[ctxt->b];  in x86_decode_insn()
5256 	if (ctxt->b == 0x0f) {  in x86_decode_insn()
5257 		ctxt->opcode_len = 2;  in x86_decode_insn()
5258 		ctxt->b = insn_fetch(u8, ctxt);  in x86_decode_insn()
5259 		opcode = twobyte_table[ctxt->b];  in x86_decode_insn()
5262 		if (ctxt->b == 0x38) {  in x86_decode_insn()
5263 			ctxt->opcode_len = 3;  in x86_decode_insn()
5264 			ctxt->b = insn_fetch(u8, ctxt);  in x86_decode_insn()
5265 			opcode = opcode_map_0f_38[ctxt->b];  in x86_decode_insn()
5268 	ctxt->d = opcode.flags;  in x86_decode_insn()
5270 	if (ctxt->d & ModRM)  in x86_decode_insn()
5271 		ctxt->modrm = insn_fetch(u8, ctxt);  in x86_decode_insn()
5274 	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&  in x86_decode_insn()
5275 	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {  in x86_decode_insn()
5276 		ctxt->d = NotImpl;  in x86_decode_insn()
5279 	while (ctxt->d & GroupMask) {  in x86_decode_insn()
5280 		switch (ctxt->d & GroupMask) {  in x86_decode_insn()
5282 			goffset = (ctxt->modrm >> 3) & 7;  in x86_decode_insn()
5286 			goffset = (ctxt->modrm >> 3) & 7;  in x86_decode_insn()
5287 			if ((ctxt->modrm >> 6) == 3)  in x86_decode_insn()
5293 			goffset = ctxt->modrm & 7;  in x86_decode_insn()
5297 			if (ctxt->rep_prefix && op_prefix)  in x86_decode_insn()
5299 			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;  in x86_decode_insn()
5308 			if (ctxt->modrm > 0xbf)  in x86_decode_insn()
5309 				opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];  in x86_decode_insn()
5311 				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];  in x86_decode_insn()
5314 			if ((ctxt->modrm >> 6) == 3)  in x86_decode_insn()
5320 			if (ctxt->mode == X86EMUL_MODE_PROT64)  in x86_decode_insn()
5329 		ctxt->d &= ~(u64)GroupMask;  in x86_decode_insn()
5330 		ctxt->d |= opcode.flags;  in x86_decode_insn()
5334 	if (ctxt->d == 0)  in x86_decode_insn()
5337 	ctxt->execute = opcode.u.execute;  in x86_decode_insn()
5339 	if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))  in x86_decode_insn()
5342 	if (unlikely(ctxt->d &  in x86_decode_insn()
5349 		ctxt->check_perm = opcode.check_perm;  in x86_decode_insn()
5350 		ctxt->intercept = opcode.intercept;  in x86_decode_insn()
5352 		if (ctxt->d & NotImpl)  in x86_decode_insn()
5356 			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))  in x86_decode_insn()
5357 				ctxt->op_bytes = 8;  in x86_decode_insn()
5358 			else if (ctxt->d & NearBranch)  in x86_decode_insn()
5359 				ctxt->op_bytes = 8;  in x86_decode_insn()
5362 		if (ctxt->d & Op3264) {  in x86_decode_insn()
5364 				ctxt->op_bytes = 8;  in x86_decode_insn()
5366 				ctxt->op_bytes = 4;  in x86_decode_insn()
5369 		if ((ctxt->d & No16) && ctxt->op_bytes == 2)  in x86_decode_insn()
5370 			ctxt->op_bytes = 4;  in x86_decode_insn()
5372 		if (ctxt->d & Sse)  in x86_decode_insn()
5373 			ctxt->op_bytes = 16;  in x86_decode_insn()
5374 		else if (ctxt->d & Mmx)  in x86_decode_insn()
5375 			ctxt->op_bytes = 8;  in x86_decode_insn()
5379 	if (ctxt->d & ModRM) {  in x86_decode_insn()
5380 		rc = decode_modrm(ctxt, &ctxt->memop);  in x86_decode_insn()
5383 			ctxt->seg_override = ctxt->modrm_seg;  in x86_decode_insn()
5385 	} else if (ctxt->d & MemAbs)  in x86_decode_insn()
5386 		rc = decode_abs(ctxt, &ctxt->memop);  in x86_decode_insn()
5391 		ctxt->seg_override = VCPU_SREG_DS;  in x86_decode_insn()
5393 	ctxt->memop.addr.mem.seg = ctxt->seg_override;  in x86_decode_insn()
5399 	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);  in x86_decode_insn()
5407 	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);  in x86_decode_insn()
5412 	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);  in x86_decode_insn()
5414 	if (ctxt->rip_relative && likely(ctxt->memopp))  in x86_decode_insn()
5415 		ctxt->memopp->addr.mem.ea = address_mask(ctxt,  in x86_decode_insn()
5416 					ctxt->memopp->addr.mem.ea + ctxt->_eip);  in x86_decode_insn()
5420 		ctxt->have_exception = true;  in x86_decode_insn()
5424 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)  in x86_page_table_writing_insn()  argument
5426 	return ctxt->d & PageTable;  in x86_page_table_writing_insn()
5429 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)  in string_insn_completed()  argument
5438 	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||  in string_insn_completed()
5439 	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))  in string_insn_completed()
5440 	    && (((ctxt->rep_prefix == REPE_PREFIX) &&  in string_insn_completed()
5441 		 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))  in string_insn_completed()
5442 		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&  in string_insn_completed()
5443 		    ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))  in string_insn_completed()
5449 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)  in flush_pending_x87_faults()  argument
5456 		return emulate_exception(ctxt, MF_VECTOR, 0, false);  in flush_pending_x87_faults()
5461 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,  in fetch_possible_mmx_operand()  argument
5465 		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);  in fetch_possible_mmx_operand()
5468 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))  in fastop()  argument
5470 	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;  in fastop()
5472 	if (!(ctxt->d & ByteOp))  in fastop()
5473 		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;  in fastop()
5476 	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),  in fastop()
5478 	    : "c"(ctxt->src2.val));  in fastop()
5480 	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);  in fastop()
5482 		return emulate_de(ctxt);  in fastop()
5486 void init_decode_cache(struct x86_emulate_ctxt *ctxt)  in init_decode_cache()  argument
5488 	memset(&ctxt->rip_relative, 0,  in init_decode_cache()
5489 	       (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);  in init_decode_cache()
5491 	ctxt->io_read.pos = 0;  in init_decode_cache()
5492 	ctxt->io_read.end = 0;  in init_decode_cache()
5493 	ctxt->mem_read.end = 0;  in init_decode_cache()
5496 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)  in x86_emulate_insn()  argument
5498 	const struct x86_emulate_ops *ops = ctxt->ops;  in x86_emulate_insn()
5500 	int saved_dst_type = ctxt->dst.type;  in x86_emulate_insn()
5503 	ctxt->mem_read.pos = 0;  in x86_emulate_insn()
5506 	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {  in x86_emulate_insn()
5507 		rc = emulate_ud(ctxt);  in x86_emulate_insn()
5511 	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {  in x86_emulate_insn()
5512 		rc = emulate_ud(ctxt);  in x86_emulate_insn()
5516 	emul_flags = ctxt->ops->get_hflags(ctxt);  in x86_emulate_insn()
5517 	if (unlikely(ctxt->d &  in x86_emulate_insn()
5519 		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||  in x86_emulate_insn()
5520 				(ctxt->d & Undefined)) {  in x86_emulate_insn()
5521 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5525 		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))  in x86_emulate_insn()
5526 		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {  in x86_emulate_insn()
5527 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5531 		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {  in x86_emulate_insn()
5532 			rc = emulate_nm(ctxt);  in x86_emulate_insn()
5536 		if (ctxt->d & Mmx) {  in x86_emulate_insn()
5537 			rc = flush_pending_x87_faults(ctxt);  in x86_emulate_insn()
5544 			fetch_possible_mmx_operand(ctxt, &ctxt->src);  in x86_emulate_insn()
5545 			fetch_possible_mmx_operand(ctxt, &ctxt->src2);  in x86_emulate_insn()
5546 			if (!(ctxt->d & Mov))  in x86_emulate_insn()
5547 				fetch_possible_mmx_operand(ctxt, &ctxt->dst);  in x86_emulate_insn()
5550 		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {  in x86_emulate_insn()
5551 			rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5558 		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {  in x86_emulate_insn()
5559 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5564 		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {  in x86_emulate_insn()
5565 			if (ctxt->d & PrivUD)  in x86_emulate_insn()
5566 				rc = emulate_ud(ctxt);  in x86_emulate_insn()
5568 				rc = emulate_gp(ctxt, 0);  in x86_emulate_insn()
5573 		if (ctxt->d & CheckPerm) {  in x86_emulate_insn()
5574 			rc = ctxt->check_perm(ctxt);  in x86_emulate_insn()
5579 		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {  in x86_emulate_insn()
5580 			rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5586 		if (ctxt->rep_prefix && (ctxt->d & String)) {  in x86_emulate_insn()
5588 			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {  in x86_emulate_insn()
5589 				string_registers_quirk(ctxt);  in x86_emulate_insn()
5590 				ctxt->eip = ctxt->_eip;  in x86_emulate_insn()
5591 				ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5597 	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {  in x86_emulate_insn()
5598 		rc = segmented_read(ctxt, ctxt->src.addr.mem,  in x86_emulate_insn()
5599 				    ctxt->src.valptr, ctxt->src.bytes);  in x86_emulate_insn()
5602 		ctxt->src.orig_val64 = ctxt->src.val64;  in x86_emulate_insn()
5605 	if (ctxt->src2.type == OP_MEM) {  in x86_emulate_insn()
5606 		rc = segmented_read(ctxt, ctxt->src2.addr.mem,  in x86_emulate_insn()
5607 				    &ctxt->src2.val, ctxt->src2.bytes);  in x86_emulate_insn()
5612 	if ((ctxt->d & DstMask) == ImplicitOps)  in x86_emulate_insn()
5616 	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {  in x86_emulate_insn()
5618 		rc = segmented_read(ctxt, ctxt->dst.addr.mem,  in x86_emulate_insn()
5619 				   &ctxt->dst.val, ctxt->dst.bytes);  in x86_emulate_insn()
5621 			if (!(ctxt->d & NoWrite) &&  in x86_emulate_insn()
5623 			    ctxt->exception.vector == PF_VECTOR)  in x86_emulate_insn()
5624 				ctxt->exception.error_code |= PFERR_WRITE_MASK;  in x86_emulate_insn()
5629 	ctxt->dst.orig_val64 = ctxt->dst.val64;  in x86_emulate_insn()
5633 	if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {  in x86_emulate_insn()
5634 		rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5640 	if (ctxt->rep_prefix && (ctxt->d & String))  in x86_emulate_insn()
5641 		ctxt->eflags |= X86_EFLAGS_RF;  in x86_emulate_insn()
5643 		ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5645 	if (ctxt->execute) {  in x86_emulate_insn()
5646 		if (ctxt->d & Fastop) {  in x86_emulate_insn()
5647 			void (*fop)(struct fastop *) = (void *)ctxt->execute;  in x86_emulate_insn()
5648 			rc = fastop(ctxt, fop);  in x86_emulate_insn()
5653 		rc = ctxt->execute(ctxt);  in x86_emulate_insn()
5659 	if (ctxt->opcode_len == 2)  in x86_emulate_insn()
5661 	else if (ctxt->opcode_len == 3)  in x86_emulate_insn()
5664 	switch (ctxt->b) {  in x86_emulate_insn()
5666 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5667 			rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5670 		ctxt->dst.val = ctxt->src.addr.mem.ea;  in x86_emulate_insn()
5673 		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))  in x86_emulate_insn()
5674 			ctxt->dst.type = OP_NONE;  in x86_emulate_insn()
5676 			rc = em_xchg(ctxt);  in x86_emulate_insn()
5679 		switch (ctxt->op_bytes) {  in x86_emulate_insn()
5680 		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;  in x86_emulate_insn()
5681 		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;  in x86_emulate_insn()
5682 		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;  in x86_emulate_insn()
5686 		rc = emulate_int(ctxt, 3);  in x86_emulate_insn()
5689 		rc = emulate_int(ctxt, ctxt->src.val);  in x86_emulate_insn()
5692 		if (ctxt->eflags & X86_EFLAGS_OF)  in x86_emulate_insn()
5693 			rc = emulate_int(ctxt, 4);  in x86_emulate_insn()
5697 		rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5698 		ctxt->dst.type = OP_NONE; /* Disable writeback. */  in x86_emulate_insn()
5701 		ctxt->ops->halt(ctxt);  in x86_emulate_insn()
5705 		ctxt->eflags ^= X86_EFLAGS_CF;  in x86_emulate_insn()
5708 		ctxt->eflags &= ~X86_EFLAGS_CF;  in x86_emulate_insn()
5711 		ctxt->eflags |= X86_EFLAGS_CF;  in x86_emulate_insn()
5714 		ctxt->eflags &= ~X86_EFLAGS_DF;  in x86_emulate_insn()
5717 		ctxt->eflags |= X86_EFLAGS_DF;  in x86_emulate_insn()
5727 	if (ctxt->d & SrcWrite) {  in x86_emulate_insn()
5728 		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);  in x86_emulate_insn()
5729 		rc = writeback(ctxt, &ctxt->src);  in x86_emulate_insn()
5733 	if (!(ctxt->d & NoWrite)) {  in x86_emulate_insn()
5734 		rc = writeback(ctxt, &ctxt->dst);  in x86_emulate_insn()
5743 	ctxt->dst.type = saved_dst_type;  in x86_emulate_insn()
5745 	if ((ctxt->d & SrcMask) == SrcSI)  in x86_emulate_insn()
5746 		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);  in x86_emulate_insn()
5748 	if ((ctxt->d & DstMask) == DstDI)  in x86_emulate_insn()
5749 		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);  in x86_emulate_insn()
5751 	if (ctxt->rep_prefix && (ctxt->d & String)) {  in x86_emulate_insn()
5753 		struct read_cache *r = &ctxt->io_read;  in x86_emulate_insn()
5754 		if ((ctxt->d & SrcMask) == SrcSI)  in x86_emulate_insn()
5755 			count = ctxt->src.count;  in x86_emulate_insn()
5757 			count = ctxt->dst.count;  in x86_emulate_insn()
5758 		register_address_increment(ctxt, VCPU_REGS_RCX, -count);  in x86_emulate_insn()
5760 		if (!string_insn_completed(ctxt)) {  in x86_emulate_insn()
5765 			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&  in x86_emulate_insn()
5772 				ctxt->mem_read.end = 0;  in x86_emulate_insn()
5773 				writeback_registers(ctxt);  in x86_emulate_insn()
5778 		ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5781 	ctxt->eip = ctxt->_eip;  in x86_emulate_insn()
5785 		WARN_ON(ctxt->exception.vector > 0x1f);  in x86_emulate_insn()
5786 		ctxt->have_exception = true;  in x86_emulate_insn()
5792 		writeback_registers(ctxt);  in x86_emulate_insn()
5797 	switch (ctxt->b) {  in x86_emulate_insn()
5799 		(ctxt->ops->wbinvd)(ctxt);  in x86_emulate_insn()
5807 		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);  in x86_emulate_insn()
5810 		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);  in x86_emulate_insn()
5813 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5814 			ctxt->dst.val = ctxt->src.val;  in x86_emulate_insn()
5815 		else if (ctxt->op_bytes != 4)  in x86_emulate_insn()
5816 			ctxt->dst.type = OP_NONE; /* no writeback */  in x86_emulate_insn()
5819 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5820 			rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5823 		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);  in x86_emulate_insn()
5826 		ctxt->dst.bytes = ctxt->op_bytes;  in x86_emulate_insn()
5827 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val  in x86_emulate_insn()
5828 						       : (u16) ctxt->src.val;  in x86_emulate_insn()
5831 		ctxt->dst.bytes = ctxt->op_bytes;  in x86_emulate_insn()
5832 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :  in x86_emulate_insn()
5833 							(s16) ctxt->src.val;  in x86_emulate_insn()
5850 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)  in emulator_invalidate_register_cache()  argument
5852 	invalidate_registers(ctxt);  in emulator_invalidate_register_cache()
5855 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)  in emulator_writeback_register_cache()  argument
5857 	writeback_registers(ctxt);  in emulator_writeback_register_cache()
5860 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)  in emulator_can_use_gpa()  argument
5862 	if (ctxt->rep_prefix && (ctxt->d & String))  in emulator_can_use_gpa()
5865 	if (ctxt->d & TwoMemOp)  in emulator_can_use_gpa()