238 static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,  in vc_fetch_insn_kernel()  argument
241 	return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);  in vc_fetch_insn_kernel()
244 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)  in vc_decode_insn()  argument
250 	if (user_mode(ctxt->regs)) {  in vc_decode_insn()
251 		res = insn_fetch_from_user(ctxt->regs, buffer);  in vc_decode_insn()
253 			ctxt->fi.vector     = X86_TRAP_PF;  in vc_decode_insn()
254 			ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;  in vc_decode_insn()
255 			ctxt->fi.cr2        = ctxt->regs->ip;  in vc_decode_insn()
259 		if (!insn_decode(&ctxt->insn, ctxt->regs, buffer, res))  in vc_decode_insn()
262 		res = vc_fetch_insn_kernel(ctxt, buffer);  in vc_decode_insn()
264 			ctxt->fi.vector     = X86_TRAP_PF;  in vc_decode_insn()
265 			ctxt->fi.error_code = X86_PF_INSTR;  in vc_decode_insn()
266 			ctxt->fi.cr2        = ctxt->regs->ip;  in vc_decode_insn()
270 		insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE - res, 1);  in vc_decode_insn()
271 		insn_get_length(&ctxt->insn);  in vc_decode_insn()
274 	ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED;  in vc_decode_insn()
279 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,  in vc_write_mem()  argument
318 	if (user_mode(ctxt->regs))  in vc_write_mem()
321 	ctxt->fi.vector = X86_TRAP_PF;  in vc_write_mem()
322 	ctxt->fi.error_code = error_code;  in vc_write_mem()
323 	ctxt->fi.cr2 = (unsigned long)dst;  in vc_write_mem()
328 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,  in vc_read_mem()  argument
367 	if (user_mode(ctxt->regs))  in vc_read_mem()
370 	ctxt->fi.vector = X86_TRAP_PF;  in vc_read_mem()
371 	ctxt->fi.error_code = error_code;  in vc_read_mem()
372 	ctxt->fi.cr2 = (unsigned long)src;  in vc_read_mem()
377 static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,  in vc_slow_virt_to_phys()  argument
390 		ctxt->fi.vector     = X86_TRAP_PF;  in vc_slow_virt_to_phys()
391 		ctxt->fi.cr2        = vaddr;  in vc_slow_virt_to_phys()
392 		ctxt->fi.error_code = 0;  in vc_slow_virt_to_phys()
394 		if (user_mode(ctxt->regs))  in vc_slow_virt_to_phys()
395 			ctxt->fi.error_code |= X86_PF_USER;  in vc_slow_virt_to_phys()
528 static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)  in vc_handle_msr()  argument
530 	struct pt_regs *regs = ctxt->regs;  in vc_handle_msr()
535 	exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0;  in vc_handle_msr()
543 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0);  in vc_handle_msr()
691 static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)  in vc_early_forward_exception()  argument
693 	int trapnr = ctxt->fi.vector;  in vc_early_forward_exception()
696 		native_write_cr2(ctxt->fi.cr2);  in vc_early_forward_exception()
698 	ctxt->regs->orig_ax = ctxt->fi.error_code;  in vc_early_forward_exception()
699 	do_early_exception(ctxt->regs, trapnr);  in vc_early_forward_exception()
702 static long *vc_insn_get_reg(struct es_em_ctxt *ctxt)  in vc_insn_get_reg()  argument
707 	reg_array = (long *)ctxt->regs;  in vc_insn_get_reg()
708 	offset    = insn_get_modrm_reg_off(&ctxt->insn, ctxt->regs);  in vc_insn_get_reg()
718 static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)  in vc_insn_get_rm()  argument
723 	reg_array = (long *)ctxt->regs;  in vc_insn_get_rm()
724 	offset    = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);  in vc_insn_get_rm()
733 static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,  in vc_do_mmio()  argument
742 	ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);  in vc_do_mmio()
748 	res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);  in vc_do_mmio()
751 			ctxt->fi.error_code |= X86_PF_WRITE;  in vc_do_mmio()
762 	return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);  in vc_do_mmio()
766 						 struct es_em_ctxt *ctxt)  in vc_handle_mmio_twobyte_ops()  argument
768 	struct insn *insn = &ctxt->insn;  in vc_handle_mmio_twobyte_ops()
783 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio_twobyte_ops()
788 		reg_data = vc_insn_get_reg(ctxt);  in vc_handle_mmio_twobyte_ops()
805 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio_twobyte_ops()
810 		reg_data = vc_insn_get_reg(ctxt);  in vc_handle_mmio_twobyte_ops()
854 static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,  in vc_handle_mmio_movs()  argument
864 	ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);  in vc_handle_mmio_movs()
865 	es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);  in vc_handle_mmio_movs()
868 		ctxt->fi.vector = X86_TRAP_GP;  in vc_handle_mmio_movs()
869 		ctxt->fi.error_code = 0;  in vc_handle_mmio_movs()
873 	src = ds_base + (unsigned char *)ctxt->regs->si;  in vc_handle_mmio_movs()
874 	dst = es_base + (unsigned char *)ctxt->regs->di;  in vc_handle_mmio_movs()
876 	ret = vc_read_mem(ctxt, src, buffer, bytes);  in vc_handle_mmio_movs()
880 	ret = vc_write_mem(ctxt, dst, buffer, bytes);  in vc_handle_mmio_movs()
884 	if (ctxt->regs->flags & X86_EFLAGS_DF)  in vc_handle_mmio_movs()
889 	ctxt->regs->si += off;  in vc_handle_mmio_movs()
890 	ctxt->regs->di += off;  in vc_handle_mmio_movs()
892 	rep = insn_has_rep_prefix(&ctxt->insn);  in vc_handle_mmio_movs()
894 		ctxt->regs->cx -= 1;  in vc_handle_mmio_movs()
896 	if (!rep || ctxt->regs->cx == 0)  in vc_handle_mmio_movs()
903 				     struct es_em_ctxt *ctxt)  in vc_handle_mmio()  argument
905 	struct insn *insn = &ctxt->insn;  in vc_handle_mmio()
919 		reg_data = vc_insn_get_reg(ctxt);  in vc_handle_mmio()
925 		ret = vc_do_mmio(ghcb, ctxt, bytes, false);  in vc_handle_mmio()
937 		ret = vc_do_mmio(ghcb, ctxt, bytes, false);  in vc_handle_mmio()
948 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio()
952 		reg_data = vc_insn_get_reg(ctxt);  in vc_handle_mmio()
971 		ret = vc_handle_mmio_movs(ctxt, bytes);  in vc_handle_mmio()
975 		ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt);  in vc_handle_mmio()
985 					  struct es_em_ctxt *ctxt)  in vc_handle_dr7_write()  argument
988 	long val, *reg = vc_insn_get_rm(ctxt);  in vc_handle_dr7_write()
998 		ctxt->fi.vector = X86_TRAP_GP;  in vc_handle_dr7_write()
999 		ctxt->fi.error_code = 0;  in vc_handle_dr7_write()
1012 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);  in vc_handle_dr7_write()
1023 					 struct es_em_ctxt *ctxt)  in vc_handle_dr7_read()  argument
1026 	long *reg = vc_insn_get_rm(ctxt);  in vc_handle_dr7_read()
1040 				       struct es_em_ctxt *ctxt)  in vc_handle_wbinvd()  argument
1042 	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);  in vc_handle_wbinvd()
1045 static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)  in vc_handle_rdpmc()  argument
1049 	ghcb_set_rcx(ghcb, ctxt->regs->cx);  in vc_handle_rdpmc()
1051 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);  in vc_handle_rdpmc()
1058 	ctxt->regs->ax = ghcb->save.rax;  in vc_handle_rdpmc()
1059 	ctxt->regs->dx = ghcb->save.rdx;  in vc_handle_rdpmc()
1065 					struct es_em_ctxt *ctxt)  in vc_handle_monitor()  argument
1075 				      struct es_em_ctxt *ctxt)  in vc_handle_mwait()  argument
1082 					struct es_em_ctxt *ctxt)  in vc_handle_vmmcall()  argument
1086 	ghcb_set_rax(ghcb, ctxt->regs->ax);  in vc_handle_vmmcall()
1087 	ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);  in vc_handle_vmmcall()
1090 		x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);  in vc_handle_vmmcall()
1092 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);  in vc_handle_vmmcall()
1099 	ctxt->regs->ax = ghcb->save.rax;  in vc_handle_vmmcall()
1107 	    !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))  in vc_handle_vmmcall()
1114 					struct es_em_ctxt *ctxt)  in vc_handle_trap_ac()  argument
1121 	ctxt->fi.vector = X86_TRAP_AC;  in vc_handle_trap_ac()
1122 	ctxt->fi.error_code = 0;  in vc_handle_trap_ac()
1134 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,  in vc_handle_exitcode()  argument
1142 		result = vc_handle_dr7_read(ghcb, ctxt);  in vc_handle_exitcode()
1145 		result = vc_handle_dr7_write(ghcb, ctxt);  in vc_handle_exitcode()
1148 		result = vc_handle_trap_ac(ghcb, ctxt);  in vc_handle_exitcode()
1152 		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);  in vc_handle_exitcode()
1155 		result = vc_handle_rdpmc(ghcb, ctxt);  in vc_handle_exitcode()
1162 		result = vc_handle_cpuid(ghcb, ctxt);  in vc_handle_exitcode()
1165 		result = vc_handle_ioio(ghcb, ctxt);  in vc_handle_exitcode()
1168 		result = vc_handle_msr(ghcb, ctxt);  in vc_handle_exitcode()
1171 		result = vc_handle_vmmcall(ghcb, ctxt);  in vc_handle_exitcode()
1174 		result = vc_handle_wbinvd(ghcb, ctxt);  in vc_handle_exitcode()
1177 		result = vc_handle_monitor(ghcb, ctxt);  in vc_handle_exitcode()
1180 		result = vc_handle_mwait(ghcb, ctxt);  in vc_handle_exitcode()
1183 		result = vc_handle_mmio(ghcb, ctxt);  in vc_handle_exitcode()
1195 static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)  in vc_forward_exception()  argument
1197 	long error_code = ctxt->fi.error_code;  in vc_forward_exception()
1198 	int trapnr = ctxt->fi.vector;  in vc_forward_exception()
1200 	ctxt->regs->orig_ax = ctxt->fi.error_code;  in vc_forward_exception()
1204 		exc_general_protection(ctxt->regs, error_code);  in vc_forward_exception()
1207 		exc_invalid_op(ctxt->regs);  in vc_forward_exception()
1210 		exc_alignment_check(ctxt->regs, error_code);  in vc_forward_exception()
1240 	struct es_em_ctxt ctxt;  in DEFINE_IDTENTRY_VC_SAFE_STACK()  local
1275 	result = vc_init_em_ctxt(&ctxt, regs, error_code);  in DEFINE_IDTENTRY_VC_SAFE_STACK()
1278 		result = vc_handle_exitcode(&ctxt, ghcb, error_code);  in DEFINE_IDTENTRY_VC_SAFE_STACK()
1285 		vc_finish_insn(&ctxt);  in DEFINE_IDTENTRY_VC_SAFE_STACK()
1300 		vc_forward_exception(&ctxt);  in DEFINE_IDTENTRY_VC_SAFE_STACK()
1363 	struct es_em_ctxt ctxt;  in handle_vc_boot_ghcb()  local
1372 	result = vc_init_em_ctxt(&ctxt, regs, exit_code);  in handle_vc_boot_ghcb()
1374 		result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);  in handle_vc_boot_ghcb()
1379 		vc_finish_insn(&ctxt);  in handle_vc_boot_ghcb()
1394 		vc_early_forward_exception(&ctxt);  in handle_vc_boot_ghcb()