37 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
43 static struct ghcb __initdata *boot_ghcb;
47 	struct ghcb ghcb_page;
66 	struct ghcb backup_ghcb;
93 	struct ghcb *ghcb;  member
200 static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)  in __sev_get_ghcb()
203 	struct ghcb *ghcb;  in __sev_get_ghcb()  local
208 	ghcb = &data->ghcb_page;  in __sev_get_ghcb()
231 		state->ghcb = &data->backup_ghcb;  in __sev_get_ghcb()
234 		*state->ghcb = *ghcb;  in __sev_get_ghcb()
236 		state->ghcb = NULL;  in __sev_get_ghcb()
240 	return ghcb;  in __sev_get_ghcb()
460 static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,  in vc_slow_virt_to_phys()  argument
501 	struct ghcb *ghcb;  in __sev_put_ghcb()  local
506 	ghcb = &data->ghcb_page;  in __sev_put_ghcb()
508 	if (state->ghcb) {  in __sev_put_ghcb()
510 		*ghcb = *state->ghcb;  in __sev_put_ghcb()
512 		state->ghcb = NULL;  in __sev_put_ghcb()
518 		vc_ghcb_invalidate(ghcb);  in __sev_put_ghcb()
526 	struct ghcb *ghcb;  in __sev_es_nmi_complete()  local
528 	ghcb = __sev_get_ghcb(&state);  in __sev_es_nmi_complete()
530 	vc_ghcb_invalidate(ghcb);  in __sev_es_nmi_complete()
531 	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);  in __sev_es_nmi_complete()
532 	ghcb_set_sw_exit_info_1(ghcb, 0);  in __sev_es_nmi_complete()
533 	ghcb_set_sw_exit_info_2(ghcb, 0);  in __sev_es_nmi_complete()
535 	sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));  in __sev_es_nmi_complete()
545 	struct ghcb *ghcb;  in get_jump_table_addr()  local
550 	ghcb = __sev_get_ghcb(&state);  in get_jump_table_addr()
552 	vc_ghcb_invalidate(ghcb);  in get_jump_table_addr()
553 	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);  in get_jump_table_addr()
554 	ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);  in get_jump_table_addr()
555 	ghcb_set_sw_exit_info_2(ghcb, 0);  in get_jump_table_addr()
557 	sev_es_wr_ghcb_msr(__pa(ghcb));  in get_jump_table_addr()
560 	if (ghcb_sw_exit_info_1_is_valid(ghcb) &&  in get_jump_table_addr()
561 	    ghcb_sw_exit_info_2_is_valid(ghcb))  in get_jump_table_addr()
562 		ret = ghcb->save.sw_exit_info_2;  in get_jump_table_addr()
636 static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)  in vc_handle_msr()  argument
645 	ghcb_set_rcx(ghcb, regs->cx);  in vc_handle_msr()
647 		ghcb_set_rax(ghcb, regs->ax);  in vc_handle_msr()
648 		ghcb_set_rdx(ghcb, regs->dx);  in vc_handle_msr()
651 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0);  in vc_handle_msr()
654 		regs->ax = ghcb->save.rax;  in vc_handle_msr()
655 		regs->dx = ghcb->save.rdx;  in vc_handle_msr()
687 	struct ghcb *ghcb;  in sev_es_ap_hlt_loop()  local
689 	ghcb = __sev_get_ghcb(&state);  in sev_es_ap_hlt_loop()
692 		vc_ghcb_invalidate(ghcb);  in sev_es_ap_hlt_loop()
693 		ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP);  in sev_es_ap_hlt_loop()
694 		ghcb_set_sw_exit_info_1(ghcb, 0);  in sev_es_ap_hlt_loop()
695 		ghcb_set_sw_exit_info_2(ghcb, 0);  in sev_es_ap_hlt_loop()
697 		sev_es_wr_ghcb_msr(__pa(ghcb));  in sev_es_ap_hlt_loop()
701 		if (ghcb_sw_exit_info_2_is_valid(ghcb) &&  in sev_es_ap_hlt_loop()
702 		    ghcb->save.sw_exit_info_2)  in sev_es_ap_hlt_loop()
841 static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,  in vc_do_mmio()  argument
845 	unsigned long ghcb_pa = __pa(ghcb);  in vc_do_mmio()
856 	res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);  in vc_do_mmio()
868 	ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));  in vc_do_mmio()
870 	return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);  in vc_do_mmio()
873 static enum es_result vc_handle_mmio_twobyte_ops(struct ghcb *ghcb,  in vc_handle_mmio_twobyte_ops()  argument
891 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio_twobyte_ops()
902 		memcpy(reg_data, ghcb->shared_buffer, bytes);  in vc_handle_mmio_twobyte_ops()
913 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio_twobyte_ops()
923 			u8 *val = (u8 *)ghcb->shared_buffer;  in vc_handle_mmio_twobyte_ops()
927 			u16 *val = (u16 *)ghcb->shared_buffer;  in vc_handle_mmio_twobyte_ops()
933 		memcpy(reg_data, ghcb->shared_buffer, bytes);  in vc_handle_mmio_twobyte_ops()
1010 static enum es_result vc_handle_mmio(struct ghcb *ghcb,  in vc_handle_mmio()  argument
1031 		memcpy(ghcb->shared_buffer, reg_data, bytes);  in vc_handle_mmio()
1033 		ret = vc_do_mmio(ghcb, ctxt, bytes, false);  in vc_handle_mmio()
1043 		memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);  in vc_handle_mmio()
1045 		ret = vc_do_mmio(ghcb, ctxt, bytes, false);  in vc_handle_mmio()
1056 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio()
1068 		memcpy(reg_data, ghcb->shared_buffer, bytes);  in vc_handle_mmio()
1083 		ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt);  in vc_handle_mmio()
1092 static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,  in vc_handle_dr7_write()  argument
1119 	ghcb_set_rax(ghcb, val);  in vc_handle_dr7_write()
1120 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);  in vc_handle_dr7_write()
1130 static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,  in vc_handle_dr7_read()  argument
1147 static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,  in vc_handle_wbinvd()  argument
1150 	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);  in vc_handle_wbinvd()
1153 static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)  in vc_handle_rdpmc()  argument
1157 	ghcb_set_rcx(ghcb, ctxt->regs->cx);  in vc_handle_rdpmc()
1159 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);  in vc_handle_rdpmc()
1163 	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))  in vc_handle_rdpmc()
1166 	ctxt->regs->ax = ghcb->save.rax;  in vc_handle_rdpmc()
1167 	ctxt->regs->dx = ghcb->save.rdx;  in vc_handle_rdpmc()
1172 static enum es_result vc_handle_monitor(struct ghcb *ghcb,  in vc_handle_monitor()  argument
1182 static enum es_result vc_handle_mwait(struct ghcb *ghcb,  in vc_handle_mwait()  argument
1189 static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,  in vc_handle_vmmcall()  argument
1194 	ghcb_set_rax(ghcb, ctxt->regs->ax);  in vc_handle_vmmcall()
1195 	ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);  in vc_handle_vmmcall()
1198 		x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);  in vc_handle_vmmcall()
1200 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);  in vc_handle_vmmcall()
1204 	if (!ghcb_rax_is_valid(ghcb))  in vc_handle_vmmcall()
1207 	ctxt->regs->ax = ghcb->save.rax;  in vc_handle_vmmcall()
1215 	    !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))  in vc_handle_vmmcall()
1221 static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,  in vc_handle_trap_ac()  argument
1235 					 struct ghcb *ghcb,  in vc_handle_exitcode()  argument
1242 		result = vc_handle_dr7_read(ghcb, ctxt);  in vc_handle_exitcode()
1245 		result = vc_handle_dr7_write(ghcb, ctxt);  in vc_handle_exitcode()
1248 		result = vc_handle_trap_ac(ghcb, ctxt);  in vc_handle_exitcode()
1252 		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);  in vc_handle_exitcode()
1255 		result = vc_handle_rdpmc(ghcb, ctxt);  in vc_handle_exitcode()
1262 		result = vc_handle_cpuid(ghcb, ctxt);  in vc_handle_exitcode()
1265 		result = vc_handle_ioio(ghcb, ctxt);  in vc_handle_exitcode()
1268 		result = vc_handle_msr(ghcb, ctxt);  in vc_handle_exitcode()
1271 		result = vc_handle_vmmcall(ghcb, ctxt);  in vc_handle_exitcode()
1274 		result = vc_handle_wbinvd(ghcb, ctxt);  in vc_handle_exitcode()
1277 		result = vc_handle_monitor(ghcb, ctxt);  in vc_handle_exitcode()
1280 		result = vc_handle_mwait(ghcb, ctxt);  in vc_handle_exitcode()
1283 		result = vc_handle_mmio(ghcb, ctxt);  in vc_handle_exitcode()
1334 	struct ghcb *ghcb;  in vc_raw_handle_exception()  local
1337 	ghcb = __sev_get_ghcb(&state);  in vc_raw_handle_exception()
1339 	vc_ghcb_invalidate(ghcb);  in vc_raw_handle_exception()
1343 		result = vc_handle_exitcode(&ctxt, ghcb, error_code);  in vc_raw_handle_exception()