168 	 * The kernel can not handle #VE's when accessing normal kernel  in tdx_parse_tdinfo()
169 	 * memory.  Ensure that no #VE will be delivered for accesses to  in tdx_parse_tdinfo()
170 	 * TD-private memory.  Only VMM-shared memory (MMIO) will #VE.  in tdx_parse_tdinfo()
185  * The TDX module spec states that #VE may be injected for a limited set of
188  *  - Emulation of the architectural #VE injection on EPT violation;
202  * information if #VE occurred due to instruction execution, but not for EPT
205 static int ve_instr_len(struct ve_info *ve)  in ve_instr_len()  argument
207 	switch (ve->exit_reason) {  in ve_instr_len()
213 		/* It is safe to use ve->instr_len for #VE due instructions */  in ve_instr_len()
214 		return ve->instr_len;  in ve_instr_len()
217 		 * For EPT violations, ve->insn_len is not defined. For those,  in ve_instr_len()
221 		WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");  in ve_instr_len()
224 		WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);  in ve_instr_len()
225 		return ve->instr_len;  in ve_instr_len()
252 static int handle_halt(struct ve_info *ve)  in handle_halt()  argument
259 	return ve_instr_len(ve);  in handle_halt()
273 static int read_msr(struct pt_regs *regs, struct ve_info *ve)  in read_msr()  argument
291 	return ve_instr_len(ve);  in read_msr()
294 static int write_msr(struct pt_regs *regs, struct ve_info *ve)  in write_msr()  argument
311 	return ve_instr_len(ve);  in write_msr()
314 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)  in handle_cpuid()  argument
332 		return ve_instr_len(ve);  in handle_cpuid()
353 	return ve_instr_len(ve);  in handle_cpuid()
379 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)  in handle_mmio()  argument
425 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
430 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
452 	if (!mmio_read(size, ve->gpa, &val))  in handle_mmio()
532 static int handle_io(struct pt_regs *regs, struct ve_info *ve)  in handle_io()  argument
534 	u32 exit_qual = ve->exit_qual;  in handle_io()
553 	return ve_instr_len(ve);  in handle_io()
557  * Early #VE exception handler. Only handles a subset of port I/O.
562 	struct ve_info ve;  in tdx_early_handle_ve()  local
565 	tdx_get_ve_info(&ve);  in tdx_early_handle_ve()
567 	if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION)  in tdx_early_handle_ve()
570 	insn_len = handle_io(regs, &ve);  in tdx_early_handle_ve()
578 void tdx_get_ve_info(struct ve_info *ve)  in tdx_get_ve_info()  argument
583 	 * Called during #VE handling to retrieve the #VE info from the  in tdx_get_ve_info()
586 	 * This has to be called early in #VE handling.  A "nested" #VE which  in tdx_get_ve_info()
589 	 * The call retrieves the #VE info from the TDX module, which also  in tdx_get_ve_info()
590 	 * clears the "#VE valid" flag. This must be done before anything else  in tdx_get_ve_info()
591 	 * because any #VE that occurs while the valid flag is set will lead to  in tdx_get_ve_info()
594 	 * Note, the TDX module treats virtual NMIs as inhibited if the #VE  in tdx_get_ve_info()
595 	 * valid flag is set. It means that NMI=>#VE will not result in a #DF.  in tdx_get_ve_info()
600 	ve->exit_reason = out.rcx;  in tdx_get_ve_info()
601 	ve->exit_qual   = out.rdx;  in tdx_get_ve_info()
602 	ve->gla         = out.r8;  in tdx_get_ve_info()
603 	ve->gpa         = out.r9;  in tdx_get_ve_info()
604 	ve->instr_len   = lower_32_bits(out.r10);  in tdx_get_ve_info()
605 	ve->instr_info  = upper_32_bits(out.r10);  in tdx_get_ve_info()
609  * Handle the user initiated #VE.
614 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_user()  argument
616 	switch (ve->exit_reason) {  in virt_exception_user()
618 		return handle_cpuid(regs, ve);  in virt_exception_user()
620 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_user()
631  * Handle the kernel #VE.
636 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_kernel()  argument
638 	switch (ve->exit_reason) {  in virt_exception_kernel()
640 		return handle_halt(ve);  in virt_exception_kernel()
642 		return read_msr(regs, ve);  in virt_exception_kernel()
644 		return write_msr(regs, ve);  in virt_exception_kernel()
646 		return handle_cpuid(regs, ve);  in virt_exception_kernel()
648 		if (is_private_gpa(ve->gpa))  in virt_exception_kernel()
650 		return handle_mmio(regs, ve);  in virt_exception_kernel()
652 		return handle_io(regs, ve);  in virt_exception_kernel()
654 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_kernel()
659 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve)  in tdx_handle_virt_exception()  argument
664 		insn_len = virt_exception_user(regs, ve);  in tdx_handle_virt_exception()
666 		insn_len = virt_exception_kernel(regs, ve);  in tdx_handle_virt_exception()
670 	/* After successful #VE handling, move the IP */  in tdx_handle_virt_exception()
794          *   - Shared mapping  => Private Page == Recoverable #VE  in tdx_early_init()
803 	 * which can result in a #VE.  But, there is never a private mapping to  in tdx_early_init()
814 	 * bringup low level code. That raises #VE which cannot be handled  in tdx_early_init()