29 static bool __hyp_text update_fp_enabled(struct kvm_vcpu *vcpu)  in update_fp_enabled()  argument
31 	if (vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE)  in update_fp_enabled()
32 		vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |  in update_fp_enabled()
35 	return !!(vcpu->arch.flags & KVM_ARM64_FP_ENABLED);  in update_fp_enabled()
39 static void __hyp_text __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu)  in __fpsimd_save_fpexc32()  argument
41 	if (!vcpu_el1_is_32bit(vcpu))  in __fpsimd_save_fpexc32()
44 	vcpu->arch.ctxt.sys_regs[FPEXC32_EL2] = read_sysreg(fpexc32_el2);  in __fpsimd_save_fpexc32()
47 static void __hyp_text __activate_traps_fpsimd32(struct kvm_vcpu *vcpu)  in __activate_traps_fpsimd32()  argument
58 	if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) {  in __activate_traps_fpsimd32()
64 static void __hyp_text __activate_traps_common(struct kvm_vcpu *vcpu)  in __activate_traps_common()  argument
77 	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);  in __activate_traps_common()
86 static void activate_traps_vhe(struct kvm_vcpu *vcpu)  in activate_traps_vhe()  argument
93 	if (update_fp_enabled(vcpu)) {  in activate_traps_vhe()
94 		if (vcpu_has_sve(vcpu))  in activate_traps_vhe()
98 		__activate_traps_fpsimd32(vcpu);  in activate_traps_vhe()
107 static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu)  in __activate_traps_nvhe()  argument
111 	__activate_traps_common(vcpu);  in __activate_traps_nvhe()
115 	if (!update_fp_enabled(vcpu)) {  in __activate_traps_nvhe()
117 		__activate_traps_fpsimd32(vcpu);  in __activate_traps_nvhe()
123 static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)  in __activate_traps()  argument
125 	u64 hcr = vcpu->arch.hcr_el2;  in __activate_traps()
133 		write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);  in __activate_traps()
136 		activate_traps_vhe(vcpu);  in __activate_traps()
138 		__activate_traps_nvhe(vcpu);  in __activate_traps()
172 static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)  in __deactivate_traps()  argument
180 	if (vcpu->arch.hcr_el2 & HCR_VSE) {  in __deactivate_traps()
181 		vcpu->arch.hcr_el2 &= ~HCR_VSE;  in __deactivate_traps()
182 		vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE;  in __deactivate_traps()
191 void activate_traps_vhe_load(struct kvm_vcpu *vcpu)  in activate_traps_vhe_load()  argument
193 	__activate_traps_common(vcpu);  in activate_traps_vhe_load()
214 static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)  in __deactivate_vm()  argument
220 static void __hyp_text __hyp_vgic_save_state(struct kvm_vcpu *vcpu)  in __hyp_vgic_save_state()  argument
223 		__vgic_v3_save_state(vcpu);  in __hyp_vgic_save_state()
224 		__vgic_v3_deactivate_traps(vcpu);  in __hyp_vgic_save_state()
229 static void __hyp_text __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)  in __hyp_vgic_restore_state()  argument
232 		__vgic_v3_activate_traps(vcpu);  in __hyp_vgic_restore_state()
233 		__vgic_v3_restore_state(vcpu);  in __hyp_vgic_restore_state()
266 static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)  in __populate_fault_info()  argument
272 	esr = vcpu->arch.fault.esr_el2;  in __populate_fault_info()
300 	vcpu->arch.fault.far_el2 = far;  in __populate_fault_info()
301 	vcpu->arch.fault.hpfar_el2 = hpfar;  in __populate_fault_info()
306 static bool __hyp_text __hyp_handle_fpsimd(struct kvm_vcpu *vcpu)  in __hyp_handle_fpsimd()  argument
315 		sve_guest = vcpu_has_sve(vcpu);  in __hyp_handle_fpsimd()
316 		sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE;  in __hyp_handle_fpsimd()
324 	hsr_ec = kvm_vcpu_trap_get_class(vcpu);  in __hyp_handle_fpsimd()
350 	if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {  in __hyp_handle_fpsimd()
357 				vcpu->arch.host_fpsimd_state,  in __hyp_handle_fpsimd()
361 				       &vcpu->arch.host_fpsimd_state->fpsr);  in __hyp_handle_fpsimd()
363 			__fpsimd_save_state(vcpu->arch.host_fpsimd_state);  in __hyp_handle_fpsimd()
366 		vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;  in __hyp_handle_fpsimd()
370 		sve_load_state(vcpu_sve_pffr(vcpu),  in __hyp_handle_fpsimd()
371 			       &vcpu->arch.ctxt.gp_regs.fp_regs.fpsr,  in __hyp_handle_fpsimd()
372 			       sve_vq_from_vl(vcpu->arch.sve_max_vl) - 1);  in __hyp_handle_fpsimd()
373 		write_sysreg_s(vcpu->arch.ctxt.sys_regs[ZCR_EL1], SYS_ZCR_EL12);  in __hyp_handle_fpsimd()
375 		__fpsimd_restore_state(&vcpu->arch.ctxt.gp_regs.fp_regs);  in __hyp_handle_fpsimd()
380 		write_sysreg(vcpu->arch.ctxt.sys_regs[FPEXC32_EL2],  in __hyp_handle_fpsimd()
383 	vcpu->arch.flags |= KVM_ARM64_FP_ENABLED;  in __hyp_handle_fpsimd()
388 static bool __hyp_text handle_tx2_tvm(struct kvm_vcpu *vcpu)  in handle_tx2_tvm()  argument
390 	u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_hsr(vcpu));  in handle_tx2_tvm()
391 	int rt = kvm_vcpu_sys_get_rt(vcpu);  in handle_tx2_tvm()
392 	u64 val = vcpu_get_reg(vcpu, rt);  in handle_tx2_tvm()
398 	if (vcpu->arch.hcr_el2 & HCR_TVM)  in handle_tx2_tvm()
439 	__kvm_skip_instr(vcpu);  in handle_tx2_tvm()
448 static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)  in fixup_guest_exit()  argument
451 		vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR);  in fixup_guest_exit()
463 	    kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 &&  in fixup_guest_exit()
464 	    handle_tx2_tvm(vcpu))  in fixup_guest_exit()
474 	if (__hyp_handle_fpsimd(vcpu))  in fixup_guest_exit()
477 	if (!__populate_fault_info(vcpu))  in fixup_guest_exit()
483 		valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&  in fixup_guest_exit()
484 			kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&  in fixup_guest_exit()
485 			kvm_vcpu_dabt_isvalid(vcpu) &&  in fixup_guest_exit()
486 			!kvm_vcpu_dabt_isextabt(vcpu) &&  in fixup_guest_exit()
487 			!kvm_vcpu_dabt_iss1tw(vcpu);  in fixup_guest_exit()
490 			int ret = __vgic_v2_perform_cpuif_access(vcpu);  in fixup_guest_exit()
504 	    (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||  in fixup_guest_exit()
505 	     kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {  in fixup_guest_exit()
506 		int ret = __vgic_v3_perform_cpuif_access(vcpu);  in fixup_guest_exit()
517 static inline bool __hyp_text __needs_ssbd_off(struct kvm_vcpu *vcpu)  in __needs_ssbd_off()  argument
522 	return !(vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG);  in __needs_ssbd_off()
525 static void __hyp_text __set_guest_arch_workaround_state(struct kvm_vcpu *vcpu)  in __set_guest_arch_workaround_state()  argument
532 	if (__needs_ssbd_off(vcpu) &&  in __set_guest_arch_workaround_state()
538 static void __hyp_text __set_host_arch_workaround_state(struct kvm_vcpu *vcpu)  in __set_host_arch_workaround_state()  argument
544 	if (__needs_ssbd_off(vcpu) &&  in __set_host_arch_workaround_state()
589 int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)  in kvm_vcpu_run_vhe()  argument
595 	host_ctxt = vcpu->arch.host_cpu_context;  in kvm_vcpu_run_vhe()
596 	host_ctxt->__hyp_running_vcpu = vcpu;  in kvm_vcpu_run_vhe()
597 	guest_ctxt = &vcpu->arch.ctxt;  in kvm_vcpu_run_vhe()
612 	__activate_vm(vcpu->kvm);  in kvm_vcpu_run_vhe()
613 	__activate_traps(vcpu);  in kvm_vcpu_run_vhe()
616 	__debug_switch_to_guest(vcpu);  in kvm_vcpu_run_vhe()
618 	__set_guest_arch_workaround_state(vcpu);  in kvm_vcpu_run_vhe()
622 		exit_code = __guest_enter(vcpu, host_ctxt);  in kvm_vcpu_run_vhe()
625 	} while (fixup_guest_exit(vcpu, &exit_code));  in kvm_vcpu_run_vhe()
627 	__set_host_arch_workaround_state(vcpu);  in kvm_vcpu_run_vhe()
631 	__deactivate_traps(vcpu);  in kvm_vcpu_run_vhe()
635 	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)  in kvm_vcpu_run_vhe()
636 		__fpsimd_save_fpexc32(vcpu);  in kvm_vcpu_run_vhe()
638 	__debug_switch_to_host(vcpu);  in kvm_vcpu_run_vhe()
645 int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)  in __kvm_vcpu_run_nvhe()  argument
663 	vcpu = kern_hyp_va(vcpu);  in __kvm_vcpu_run_nvhe()
665 	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);  in __kvm_vcpu_run_nvhe()
666 	host_ctxt->__hyp_running_vcpu = vcpu;  in __kvm_vcpu_run_nvhe()
667 	guest_ctxt = &vcpu->arch.ctxt;  in __kvm_vcpu_run_nvhe()
673 	__activate_vm(kern_hyp_va(vcpu->kvm));  in __kvm_vcpu_run_nvhe()
674 	__activate_traps(vcpu);  in __kvm_vcpu_run_nvhe()
676 	__hyp_vgic_restore_state(vcpu);  in __kvm_vcpu_run_nvhe()
677 	__timer_enable_traps(vcpu);  in __kvm_vcpu_run_nvhe()
683 	__sysreg32_restore_state(vcpu);  in __kvm_vcpu_run_nvhe()
685 	__debug_switch_to_guest(vcpu);  in __kvm_vcpu_run_nvhe()
687 	__set_guest_arch_workaround_state(vcpu);  in __kvm_vcpu_run_nvhe()
691 		exit_code = __guest_enter(vcpu, host_ctxt);  in __kvm_vcpu_run_nvhe()
694 	} while (fixup_guest_exit(vcpu, &exit_code));  in __kvm_vcpu_run_nvhe()
696 	__set_host_arch_workaround_state(vcpu);  in __kvm_vcpu_run_nvhe()
699 	__sysreg32_save_state(vcpu);  in __kvm_vcpu_run_nvhe()
700 	__timer_disable_traps(vcpu);  in __kvm_vcpu_run_nvhe()
701 	__hyp_vgic_save_state(vcpu);  in __kvm_vcpu_run_nvhe()
703 	__deactivate_traps(vcpu);  in __kvm_vcpu_run_nvhe()
704 	__deactivate_vm(vcpu);  in __kvm_vcpu_run_nvhe()
708 	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)  in __kvm_vcpu_run_nvhe()
709 		__fpsimd_save_fpexc32(vcpu);  in __kvm_vcpu_run_nvhe()
715 	__debug_switch_to_host(vcpu);  in __kvm_vcpu_run_nvhe()
732 	struct kvm_vcpu *vcpu;  in __hyp_call_panic_nvhe()  local
735 	vcpu = __host_ctxt->__hyp_running_vcpu;  in __hyp_call_panic_nvhe()
738 		__timer_disable_traps(vcpu);  in __hyp_call_panic_nvhe()
739 		__deactivate_traps(vcpu);  in __hyp_call_panic_nvhe()
740 		__deactivate_vm(vcpu);  in __hyp_call_panic_nvhe()
754 		       read_sysreg(hpfar_el2), par, vcpu);  in __hyp_call_panic_nvhe()
760 	struct kvm_vcpu *vcpu;  in __hyp_call_panic_vhe()  local
761 	vcpu = host_ctxt->__hyp_running_vcpu;  in __hyp_call_panic_vhe()
763 	__deactivate_traps(vcpu);  in __hyp_call_panic_vhe()
769 	      read_sysreg(hpfar_el2), par, vcpu);  in __hyp_call_panic_vhe()