37 	struct vcpu_svm *svm = to_svm(vcpu);  in nested_svm_inject_npf_exit()  local
39 	if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {  in nested_svm_inject_npf_exit()
44 		svm->vmcb->control.exit_code = SVM_EXIT_NPF;  in nested_svm_inject_npf_exit()
45 		svm->vmcb->control.exit_code_hi = 0;  in nested_svm_inject_npf_exit()
46 		svm->vmcb->control.exit_info_1 = (1ULL << 32);  in nested_svm_inject_npf_exit()
47 		svm->vmcb->control.exit_info_2 = fault->address;  in nested_svm_inject_npf_exit()
50 	svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;  in nested_svm_inject_npf_exit()
51 	svm->vmcb->control.exit_info_1 |= fault->error_code;  in nested_svm_inject_npf_exit()
53 	nested_svm_vmexit(svm);  in nested_svm_inject_npf_exit()
58        struct vcpu_svm *svm = to_svm(vcpu);  in svm_inject_page_fault_nested()  local
61        if (vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) &&  in svm_inject_page_fault_nested()
62 	   !svm->nested.nested_run_pending) {  in svm_inject_page_fault_nested()
63                svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR;  in svm_inject_page_fault_nested()
64                svm->vmcb->control.exit_code_hi = 0;  in svm_inject_page_fault_nested()
65                svm->vmcb->control.exit_info_1 = fault->error_code;  in svm_inject_page_fault_nested()
66                svm->vmcb->control.exit_info_2 = fault->address;  in svm_inject_page_fault_nested()
67                nested_svm_vmexit(svm);  in svm_inject_page_fault_nested()
75 	struct vcpu_svm *svm = to_svm(vcpu);  in nested_svm_get_tdp_pdptr()  local
76 	u64 cr3 = svm->nested.ctl.nested_cr3;  in nested_svm_get_tdp_pdptr()
89 	struct vcpu_svm *svm = to_svm(vcpu);  in nested_svm_get_tdp_cr3()  local
91 	return svm->nested.ctl.nested_cr3;  in nested_svm_get_tdp_cr3()
96 	struct vcpu_svm *svm = to_svm(vcpu);  in nested_svm_init_mmu_context()  local
107 	kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4,  in nested_svm_init_mmu_context()
108 				svm->vmcb01.ptr->save.efer,  in nested_svm_init_mmu_context()
109 				svm->nested.ctl.nested_cr3);  in nested_svm_init_mmu_context()
122 void recalc_intercepts(struct vcpu_svm *svm)  in recalc_intercepts()  argument
127 	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);  in recalc_intercepts()
129 	if (!is_guest_mode(&svm->vcpu))  in recalc_intercepts()
132 	c = &svm->vmcb->control;  in recalc_intercepts()
133 	h = &svm->vmcb01.ptr->control;  in recalc_intercepts()
134 	g = &svm->nested.ctl;  in recalc_intercepts()
197 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)  in nested_svm_vmrun_msrpm()  argument
206 	if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))  in nested_svm_vmrun_msrpm()
217 		offset = svm->nested.ctl.msrpm_base_pa + (p * 4);  in nested_svm_vmrun_msrpm()
219 		if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))  in nested_svm_vmrun_msrpm()
222 		svm->nested.msrpm[p] = svm->msrpm[p] | value;  in nested_svm_vmrun_msrpm()
225 	svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));  in nested_svm_vmrun_msrpm()
314 void nested_load_control_from_vmcb12(struct vcpu_svm *svm,  in nested_load_control_from_vmcb12()  argument
317 	copy_vmcb_control_area(&svm->nested.ctl, control);  in nested_load_control_from_vmcb12()
320 	svm->nested.ctl.asid           = control->asid;  in nested_load_control_from_vmcb12()
321 	svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL;  in nested_load_control_from_vmcb12()
322 	svm->nested.ctl.iopm_base_pa  &= ~0x0fffULL;  in nested_load_control_from_vmcb12()
329 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm)  in nested_sync_control_from_vmcb02()  argument
332 	svm->nested.ctl.event_inj      = svm->vmcb->control.event_inj;  in nested_sync_control_from_vmcb02()
333 	svm->nested.ctl.event_inj_err  = svm->vmcb->control.event_inj_err;  in nested_sync_control_from_vmcb02()
337 	if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) &&  in nested_sync_control_from_vmcb02()
338 	    svm_is_intercept(svm, INTERCEPT_VINTR)) {  in nested_sync_control_from_vmcb02()
349 	svm->nested.ctl.int_ctl        &= ~mask;  in nested_sync_control_from_vmcb02()
350 	svm->nested.ctl.int_ctl        |= svm->vmcb->control.int_ctl & mask;  in nested_sync_control_from_vmcb02()
357 static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm,  in nested_save_pending_event_to_vmcb12()  argument
360 	struct kvm_vcpu *vcpu = &svm->vcpu;  in nested_save_pending_event_to_vmcb12()
390 static inline bool nested_npt_enabled(struct vcpu_svm *svm)  in nested_npt_enabled()  argument
392 	return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE;  in nested_npt_enabled()
440 void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm)  in nested_vmcb02_compute_g_pat()  argument
442 	if (!svm->nested.vmcb02.ptr)  in nested_vmcb02_compute_g_pat()
446 	svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat;  in nested_vmcb02_compute_g_pat()
449 static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12)  in nested_vmcb02_prepare_save()  argument
453 	nested_vmcb02_compute_g_pat(svm);  in nested_vmcb02_prepare_save()
456 	if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) {  in nested_vmcb02_prepare_save()
458 		svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa;  in nested_vmcb02_prepare_save()
462 		svm->vmcb->save.es = vmcb12->save.es;  in nested_vmcb02_prepare_save()
463 		svm->vmcb->save.cs = vmcb12->save.cs;  in nested_vmcb02_prepare_save()
464 		svm->vmcb->save.ss = vmcb12->save.ss;  in nested_vmcb02_prepare_save()
465 		svm->vmcb->save.ds = vmcb12->save.ds;  in nested_vmcb02_prepare_save()
466 		svm->vmcb->save.cpl = vmcb12->save.cpl;  in nested_vmcb02_prepare_save()
467 		vmcb_mark_dirty(svm->vmcb, VMCB_SEG);  in nested_vmcb02_prepare_save()
471 		svm->vmcb->save.gdtr = vmcb12->save.gdtr;  in nested_vmcb02_prepare_save()
472 		svm->vmcb->save.idtr = vmcb12->save.idtr;  in nested_vmcb02_prepare_save()
473 		vmcb_mark_dirty(svm->vmcb, VMCB_DT);  in nested_vmcb02_prepare_save()
476 	kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED);  in nested_vmcb02_prepare_save()
483 	svm_set_efer(&svm->vcpu, vmcb12->save.efer | EFER_SVME);  in nested_vmcb02_prepare_save()
485 	svm_set_cr0(&svm->vcpu, vmcb12->save.cr0);  in nested_vmcb02_prepare_save()
486 	svm_set_cr4(&svm->vcpu, vmcb12->save.cr4);  in nested_vmcb02_prepare_save()
488 	svm->vcpu.arch.cr2 = vmcb12->save.cr2;  in nested_vmcb02_prepare_save()
490 	kvm_rax_write(&svm->vcpu, vmcb12->save.rax);  in nested_vmcb02_prepare_save()
491 	kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp);  in nested_vmcb02_prepare_save()
492 	kvm_rip_write(&svm->vcpu, vmcb12->save.rip);  in nested_vmcb02_prepare_save()
495 	svm->vmcb->save.rax = vmcb12->save.rax;  in nested_vmcb02_prepare_save()
496 	svm->vmcb->save.rsp = vmcb12->save.rsp;  in nested_vmcb02_prepare_save()
497 	svm->vmcb->save.rip = vmcb12->save.rip;  in nested_vmcb02_prepare_save()
501 		svm->vmcb->save.dr7 = vmcb12->save.dr7 | DR7_FIXED_1;  in nested_vmcb02_prepare_save()
502 		svm->vcpu.arch.dr6  = vmcb12->save.dr6 | DR6_ACTIVE_LOW;  in nested_vmcb02_prepare_save()
503 		vmcb_mark_dirty(svm->vmcb, VMCB_DR);  in nested_vmcb02_prepare_save()
507 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)  in nested_vmcb02_prepare_control()  argument
514 	struct kvm_vcpu *vcpu = &svm->vcpu;  in nested_vmcb02_prepare_control()
525 	WARN_ON(kvm_apicv_activated(svm->vcpu.kvm));  in nested_vmcb02_prepare_control()
528 	svm->vmcb->control.nested_ctl = svm->vmcb01.ptr->control.nested_ctl;  in nested_vmcb02_prepare_control()
529 	svm->vmcb->control.iopm_base_pa = svm->vmcb01.ptr->control.iopm_base_pa;  in nested_vmcb02_prepare_control()
530 	svm->vmcb->control.msrpm_base_pa = svm->vmcb01.ptr->control.msrpm_base_pa;  in nested_vmcb02_prepare_control()
535 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;  in nested_vmcb02_prepare_control()
538 	if (nested_npt_enabled(svm))  in nested_vmcb02_prepare_control()
541 	svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset =  in nested_vmcb02_prepare_control()
542 		vcpu->arch.l1_tsc_offset + svm->nested.ctl.tsc_offset;  in nested_vmcb02_prepare_control()
544 	svm->vmcb->control.int_ctl             =  in nested_vmcb02_prepare_control()
545 		(svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) |  in nested_vmcb02_prepare_control()
546 		(svm->vmcb01.ptr->control.int_ctl & int_ctl_vmcb01_bits);  in nested_vmcb02_prepare_control()
548 	svm->vmcb->control.int_vector          = svm->nested.ctl.int_vector;  in nested_vmcb02_prepare_control()
549 	svm->vmcb->control.int_state           = svm->nested.ctl.int_state;  in nested_vmcb02_prepare_control()
550 	svm->vmcb->control.event_inj           = svm->nested.ctl.event_inj;  in nested_vmcb02_prepare_control()
551 	svm->vmcb->control.event_inj_err       = svm->nested.ctl.event_inj_err;  in nested_vmcb02_prepare_control()
553 	svm->vmcb->control.pause_filter_count  = svm->nested.ctl.pause_filter_count;  in nested_vmcb02_prepare_control()
554 	svm->vmcb->control.pause_filter_thresh = svm->nested.ctl.pause_filter_thresh;  in nested_vmcb02_prepare_control()
565 	recalc_intercepts(svm);  in nested_vmcb02_prepare_control()
583 	struct vcpu_svm *svm = to_svm(vcpu);  in enter_svm_guest_mode()  local
586 	trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa,  in enter_svm_guest_mode()
600 	svm->nested.vmcb12_gpa = vmcb12_gpa;  in enter_svm_guest_mode()
602 	WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr);  in enter_svm_guest_mode()
604 	nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr);  in enter_svm_guest_mode()
606 	svm_switch_vmcb(svm, &svm->nested.vmcb02);  in enter_svm_guest_mode()
607 	nested_vmcb02_prepare_control(svm);  in enter_svm_guest_mode()
608 	nested_vmcb02_prepare_save(svm, vmcb12);  in enter_svm_guest_mode()
610 	ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3,  in enter_svm_guest_mode()
611 				  nested_npt_enabled(svm), from_vmrun);  in enter_svm_guest_mode()
621 	svm_set_gif(svm, true);  in enter_svm_guest_mode()
628 	struct vcpu_svm *svm = to_svm(vcpu);  in nested_svm_vmrun()  local
634 	if (!svm->nested.hsave_msr) {  in nested_svm_vmrun()
644 	vmcb12_gpa = svm->vmcb->save.rax;  in nested_svm_vmrun()
657 	if (WARN_ON_ONCE(!svm->nested.initialized))  in nested_svm_vmrun()
660 	nested_load_control_from_vmcb12(svm, &vmcb12->control);  in nested_svm_vmrun()
663 	    !nested_vmcb_check_controls(vcpu, &svm->nested.ctl)) {  in nested_svm_vmrun()
675 	svm->vmcb01.ptr->save.efer   = vcpu->arch.efer;  in nested_svm_vmrun()
676 	svm->vmcb01.ptr->save.cr0    = kvm_read_cr0(vcpu);  in nested_svm_vmrun()
677 	svm->vmcb01.ptr->save.cr4    = vcpu->arch.cr4;  in nested_svm_vmrun()
678 	svm->vmcb01.ptr->save.rflags = kvm_get_rflags(vcpu);  in nested_svm_vmrun()
679 	svm->vmcb01.ptr->save.rip    = kvm_rip_read(vcpu);  in nested_svm_vmrun()
682 		svm->vmcb01.ptr->save.cr3 = kvm_read_cr3(vcpu);  in nested_svm_vmrun()
684 	svm->nested.nested_run_pending = 1;  in nested_svm_vmrun()
689 	if (nested_svm_vmrun_msrpm(svm))  in nested_svm_vmrun()
693 	svm->nested.nested_run_pending = 0;  in nested_svm_vmrun()
695 	svm->vmcb->control.exit_code    = SVM_EXIT_ERR;  in nested_svm_vmrun()
696 	svm->vmcb->control.exit_code_hi = 0;  in nested_svm_vmrun()
697 	svm->vmcb->control.exit_info_1  = 0;  in nested_svm_vmrun()
698 	svm->vmcb->control.exit_info_2  = 0;  in nested_svm_vmrun()
700 	nested_svm_vmexit(svm);  in nested_svm_vmrun()
745 int nested_svm_vmexit(struct vcpu_svm *svm)  in nested_svm_vmexit()  argument
747 	struct kvm_vcpu *vcpu = &svm->vcpu;  in nested_svm_vmexit()
749 	struct vmcb *vmcb = svm->vmcb;  in nested_svm_vmexit()
756 	rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map);  in nested_svm_vmexit()
767 	svm->nested.vmcb12_gpa = 0;  in nested_svm_vmexit()
768 	WARN_ON_ONCE(svm->nested.nested_run_pending);  in nested_svm_vmexit()
773 	svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE;  in nested_svm_vmexit()
783 	vmcb12->save.efer   = svm->vcpu.arch.efer;  in nested_svm_vmexit()
787 	vmcb12->save.cr4    = svm->vcpu.arch.cr4;  in nested_svm_vmexit()
793 	vmcb12->save.dr6    = svm->vcpu.arch.dr6;  in nested_svm_vmexit()
803 		nested_save_pending_event_to_vmcb12(svm, vmcb12);  in nested_svm_vmexit()
805 	if (svm->nrips_enabled)  in nested_svm_vmexit()
808 	vmcb12->control.int_ctl           = svm->nested.ctl.int_ctl;  in nested_svm_vmexit()
809 	vmcb12->control.tlb_ctl           = svm->nested.ctl.tlb_ctl;  in nested_svm_vmexit()
810 	vmcb12->control.event_inj         = svm->nested.ctl.event_inj;  in nested_svm_vmexit()
811 	vmcb12->control.event_inj_err     = svm->nested.ctl.event_inj_err;  in nested_svm_vmexit()
814 		svm->vmcb->control.pause_filter_count;  in nested_svm_vmexit()
816 		svm->vmcb->control.pause_filter_thresh;  in nested_svm_vmexit()
818 	nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr);  in nested_svm_vmexit()
820 	svm_switch_vmcb(svm, &svm->vmcb01);  in nested_svm_vmexit()
826 	svm_set_gif(svm, false);  in nested_svm_vmexit()
827 	svm->vmcb->control.exit_int_info = 0;  in nested_svm_vmexit()
829 	svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset;  in nested_svm_vmexit()
830 	if (svm->vmcb->control.tsc_offset != svm->vcpu.arch.tsc_offset) {  in nested_svm_vmexit()
831 		svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset;  in nested_svm_vmexit()
832 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);  in nested_svm_vmexit()
835 	svm->nested.ctl.nested_cr3 = 0;  in nested_svm_vmexit()
840 	kvm_set_rflags(vcpu, svm->vmcb->save.rflags);  in nested_svm_vmexit()
841 	svm_set_efer(vcpu, svm->vmcb->save.efer);  in nested_svm_vmexit()
842 	svm_set_cr0(vcpu, svm->vmcb->save.cr0 | X86_CR0_PE);  in nested_svm_vmexit()
843 	svm_set_cr4(vcpu, svm->vmcb->save.cr4);  in nested_svm_vmexit()
844 	kvm_rax_write(vcpu, svm->vmcb->save.rax);  in nested_svm_vmexit()
845 	kvm_rsp_write(vcpu, svm->vmcb->save.rsp);  in nested_svm_vmexit()
846 	kvm_rip_write(vcpu, svm->vmcb->save.rip);  in nested_svm_vmexit()
848 	svm->vcpu.arch.dr7 = DR7_FIXED_1;  in nested_svm_vmexit()
849 	kvm_update_dr7(&svm->vcpu);  in nested_svm_vmexit()
864 	rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false, true);  in nested_svm_vmexit()
872 	svm->vcpu.arch.nmi_injected = false;  in nested_svm_vmexit()
882 	if (unlikely(svm->vmcb->save.rflags & X86_EFLAGS_TF))  in nested_svm_vmexit()
883 		kvm_queue_exception(&(svm->vcpu), DB_VECTOR);  in nested_svm_vmexit()
893 int svm_allocate_nested(struct vcpu_svm *svm)  in svm_allocate_nested()  argument
897 	if (svm->nested.initialized)  in svm_allocate_nested()
903 	svm->nested.vmcb02.ptr = page_address(vmcb02_page);  in svm_allocate_nested()
904 	svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT);  in svm_allocate_nested()
906 	svm->nested.msrpm = svm_vcpu_alloc_msrpm();  in svm_allocate_nested()
907 	if (!svm->nested.msrpm)  in svm_allocate_nested()
909 	svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm);  in svm_allocate_nested()
911 	svm->nested.initialized = true;  in svm_allocate_nested()
919 void svm_free_nested(struct vcpu_svm *svm)  in svm_free_nested()  argument
921 	if (!svm->nested.initialized)  in svm_free_nested()
924 	svm_vcpu_free_msrpm(svm->nested.msrpm);  in svm_free_nested()
925 	svm->nested.msrpm = NULL;  in svm_free_nested()
927 	__free_page(virt_to_page(svm->nested.vmcb02.ptr));  in svm_free_nested()
928 	svm->nested.vmcb02.ptr = NULL;  in svm_free_nested()
937 	svm->nested.last_vmcb12_gpa = INVALID_GPA;  in svm_free_nested()
939 	svm->nested.initialized = false;  in svm_free_nested()
945 void svm_leave_nested(struct vcpu_svm *svm)  in svm_leave_nested()  argument
947 	struct kvm_vcpu *vcpu = &svm->vcpu;  in svm_leave_nested()
950 		svm->nested.nested_run_pending = 0;  in svm_leave_nested()
951 		svm->nested.vmcb12_gpa = INVALID_GPA;  in svm_leave_nested()
955 		svm_switch_vmcb(svm, &svm->vmcb01);  in svm_leave_nested()
958 		vmcb_mark_all_dirty(svm->vmcb);  in svm_leave_nested()
964 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)  in nested_svm_exit_handled_msr()  argument
969 	if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))  in nested_svm_exit_handled_msr()
972 	msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];  in nested_svm_exit_handled_msr()
974 	write  = svm->vmcb->control.exit_info_1 & 1;  in nested_svm_exit_handled_msr()
983 	if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4))  in nested_svm_exit_handled_msr()
989 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)  in nested_svm_intercept_ioio()  argument
996 	if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT)))  in nested_svm_intercept_ioio()
999 	port = svm->vmcb->control.exit_info_1 >> 16;  in nested_svm_intercept_ioio()
1000 	size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>  in nested_svm_intercept_ioio()
1002 	gpa  = svm->nested.ctl.iopm_base_pa + (port / 8);  in nested_svm_intercept_ioio()
1008 	if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))  in nested_svm_intercept_ioio()
1014 static int nested_svm_intercept(struct vcpu_svm *svm)  in nested_svm_intercept()  argument
1016 	u32 exit_code = svm->vmcb->control.exit_code;  in nested_svm_intercept()
1021 		vmexit = nested_svm_exit_handled_msr(svm);  in nested_svm_intercept()
1024 		vmexit = nested_svm_intercept_ioio(svm);  in nested_svm_intercept()
1027 		if (vmcb_is_intercept(&svm->nested.ctl, exit_code))  in nested_svm_intercept()
1032 		if (vmcb_is_intercept(&svm->nested.ctl, exit_code))  in nested_svm_intercept()
1050 		if (vmcb_is_intercept(&svm->nested.ctl, exit_code))  in nested_svm_intercept()
1058 int nested_svm_exit_handled(struct vcpu_svm *svm)  in nested_svm_exit_handled()  argument
1062 	vmexit = nested_svm_intercept(svm);  in nested_svm_exit_handled()
1065 		nested_svm_vmexit(svm);  in nested_svm_exit_handled()
1085 static bool nested_exit_on_exception(struct vcpu_svm *svm)  in nested_exit_on_exception()  argument
1087 	unsigned int nr = svm->vcpu.arch.exception.nr;  in nested_exit_on_exception()
1089 	return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr));  in nested_exit_on_exception()
1092 static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm)  in nested_svm_inject_exception_vmexit()  argument
1094 	unsigned int nr = svm->vcpu.arch.exception.nr;  in nested_svm_inject_exception_vmexit()
1096 	svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;  in nested_svm_inject_exception_vmexit()
1097 	svm->vmcb->control.exit_code_hi = 0;  in nested_svm_inject_exception_vmexit()
1099 	if (svm->vcpu.arch.exception.has_error_code)  in nested_svm_inject_exception_vmexit()
1100 		svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code;  in nested_svm_inject_exception_vmexit()
1107 		if (svm->vcpu.arch.exception.nested_apf)  in nested_svm_inject_exception_vmexit()
1108 			svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;  in nested_svm_inject_exception_vmexit()
1109 		else if (svm->vcpu.arch.exception.has_payload)  in nested_svm_inject_exception_vmexit()
1110 			svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload;  in nested_svm_inject_exception_vmexit()
1112 			svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;  in nested_svm_inject_exception_vmexit()
1115 		kvm_deliver_exception_payload(&svm->vcpu);  in nested_svm_inject_exception_vmexit()
1116 		if (svm->vcpu.arch.dr7 & DR7_GD) {  in nested_svm_inject_exception_vmexit()
1117 			svm->vcpu.arch.dr7 &= ~DR7_GD;  in nested_svm_inject_exception_vmexit()
1118 			kvm_update_dr7(&svm->vcpu);  in nested_svm_inject_exception_vmexit()
1121 		WARN_ON(svm->vcpu.arch.exception.has_payload);  in nested_svm_inject_exception_vmexit()
1123 	nested_svm_vmexit(svm);  in nested_svm_inject_exception_vmexit()
1126 static inline bool nested_exit_on_init(struct vcpu_svm *svm)  in nested_exit_on_init()  argument
1128 	return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_INIT);  in nested_exit_on_init()
1133 	struct vcpu_svm *svm = to_svm(vcpu);  in svm_check_nested_events()  local
1135 		kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending;  in svm_check_nested_events()
1142 		if (!nested_exit_on_init(svm))  in svm_check_nested_events()
1144 		nested_svm_simple_vmexit(svm, SVM_EXIT_INIT);  in svm_check_nested_events()
1155 		if (svm->nested.nested_run_pending)  in svm_check_nested_events()
1157 		if (!nested_exit_on_exception(svm))  in svm_check_nested_events()
1159 		nested_svm_inject_exception_vmexit(svm);  in svm_check_nested_events()
1166 		if (!nested_exit_on_smi(svm))  in svm_check_nested_events()
1168 		nested_svm_simple_vmexit(svm, SVM_EXIT_SMI);  in svm_check_nested_events()
1175 		if (!nested_exit_on_nmi(svm))  in svm_check_nested_events()
1177 		nested_svm_simple_vmexit(svm, SVM_EXIT_NMI);  in svm_check_nested_events()
1184 		if (!nested_exit_on_intr(svm))  in svm_check_nested_events()
1186 		trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);  in svm_check_nested_events()
1187 		nested_svm_simple_vmexit(svm, SVM_EXIT_INTR);  in svm_check_nested_events()
1194 int nested_svm_exit_special(struct vcpu_svm *svm)  in nested_svm_exit_special()  argument
1196 	u32 exit_code = svm->vmcb->control.exit_code;  in nested_svm_exit_special()
1206 		if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &  in nested_svm_exit_special()
1210 			 svm->vcpu.arch.apf.host_apf_flags)  in nested_svm_exit_special()
1226 	struct vcpu_svm *svm;  in svm_get_nested_state()  local
1233 		&user_kvm_nested_state->data.svm[0];  in svm_get_nested_state()
1238 	svm = to_svm(vcpu);  in svm_get_nested_state()
1245 		kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa;  in svm_get_nested_state()
1249 		if (svm->nested.nested_run_pending)  in svm_get_nested_state()
1253 	if (gif_set(svm))  in svm_get_nested_state()
1268 	if (copy_to_user(&user_vmcb->control, &svm->nested.ctl,  in svm_get_nested_state()
1271 	if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save,  in svm_get_nested_state()
1282 	struct vcpu_svm *svm = to_svm(vcpu);  in svm_set_nested_state()  local
1284 		&user_kvm_nested_state->data.svm[0];  in svm_set_nested_state()
1316 		svm_leave_nested(svm);  in svm_set_nested_state()
1317 		svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));  in svm_set_nested_state()
1321 	if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa))  in svm_set_nested_state()
1367 	ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3,  in svm_set_nested_state()
1368 				  nested_npt_enabled(svm), false);  in svm_set_nested_state()
1381 		svm_leave_nested(svm);  in svm_set_nested_state()
1383 		svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save;  in svm_set_nested_state()
1385 	svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));  in svm_set_nested_state()
1387 	svm->nested.nested_run_pending =  in svm_set_nested_state()
1390 	svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa;  in svm_set_nested_state()
1392 	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save);  in svm_set_nested_state()
1393 	nested_load_control_from_vmcb12(svm, ctl);  in svm_set_nested_state()
1395 	svm_switch_vmcb(svm, &svm->nested.vmcb02);  in svm_set_nested_state()
1396 	nested_vmcb02_prepare_control(svm);  in svm_set_nested_state()
1408 	struct vcpu_svm *svm = to_svm(vcpu);  in svm_get_nested_state_pages()  local
1414 	    !nested_npt_enabled(svm) && is_pae_paging(vcpu))  in svm_get_nested_state_pages()
1423 	if (!nested_svm_vmrun_msrpm(svm)) {  in svm_get_nested_state_pages()