88 	((struct kvm_vcpu *)(ctxt)->vcpu)
106 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
107 static void process_nmi(struct kvm_vcpu *vcpu);
108 static void enter_smm(struct kvm_vcpu *vcpu);
109 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
110 static void store_regs(struct kvm_vcpu *vcpu);
111 static int sync_regs(struct kvm_vcpu *vcpu);
260 static bool kvm_msr_ignored_check(struct kvm_vcpu *vcpu, u32 msr,  in kvm_msr_ignored_check()  argument
291 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)  in kvm_async_pf_hash_reset()  argument
295 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_async_pf_hash_reset()
380 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)  in kvm_get_apic_base()  argument
382 	return vcpu->arch.apic_base;  in kvm_get_apic_base()
386 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)  in kvm_get_apic_mode()  argument
388 	return kvm_apic_mode(kvm_get_apic_base(vcpu));  in kvm_get_apic_mode()
392 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_apic_base()  argument
394 	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);  in kvm_set_apic_base()
396 	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |  in kvm_set_apic_base()
397 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);  in kvm_set_apic_base()
408 	kvm_lapic_set_base(vcpu, msr_info->data);  in kvm_set_apic_base()
409 	kvm_recalculate_apic_map(vcpu->kvm);  in kvm_set_apic_base()
467 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)  in kvm_deliver_exception_payload()  argument
469 	unsigned nr = vcpu->arch.exception.nr;  in kvm_deliver_exception_payload()
470 	bool has_payload = vcpu->arch.exception.has_payload;  in kvm_deliver_exception_payload()
471 	unsigned long payload = vcpu->arch.exception.payload;  in kvm_deliver_exception_payload()
483 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;  in kvm_deliver_exception_payload()
487 		vcpu->arch.dr6 |= DR6_RTM;  in kvm_deliver_exception_payload()
488 		vcpu->arch.dr6 |= payload;  in kvm_deliver_exception_payload()
497 		vcpu->arch.dr6 ^= payload & DR6_RTM;  in kvm_deliver_exception_payload()
505 		vcpu->arch.dr6 &= ~BIT(12);  in kvm_deliver_exception_payload()
508 		vcpu->arch.cr2 = payload;  in kvm_deliver_exception_payload()
512 	vcpu->arch.exception.has_payload = false;  in kvm_deliver_exception_payload()
513 	vcpu->arch.exception.payload = 0;  in kvm_deliver_exception_payload()
517 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,  in kvm_multiple_exception()  argument
524 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_multiple_exception()
526 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {  in kvm_multiple_exception()
528 		if (has_error && !is_protmode(vcpu))  in kvm_multiple_exception()
539 			WARN_ON_ONCE(vcpu->arch.exception.pending);  in kvm_multiple_exception()
540 			vcpu->arch.exception.injected = true;  in kvm_multiple_exception()
550 			vcpu->arch.exception.pending = true;  in kvm_multiple_exception()
551 			vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
553 		vcpu->arch.exception.has_error_code = has_error;  in kvm_multiple_exception()
554 		vcpu->arch.exception.nr = nr;  in kvm_multiple_exception()
555 		vcpu->arch.exception.error_code = error_code;  in kvm_multiple_exception()
556 		vcpu->arch.exception.has_payload = has_payload;  in kvm_multiple_exception()
557 		vcpu->arch.exception.payload = payload;  in kvm_multiple_exception()
558 		if (!is_guest_mode(vcpu))  in kvm_multiple_exception()
559 			kvm_deliver_exception_payload(vcpu);  in kvm_multiple_exception()
564 	prev_nr = vcpu->arch.exception.nr;  in kvm_multiple_exception()
567 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_multiple_exception()
579 		vcpu->arch.exception.pending = true;  in kvm_multiple_exception()
580 		vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
581 		vcpu->arch.exception.has_error_code = true;  in kvm_multiple_exception()
582 		vcpu->arch.exception.nr = DF_VECTOR;  in kvm_multiple_exception()
583 		vcpu->arch.exception.error_code = 0;  in kvm_multiple_exception()
584 		vcpu->arch.exception.has_payload = false;  in kvm_multiple_exception()
585 		vcpu->arch.exception.payload = 0;  in kvm_multiple_exception()
593 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_queue_exception()  argument
595 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);  in kvm_queue_exception()
599 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_requeue_exception()  argument
601 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);  in kvm_requeue_exception()
605 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_p()  argument
608 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);  in kvm_queue_exception_p()
612 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_e_p()  argument
615 	kvm_multiple_exception(vcpu, nr, true, error_code,  in kvm_queue_exception_e_p()
619 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)  in kvm_complete_insn_gp()  argument
622 		kvm_inject_gp(vcpu, 0);  in kvm_complete_insn_gp()
624 		return kvm_skip_emulated_instruction(vcpu);  in kvm_complete_insn_gp()
630 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)  in kvm_inject_page_fault()  argument
632 	++vcpu->stat.pf_guest;  in kvm_inject_page_fault()
633 	vcpu->arch.exception.nested_apf =  in kvm_inject_page_fault()
634 		is_guest_mode(vcpu) && fault->async_page_fault;  in kvm_inject_page_fault()
635 	if (vcpu->arch.exception.nested_apf) {  in kvm_inject_page_fault()
636 		vcpu->arch.apf.nested_apf_token = fault->address;  in kvm_inject_page_fault()
637 		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);  in kvm_inject_page_fault()
639 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,  in kvm_inject_page_fault()
645 bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,  in kvm_inject_emulated_page_fault()  argument
651 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :  in kvm_inject_emulated_page_fault()
652 					       vcpu->arch.walk_mmu;  in kvm_inject_emulated_page_fault()
660 		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,  in kvm_inject_emulated_page_fault()
663 	fault_mmu->inject_page_fault(vcpu, fault);  in kvm_inject_emulated_page_fault()
668 void kvm_inject_nmi(struct kvm_vcpu *vcpu)  in kvm_inject_nmi()  argument
670 	atomic_inc(&vcpu->arch.nmi_queued);  in kvm_inject_nmi()
671 	kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_inject_nmi()
675 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_queue_exception_e()  argument
677 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);  in kvm_queue_exception_e()
681 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_requeue_exception_e()  argument
683 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);  in kvm_requeue_exception_e()
691 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)  in kvm_require_cpl()  argument
693 	if (kvm_x86_ops.get_cpl(vcpu) <= required_cpl)  in kvm_require_cpl()
695 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in kvm_require_cpl()
700 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)  in kvm_require_dr()  argument
702 	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))  in kvm_require_dr()
705 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_require_dr()
715 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,  in kvm_read_guest_page_mmu()  argument
724 	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);  in kvm_read_guest_page_mmu()
730 	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);  in kvm_read_guest_page_mmu()
734 static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,  in kvm_read_nested_guest_page()  argument
737 	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,  in kvm_read_nested_guest_page()
741 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)  in pdptr_rsvd_bits()  argument
743 	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |  in pdptr_rsvd_bits()
750 int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)  in load_pdptrs()  argument
758 	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,  in load_pdptrs()
767 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {  in load_pdptrs()
775 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in load_pdptrs()
783 bool pdptrs_changed(struct kvm_vcpu *vcpu)  in pdptrs_changed()  argument
785 	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];  in pdptrs_changed()
790 	if (!is_pae_paging(vcpu))  in pdptrs_changed()
793 	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))  in pdptrs_changed()
796 	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;  in pdptrs_changed()
797 	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);  in pdptrs_changed()
798 	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),  in pdptrs_changed()
803 	return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;  in pdptrs_changed()
807 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_set_cr0()  argument
809 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_set_cr0()
829 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&  in kvm_set_cr0()
833 		if (!is_pae(vcpu))  in kvm_set_cr0()
835 		kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);  in kvm_set_cr0()
840 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&  in kvm_set_cr0()
841 	    is_pae(vcpu) && ((cr0 ^ old_cr0) & pdptr_bits) &&  in kvm_set_cr0()
842 	    !load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)))  in kvm_set_cr0()
845 	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))  in kvm_set_cr0()
848 	kvm_x86_ops.set_cr0(vcpu, cr0);  in kvm_set_cr0()
851 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_set_cr0()
852 		kvm_async_pf_hash_reset(vcpu);  in kvm_set_cr0()
856 		kvm_mmu_reset_context(vcpu);  in kvm_set_cr0()
859 	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&  in kvm_set_cr0()
860 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))  in kvm_set_cr0()
861 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);  in kvm_set_cr0()
867 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)  in kvm_lmsw()  argument
869 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));  in kvm_lmsw()
873 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_guest_xsave_state()  argument
875 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_guest_xsave_state()
877 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_guest_xsave_state()
878 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);  in kvm_load_guest_xsave_state()
880 		if (vcpu->arch.xsaves_enabled &&  in kvm_load_guest_xsave_state()
881 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_guest_xsave_state()
882 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);  in kvm_load_guest_xsave_state()
886 	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||  in kvm_load_guest_xsave_state()
887 	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) &&  in kvm_load_guest_xsave_state()
888 	    vcpu->arch.pkru != vcpu->arch.host_pkru)  in kvm_load_guest_xsave_state()
889 		__write_pkru(vcpu->arch.pkru);  in kvm_load_guest_xsave_state()
893 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_host_xsave_state()  argument
896 	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||  in kvm_load_host_xsave_state()
897 	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) {  in kvm_load_host_xsave_state()
898 		vcpu->arch.pkru = rdpkru();  in kvm_load_host_xsave_state()
899 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)  in kvm_load_host_xsave_state()
900 			__write_pkru(vcpu->arch.host_pkru);  in kvm_load_host_xsave_state()
903 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_host_xsave_state()
905 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_host_xsave_state()
908 		if (vcpu->arch.xsaves_enabled &&  in kvm_load_host_xsave_state()
909 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_host_xsave_state()
916 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)  in __kvm_set_xcr()  argument
919 	u64 old_xcr0 = vcpu->arch.xcr0;  in __kvm_set_xcr()
935 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;  in __kvm_set_xcr()
949 	vcpu->arch.xcr0 = xcr0;  in __kvm_set_xcr()
952 		kvm_update_cpuid_runtime(vcpu);  in __kvm_set_xcr()
956 int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)  in kvm_set_xcr()  argument
958 	if (kvm_x86_ops.get_cpl(vcpu) != 0 ||  in kvm_set_xcr()
959 	    __kvm_set_xcr(vcpu, index, xcr)) {  in kvm_set_xcr()
960 		kvm_inject_gp(vcpu, 0);  in kvm_set_xcr()
967 int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_valid_cr4()  argument
972 	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)  in kvm_valid_cr4()
979 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_set_cr4()  argument
981 	unsigned long old_cr4 = kvm_read_cr4(vcpu);  in kvm_set_cr4()
986 	if (kvm_valid_cr4(vcpu, cr4))  in kvm_set_cr4()
989 	if (is_long_mode(vcpu)) {  in kvm_set_cr4()
994 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)  in kvm_set_cr4()
996 		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,  in kvm_set_cr4()
997 				   kvm_read_cr3(vcpu)))  in kvm_set_cr4()
1001 		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))  in kvm_set_cr4()
1005 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))  in kvm_set_cr4()
1009 	if (kvm_x86_ops.set_cr4(vcpu, cr4))  in kvm_set_cr4()
1014 		kvm_mmu_reset_context(vcpu);  in kvm_set_cr4()
1017 		kvm_update_cpuid_runtime(vcpu);  in kvm_set_cr4()
1023 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)  in kvm_set_cr3()  argument
1027 	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);  in kvm_set_cr3()
1035 	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {  in kvm_set_cr3()
1037 			kvm_mmu_sync_roots(vcpu);  in kvm_set_cr3()
1038 			kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_set_cr3()
1043 	if (is_long_mode(vcpu) &&  in kvm_set_cr3()
1044 	    (cr3 & vcpu->arch.cr3_lm_rsvd_bits))  in kvm_set_cr3()
1046 	else if (is_pae_paging(vcpu) &&  in kvm_set_cr3()
1047 		 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))  in kvm_set_cr3()
1050 	kvm_mmu_new_pgd(vcpu, cr3, skip_tlb_flush, skip_tlb_flush);  in kvm_set_cr3()
1051 	vcpu->arch.cr3 = cr3;  in kvm_set_cr3()
1052 	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);  in kvm_set_cr3()
1058 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)  in kvm_set_cr8()  argument
1062 	if (lapic_in_kernel(vcpu))  in kvm_set_cr8()
1063 		kvm_lapic_set_tpr(vcpu, cr8);  in kvm_set_cr8()
1065 		vcpu->arch.cr8 = cr8;  in kvm_set_cr8()
1070 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)  in kvm_get_cr8()  argument
1072 	if (lapic_in_kernel(vcpu))  in kvm_get_cr8()
1073 		return kvm_lapic_get_cr8(vcpu);  in kvm_get_cr8()
1075 		return vcpu->arch.cr8;  in kvm_get_cr8()
1079 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)  in kvm_update_dr0123()  argument
1083 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {  in kvm_update_dr0123()
1085 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_update_dr0123()
1086 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;  in kvm_update_dr0123()
1090 void kvm_update_dr7(struct kvm_vcpu *vcpu)  in kvm_update_dr7()  argument
1094 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)  in kvm_update_dr7()
1095 		dr7 = vcpu->arch.guest_debug_dr7;  in kvm_update_dr7()
1097 		dr7 = vcpu->arch.dr7;  in kvm_update_dr7()
1098 	kvm_x86_ops.set_dr7(vcpu, dr7);  in kvm_update_dr7()
1099 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1101 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1105 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)  in kvm_dr6_fixed()  argument
1109 	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))  in kvm_dr6_fixed()
1114 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)  in __kvm_set_dr()  argument
1116 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in __kvm_set_dr()
1120 		vcpu->arch.db[array_index_nospec(dr, size)] = val;  in __kvm_set_dr()
1121 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))  in __kvm_set_dr()
1122 			vcpu->arch.eff_db[dr] = val;  in __kvm_set_dr()
1128 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);  in __kvm_set_dr()
1134 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;  in __kvm_set_dr()
1135 		kvm_update_dr7(vcpu);  in __kvm_set_dr()
1142 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)  in kvm_set_dr()  argument
1144 	if (__kvm_set_dr(vcpu, dr, val)) {  in kvm_set_dr()
1145 		kvm_inject_gp(vcpu, 0);  in kvm_set_dr()
1152 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)  in kvm_get_dr()  argument
1154 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_get_dr()
1158 		*val = vcpu->arch.db[array_index_nospec(dr, size)];  in kvm_get_dr()
1162 		*val = vcpu->arch.dr6;  in kvm_get_dr()
1166 		*val = vcpu->arch.dr7;  in kvm_get_dr()
1173 bool kvm_rdpmc(struct kvm_vcpu *vcpu)  in kvm_rdpmc()  argument
1175 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_rdpmc()
1179 	err = kvm_pmu_rdpmc(vcpu, ecx, &data);  in kvm_rdpmc()
1182 	kvm_rax_write(vcpu, (u32)data);  in kvm_rdpmc()
1183 	kvm_rdx_write(vcpu, data >> 32);  in kvm_rdpmc()
1407 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr_feature()  argument
1418 		if (kvm_msr_ignored_check(vcpu, index, 0, false))  in do_get_msr_feature()
1430 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in __kvm_valid_efer()  argument
1432 	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))  in __kvm_valid_efer()
1435 	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))  in __kvm_valid_efer()
1439 	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))  in __kvm_valid_efer()
1442 	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))  in __kvm_valid_efer()
1448 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in kvm_valid_efer()  argument
1453 	return __kvm_valid_efer(vcpu, efer);  in kvm_valid_efer()
1457 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_efer()  argument
1459 	u64 old_efer = vcpu->arch.efer;  in set_efer()
1467 		if (!__kvm_valid_efer(vcpu, efer))  in set_efer()
1470 		if (is_paging(vcpu) &&  in set_efer()
1471 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))  in set_efer()
1476 	efer |= vcpu->arch.efer & EFER_LMA;  in set_efer()
1478 	r = kvm_x86_ops.set_efer(vcpu, efer);  in set_efer()
1486 		kvm_mmu_reset_context(vcpu);  in set_efer()
1497 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)  in kvm_msr_allowed()  argument
1499 	struct kvm *kvm = vcpu->kvm;  in kvm_msr_allowed()
1537 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,  in __kvm_set_msr()  argument
1542 	if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))  in __kvm_set_msr()
1551 		if (is_noncanonical_address(data, vcpu))  in __kvm_set_msr()
1568 		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));  in __kvm_set_msr()
1575 	return kvm_x86_ops.set_msr(vcpu, &msr);  in __kvm_set_msr()
1578 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_set_msr_ignored_check()  argument
1581 	int ret = __kvm_set_msr(vcpu, index, data, host_initiated);  in kvm_set_msr_ignored_check()
1584 		if (kvm_msr_ignored_check(vcpu, index, data, true))  in kvm_set_msr_ignored_check()
1596 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,  in __kvm_get_msr()  argument
1602 	if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))  in __kvm_get_msr()
1608 	ret = kvm_x86_ops.get_msr(vcpu, &msr);  in __kvm_get_msr()
1614 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_get_msr_ignored_check()  argument
1617 	int ret = __kvm_get_msr(vcpu, index, data, host_initiated);  in kvm_get_msr_ignored_check()
1622 		if (kvm_msr_ignored_check(vcpu, index, 0, false))  in kvm_get_msr_ignored_check()
1629 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr()  argument
1631 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr()
1635 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr()  argument
1637 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr()
1641 static int complete_emulated_msr(struct kvm_vcpu *vcpu, bool is_read)  in complete_emulated_msr()  argument
1643 	if (vcpu->run->msr.error) {  in complete_emulated_msr()
1644 		kvm_inject_gp(vcpu, 0);  in complete_emulated_msr()
1647 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);  in complete_emulated_msr()
1648 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);  in complete_emulated_msr()
1651 	return kvm_skip_emulated_instruction(vcpu);  in complete_emulated_msr()
1654 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)  in complete_emulated_rdmsr()  argument
1656 	return complete_emulated_msr(vcpu, true);  in complete_emulated_rdmsr()
1659 static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)  in complete_emulated_wrmsr()  argument
1661 	return complete_emulated_msr(vcpu, false);  in complete_emulated_wrmsr()
1676 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,  in kvm_msr_user_space()  argument
1678 			      int (*completion)(struct kvm_vcpu *vcpu),  in kvm_msr_user_space()  argument
1684 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))  in kvm_msr_user_space()
1687 	vcpu->run->exit_reason = exit_reason;  in kvm_msr_user_space()
1688 	vcpu->run->msr.error = 0;  in kvm_msr_user_space()
1689 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));  in kvm_msr_user_space()
1690 	vcpu->run->msr.reason = msr_reason;  in kvm_msr_user_space()
1691 	vcpu->run->msr.index = index;  in kvm_msr_user_space()
1692 	vcpu->run->msr.data = data;  in kvm_msr_user_space()
1693 	vcpu->arch.complete_userspace_io = completion;  in kvm_msr_user_space()
1698 static int kvm_get_msr_user_space(struct kvm_vcpu *vcpu, u32 index, int r)  in kvm_get_msr_user_space()  argument
1700 	return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_RDMSR, 0,  in kvm_get_msr_user_space()
1704 static int kvm_set_msr_user_space(struct kvm_vcpu *vcpu, u32 index, u64 data, int r)  in kvm_set_msr_user_space()  argument
1706 	return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_WRMSR, data,  in kvm_set_msr_user_space()
1710 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_rdmsr()  argument
1712 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdmsr()
1716 	r = kvm_get_msr(vcpu, ecx, &data);  in kvm_emulate_rdmsr()
1719 	if (r && kvm_get_msr_user_space(vcpu, ecx, r)) {  in kvm_emulate_rdmsr()
1727 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_rdmsr()
1733 	kvm_rax_write(vcpu, data & -1u);  in kvm_emulate_rdmsr()
1734 	kvm_rdx_write(vcpu, (data >> 32) & -1u);  in kvm_emulate_rdmsr()
1735 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_rdmsr()
1739 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_wrmsr()  argument
1741 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_wrmsr()
1742 	u64 data = kvm_read_edx_eax(vcpu);  in kvm_emulate_wrmsr()
1745 	r = kvm_set_msr(vcpu, ecx, data);  in kvm_emulate_wrmsr()
1748 	if (r && kvm_set_msr_user_space(vcpu, ecx, data, r))  in kvm_emulate_wrmsr()
1759 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_wrmsr()
1764 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_wrmsr()
1768 bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)  in kvm_vcpu_exit_request()  argument
1770 	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||  in kvm_vcpu_exit_request()
1782 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_x2apic_icr_irqoff()  argument
1784 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))  in handle_fastpath_set_x2apic_icr_irqoff()
1793 		kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32));  in handle_fastpath_set_x2apic_icr_irqoff()
1794 		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));  in handle_fastpath_set_x2apic_icr_irqoff()
1795 		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data);  in handle_fastpath_set_x2apic_icr_irqoff()
1803 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_tscdeadline()  argument
1805 	if (!kvm_can_use_hv_timer(vcpu))  in handle_fastpath_set_tscdeadline()
1808 	kvm_set_lapic_tscdeadline_msr(vcpu, data);  in handle_fastpath_set_tscdeadline()
1812 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)  in handle_fastpath_set_msr_irqoff()  argument
1814 	u32 msr = kvm_rcx_read(vcpu);  in handle_fastpath_set_msr_irqoff()
1820 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
1821 		if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
1822 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
1827 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
1828 		if (!handle_fastpath_set_tscdeadline(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
1829 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
1847 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr()  argument
1849 	return kvm_get_msr_ignored_check(vcpu, index, data, true);  in do_get_msr()
1852 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_set_msr()  argument
1854 	return kvm_set_msr_ignored_check(vcpu, index, *data, true);  in do_set_msr()
1964 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,  in kvm_write_system_time()  argument
1967 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_write_system_time()
1969 	if (vcpu->vcpu_id == 0 && !host_initiated) {  in kvm_write_system_time()
1971 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_write_system_time()
1976 	vcpu->arch.time = system_time;  in kvm_write_system_time()
1977 	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_write_system_time()
1980 	vcpu->arch.pv_time_enabled = false;  in kvm_write_system_time()
1984 	if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,  in kvm_write_system_time()
1985 				       &vcpu->arch.pv_time, system_time & ~1ULL,  in kvm_write_system_time()
1987 		vcpu->arch.pv_time_enabled = true;  in kvm_write_system_time()
2040 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)  in set_tsc_khz()  argument
2046 		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;  in set_tsc_khz()
2053 			vcpu->arch.tsc_catchup = 1;  in set_tsc_khz()
2054 			vcpu->arch.tsc_always_catchup = 1;  in set_tsc_khz()
2072 	vcpu->arch.tsc_scaling_ratio = ratio;  in set_tsc_khz()
2076 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)  in kvm_set_tsc_khz()  argument
2084 		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;  in kvm_set_tsc_khz()
2090 			   &vcpu->arch.virtual_tsc_shift,  in kvm_set_tsc_khz()
2091 			   &vcpu->arch.virtual_tsc_mult);  in kvm_set_tsc_khz()
2092 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;  in kvm_set_tsc_khz()
2106 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);  in kvm_set_tsc_khz()
2109 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)  in compute_guest_tsc()  argument
2111 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,  in compute_guest_tsc()
2112 				      vcpu->arch.virtual_tsc_mult,  in compute_guest_tsc()
2113 				      vcpu->arch.virtual_tsc_shift);  in compute_guest_tsc()
2114 	tsc += vcpu->arch.this_tsc_write;  in compute_guest_tsc()
2123 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)  in kvm_track_tsc_matching()  argument
2127 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_track_tsc_matching()
2131 			 atomic_read(&vcpu->kvm->online_vcpus));  in kvm_track_tsc_matching()
2143 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_track_tsc_matching()
2145 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,  in kvm_track_tsc_matching()
2146 			    atomic_read(&vcpu->kvm->online_vcpus),  in kvm_track_tsc_matching()
2166 u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)  in kvm_scale_tsc()  argument
2169 	u64 ratio = vcpu->arch.tsc_scaling_ratio;  in kvm_scale_tsc()
2178 static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)  in kvm_compute_tsc_offset()  argument
2182 	tsc = kvm_scale_tsc(vcpu, rdtsc());  in kvm_compute_tsc_offset()
2187 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)  in kvm_read_l1_tsc()  argument
2189 	return vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, host_tsc);  in kvm_read_l1_tsc()
2193 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)  in kvm_vcpu_write_tsc_offset()  argument
2195 	vcpu->arch.l1_tsc_offset = offset;  in kvm_vcpu_write_tsc_offset()
2196 	vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset);  in kvm_vcpu_write_tsc_offset()
2212 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)  in kvm_synchronize_tsc()  argument
2214 	struct kvm *kvm = vcpu->kvm;  in kvm_synchronize_tsc()
2222 	offset = kvm_compute_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2226 	if (vcpu->arch.virtual_tsc_khz) {  in kvm_synchronize_tsc()
2236 						nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2237 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;  in kvm_synchronize_tsc()
2255 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {  in kvm_synchronize_tsc()
2259 			u64 delta = nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2261 			offset = kvm_compute_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2264 		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);  in kvm_synchronize_tsc()
2288 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;  in kvm_synchronize_tsc()
2290 	vcpu->arch.last_guest_tsc = data;  in kvm_synchronize_tsc()
2293 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;  in kvm_synchronize_tsc()
2294 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;  in kvm_synchronize_tsc()
2295 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;  in kvm_synchronize_tsc()
2297 	kvm_vcpu_write_tsc_offset(vcpu, offset);  in kvm_synchronize_tsc()
2307 	kvm_track_tsc_matching(vcpu);  in kvm_synchronize_tsc()
2311 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,  in adjust_tsc_offset_guest()  argument
2314 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;  in adjust_tsc_offset_guest()
2315 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);  in adjust_tsc_offset_guest()
2318 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)  in adjust_tsc_offset_host()  argument
2320 	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)  in adjust_tsc_offset_host()
2322 	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);  in adjust_tsc_offset_host()
2323 	adjust_tsc_offset_guest(vcpu, adjustment);  in adjust_tsc_offset_host()
2528 	struct kvm_vcpu *vcpu;  in kvm_gen_update_masterclock()  local
2536 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_gen_update_masterclock()
2537 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_gen_update_masterclock()
2540 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_gen_update_masterclock()
2541 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);  in kvm_gen_update_masterclock()
2581 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_setup_pvclock_page()  local
2584 	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,  in kvm_setup_pvclock_page()
2607 	vcpu->hv_clock.version = guest_hv_clock.version + 1;  in kvm_setup_pvclock_page()
2608 	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,  in kvm_setup_pvclock_page()
2609 				&vcpu->hv_clock,  in kvm_setup_pvclock_page()
2610 				sizeof(vcpu->hv_clock.version));  in kvm_setup_pvclock_page()
2615 	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);  in kvm_setup_pvclock_page()
2617 	if (vcpu->pvclock_set_guest_stopped_request) {  in kvm_setup_pvclock_page()
2618 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;  in kvm_setup_pvclock_page()
2619 		vcpu->pvclock_set_guest_stopped_request = false;  in kvm_setup_pvclock_page()
2622 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);  in kvm_setup_pvclock_page()
2624 	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,  in kvm_setup_pvclock_page()
2625 				&vcpu->hv_clock,  in kvm_setup_pvclock_page()
2626 				sizeof(vcpu->hv_clock));  in kvm_setup_pvclock_page()
2630 	vcpu->hv_clock.version++;  in kvm_setup_pvclock_page()
2631 	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,  in kvm_setup_pvclock_page()
2632 				&vcpu->hv_clock,  in kvm_setup_pvclock_page()
2633 				sizeof(vcpu->hv_clock.version));  in kvm_setup_pvclock_page()
2639 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_guest_time_update()  local
2686 	if (vcpu->tsc_catchup) {  in kvm_guest_time_update()
2701 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {  in kvm_guest_time_update()
2703 				   &vcpu->hv_clock.tsc_shift,  in kvm_guest_time_update()
2704 				   &vcpu->hv_clock.tsc_to_system_mul);  in kvm_guest_time_update()
2705 		vcpu->hw_tsc_khz = tgt_tsc_khz;  in kvm_guest_time_update()
2708 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;  in kvm_guest_time_update()
2709 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;  in kvm_guest_time_update()
2710 	vcpu->last_guest_tsc = tsc_timestamp;  in kvm_guest_time_update()
2717 	vcpu->hv_clock.flags = pvclock_flags;  in kvm_guest_time_update()
2719 	if (vcpu->pv_time_enabled)  in kvm_guest_time_update()
2722 		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);  in kvm_guest_time_update()
2749 	struct kvm_vcpu *vcpu;  in kvmclock_update_fn()  local
2751 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmclock_update_fn()
2752 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvmclock_update_fn()
2753 		kvm_vcpu_kick(vcpu);  in kvmclock_update_fn()
2786 static bool can_set_mci_status(struct kvm_vcpu *vcpu)  in can_set_mci_status()  argument
2789 	if (guest_cpuid_is_amd_or_hygon(vcpu))  in can_set_mci_status()
2790 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));  in can_set_mci_status()
2795 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_msr_mce()  argument
2797 	u64 mcg_cap = vcpu->arch.mcg_cap;  in set_msr_mce()
2804 		vcpu->arch.mcg_status = data;  in set_msr_mce()
2812 		vcpu->arch.mcg_ctl = data;  in set_msr_mce()
2833 				if (!can_set_mci_status(vcpu))  in set_msr_mce()
2837 			vcpu->arch.mce_banks[offset] = data;  in set_msr_mce()
2845 static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)  in xen_hvm_config()  argument
2847 	struct kvm *kvm = vcpu->kvm;  in xen_hvm_config()
2848 	int lm = is_long_mode(vcpu);  in xen_hvm_config()
2864 	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) {  in xen_hvm_config()
2871 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)  in kvm_pv_async_pf_enabled()  argument
2875 	return (vcpu->arch.apf.msr_en_val & mask) == mask;  in kvm_pv_async_pf_enabled()
2878 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf()  argument
2886 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&  in kvm_pv_enable_async_pf()
2890 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&  in kvm_pv_enable_async_pf()
2894 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf()
2897 	vcpu->arch.apf.msr_en_val = data;  in kvm_pv_enable_async_pf()
2899 	if (!kvm_pv_async_pf_enabled(vcpu)) {  in kvm_pv_enable_async_pf()
2900 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_pv_enable_async_pf()
2901 		kvm_async_pf_hash_reset(vcpu);  in kvm_pv_enable_async_pf()
2905 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,  in kvm_pv_enable_async_pf()
2909 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);  in kvm_pv_enable_async_pf()
2910 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;  in kvm_pv_enable_async_pf()
2912 	kvm_async_pf_wakeup_all(vcpu);  in kvm_pv_enable_async_pf()
2917 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf_int()  argument
2923 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf_int()
2926 	vcpu->arch.apf.msr_int_val = data;  in kvm_pv_enable_async_pf_int()
2928 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;  in kvm_pv_enable_async_pf_int()
2933 static void kvmclock_reset(struct kvm_vcpu *vcpu)  in kvmclock_reset()  argument
2935 	vcpu->arch.pv_time_enabled = false;  in kvmclock_reset()
2936 	vcpu->arch.time = 0;  in kvmclock_reset()
2939 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_all()  argument
2941 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_all()
2942 	kvm_x86_ops.tlb_flush_all(vcpu);  in kvm_vcpu_flush_tlb_all()
2945 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_guest()  argument
2947 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_guest()
2948 	kvm_x86_ops.tlb_flush_guest(vcpu);  in kvm_vcpu_flush_tlb_guest()
2951 static void record_steal_time(struct kvm_vcpu *vcpu)  in record_steal_time()  argument
2956 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in record_steal_time()
2960 	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT,  in record_steal_time()
2961 			&map, &vcpu->arch.st.cache, false))  in record_steal_time()
2965 		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);  in record_steal_time()
2971 	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {  in record_steal_time()
2972 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,  in record_steal_time()
2975 			kvm_vcpu_flush_tlb_guest(vcpu);  in record_steal_time()
2978 	vcpu->arch.st.preempted = 0;  in record_steal_time()
2988 		vcpu->arch.st.last_steal;  in record_steal_time()
2989 	vcpu->arch.st.last_steal = current->sched_info.run_delay;  in record_steal_time()
2995 	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false);  in record_steal_time()
2998 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_msr_common()  argument
3016 			vcpu->arch.microcode_version = data;  in kvm_set_msr_common()
3021 		vcpu->arch.arch_capabilities = data;  in kvm_set_msr_common()
3028 		if (guest_cpuid_has(vcpu, X86_FEATURE_PDCM) && kvm_get_msr_feature(&msr_ent))  in kvm_set_msr_common()
3033 		vcpu->arch.perf_capabilities = data;  in kvm_set_msr_common()
3038 		return set_efer(vcpu, msr_info);  in kvm_set_msr_common()
3046 			vcpu->arch.msr_hwcr = data;  in kvm_set_msr_common()
3048 			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",  in kvm_set_msr_common()
3055 			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "  in kvm_set_msr_common()
3069 			vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",  in kvm_set_msr_common()
3073 		return kvm_mtrr_set_msr(vcpu, msr, data);  in kvm_set_msr_common()
3075 		return kvm_set_apic_base(vcpu, msr_info);  in kvm_set_msr_common()
3077 		return kvm_x2apic_msr_write(vcpu, msr, data);  in kvm_set_msr_common()
3079 		kvm_set_lapic_tscdeadline_msr(vcpu, data);  in kvm_set_msr_common()
3082 		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {  in kvm_set_msr_common()
3084 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;  in kvm_set_msr_common()
3085 				adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3087 			vcpu->arch.ia32_tsc_adjust_msr = data;  in kvm_set_msr_common()
3091 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&  in kvm_set_msr_common()
3092 		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {  in kvm_set_msr_common()
3093 			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))  in kvm_set_msr_common()
3095 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3096 			kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3098 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3104 		vcpu->arch.smbase = data;  in kvm_set_msr_common()
3107 		vcpu->arch.msr_ia32_power_ctl = data;  in kvm_set_msr_common()
3111 			kvm_synchronize_tsc(vcpu, data);  in kvm_set_msr_common()
3113 			u64 adj = kvm_compute_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;  in kvm_set_msr_common()
3114 			adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3115 			vcpu->arch.ia32_tsc_adjust_msr += adj;  in kvm_set_msr_common()
3120 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_set_msr_common()
3129 		vcpu->arch.ia32_xss = data;  in kvm_set_msr_common()
3134 		vcpu->arch.smi_count = data;  in kvm_set_msr_common()
3137 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3140 		kvm_write_wall_clock(vcpu->kvm, data);  in kvm_set_msr_common()
3143 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3146 		kvm_write_wall_clock(vcpu->kvm, data);  in kvm_set_msr_common()
3149 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3152 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);  in kvm_set_msr_common()
3155 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3158 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);  in kvm_set_msr_common()
3161 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_set_msr_common()
3164 		if (kvm_pv_enable_async_pf(vcpu, data))  in kvm_set_msr_common()
3168 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3171 		if (kvm_pv_enable_async_pf_int(vcpu, data))  in kvm_set_msr_common()
3175 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_set_msr_common()
3178 			vcpu->arch.apf.pageready_pending = false;  in kvm_set_msr_common()
3179 			kvm_check_async_pf_completion(vcpu);  in kvm_set_msr_common()
3183 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_set_msr_common()
3192 		vcpu->arch.st.msr_val = data;  in kvm_set_msr_common()
3197 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_set_msr_common()
3201 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_set_msr_common()
3204 		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))  in kvm_set_msr_common()
3209 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_set_msr_common()
3216 		vcpu->arch.msr_kvm_poll_control = data;  in kvm_set_msr_common()
3222 		return set_msr_mce(vcpu, msr_info);  in kvm_set_msr_common()
3230 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
3231 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
3234 			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "  in kvm_set_msr_common()
3256 		return kvm_hv_set_msr_common(vcpu, msr, data,  in kvm_set_msr_common()
3263 			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",  in kvm_set_msr_common()
3267 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
3269 		vcpu->arch.osvw.length = data;  in kvm_set_msr_common()
3272 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
3274 		vcpu->arch.osvw.status = data;  in kvm_set_msr_common()
3279 		     cpuid_fault_enabled(vcpu)))  in kvm_set_msr_common()
3281 		vcpu->arch.msr_platform_info = data;  in kvm_set_msr_common()
3286 		     !supports_cpuid_fault(vcpu)))  in kvm_set_msr_common()
3288 		vcpu->arch.msr_misc_features_enables = data;  in kvm_set_msr_common()
3291 		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))  in kvm_set_msr_common()
3292 			return xen_hvm_config(vcpu, data);  in kvm_set_msr_common()
3293 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
3294 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
3301 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)  in get_msr_mce()  argument
3304 	u64 mcg_cap = vcpu->arch.mcg_cap;  in get_msr_mce()
3313 		data = vcpu->arch.mcg_cap;  in get_msr_mce()
3318 		data = vcpu->arch.mcg_ctl;  in get_msr_mce()
3321 		data = vcpu->arch.mcg_status;  in get_msr_mce()
3330 			data = vcpu->arch.mce_banks[offset];  in get_msr_mce()
3339 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_get_msr_common()  argument
3378 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
3379 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
3383 		msr_info->data = vcpu->arch.microcode_version;  in kvm_get_msr_common()
3387 		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))  in kvm_get_msr_common()
3389 		msr_info->data = vcpu->arch.arch_capabilities;  in kvm_get_msr_common()
3393 		    !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))  in kvm_get_msr_common()
3395 		msr_info->data = vcpu->arch.perf_capabilities;  in kvm_get_msr_common()
3398 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;  in kvm_get_msr_common()
3410 		u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset :  in kvm_get_msr_common()
3411 							    vcpu->arch.tsc_offset;  in kvm_get_msr_common()
3413 		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + tsc_offset;  in kvm_get_msr_common()
3418 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
3437 		msr_info->data = kvm_get_apic_base(vcpu);  in kvm_get_msr_common()
3440 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
3442 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);  in kvm_get_msr_common()
3445 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;  in kvm_get_msr_common()
3448 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;  in kvm_get_msr_common()
3453 		msr_info->data = vcpu->arch.smbase;  in kvm_get_msr_common()
3456 		msr_info->data = vcpu->arch.smi_count;  in kvm_get_msr_common()
3465 		msr_info->data = vcpu->arch.efer;  in kvm_get_msr_common()
3468 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
3471 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
3474 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
3477 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
3480 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
3483 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
3486 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
3489 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
3492 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_get_msr_common()
3495 		msr_info->data = vcpu->arch.apf.msr_en_val;  in kvm_get_msr_common()
3498 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
3501 		msr_info->data = vcpu->arch.apf.msr_int_val;  in kvm_get_msr_common()
3504 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_get_msr_common()
3510 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_get_msr_common()
3513 		msr_info->data = vcpu->arch.st.msr_val;  in kvm_get_msr_common()
3516 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_get_msr_common()
3519 		msr_info->data = vcpu->arch.pv_eoi.msr_val;  in kvm_get_msr_common()
3522 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_get_msr_common()
3525 		msr_info->data = vcpu->arch.msr_kvm_poll_control;  in kvm_get_msr_common()
3533 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,  in kvm_get_msr_common()
3537 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_get_msr_common()
3539 		msr_info->data = vcpu->arch.ia32_xss;  in kvm_get_msr_common()
3562 		return kvm_hv_get_msr_common(vcpu,  in kvm_get_msr_common()
3579 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
3581 		msr_info->data = vcpu->arch.osvw.length;  in kvm_get_msr_common()
3584 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
3586 		msr_info->data = vcpu->arch.osvw.status;  in kvm_get_msr_common()
3590 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)  in kvm_get_msr_common()
3592 		msr_info->data = vcpu->arch.msr_platform_info;  in kvm_get_msr_common()
3595 		msr_info->data = vcpu->arch.msr_misc_features_enables;  in kvm_get_msr_common()
3598 		msr_info->data = vcpu->arch.msr_hwcr;  in kvm_get_msr_common()
3601 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
3602 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
3614 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,  in __msr_io()  argument
3616 		    int (*do_msr)(struct kvm_vcpu *vcpu,  in __msr_io()  argument
3622 		if (do_msr(vcpu, entries[i].index, &entries[i].data))  in __msr_io()
3633 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,  in msr_io()  argument
3634 		  int (*do_msr)(struct kvm_vcpu *vcpu,  in msr_io()  argument
3658 	r = n = __msr_io(vcpu, &msrs, entries, do_msr);  in msr_io()
3915 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)  in need_emulate_wbinvd()  argument
3917 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);  in need_emulate_wbinvd()
3920 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_vcpu_load()  argument
3923 	if (need_emulate_wbinvd(vcpu)) {  in kvm_arch_vcpu_load()
3925 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_load()
3926 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
3927 			smp_call_function_single(vcpu->cpu,  in kvm_arch_vcpu_load()
3931 	kvm_x86_ops.vcpu_load(vcpu, cpu);  in kvm_arch_vcpu_load()
3934 	vcpu->arch.host_pkru = read_pkru();  in kvm_arch_vcpu_load()
3937 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {  in kvm_arch_vcpu_load()
3938 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);  in kvm_arch_vcpu_load()
3939 		vcpu->arch.tsc_offset_adjustment = 0;  in kvm_arch_vcpu_load()
3940 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
3943 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {  in kvm_arch_vcpu_load()
3944 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :  in kvm_arch_vcpu_load()
3945 				rdtsc() - vcpu->arch.last_host_tsc;  in kvm_arch_vcpu_load()
3950 			u64 offset = kvm_compute_tsc_offset(vcpu,  in kvm_arch_vcpu_load()
3951 						vcpu->arch.last_guest_tsc);  in kvm_arch_vcpu_load()
3952 			kvm_vcpu_write_tsc_offset(vcpu, offset);  in kvm_arch_vcpu_load()
3953 			vcpu->arch.tsc_catchup = 1;  in kvm_arch_vcpu_load()
3956 		if (kvm_lapic_hv_timer_in_use(vcpu))  in kvm_arch_vcpu_load()
3957 			kvm_lapic_restart_hv_timer(vcpu);  in kvm_arch_vcpu_load()
3963 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)  in kvm_arch_vcpu_load()
3964 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
3965 		if (vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
3966 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);  in kvm_arch_vcpu_load()
3967 		vcpu->cpu = cpu;  in kvm_arch_vcpu_load()
3970 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_arch_vcpu_load()
3973 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)  in kvm_steal_time_set_preempted()  argument
3978 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in kvm_steal_time_set_preempted()
3981 	if (vcpu->arch.st.preempted)  in kvm_steal_time_set_preempted()
3984 	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,  in kvm_steal_time_set_preempted()
3985 			&vcpu->arch.st.cache, true))  in kvm_steal_time_set_preempted()
3989 		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);  in kvm_steal_time_set_preempted()
3991 	st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;  in kvm_steal_time_set_preempted()
3993 	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);  in kvm_steal_time_set_preempted()
3996 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_put()  argument
4000 	if (vcpu->preempted)  in kvm_arch_vcpu_put()
4001 		vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);  in kvm_arch_vcpu_put()
4016 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_put()
4017 	kvm_steal_time_set_preempted(vcpu);  in kvm_arch_vcpu_put()
4018 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_put()
4020 	kvm_x86_ops.vcpu_put(vcpu);  in kvm_arch_vcpu_put()
4021 	vcpu->arch.last_host_tsc = rdtsc();  in kvm_arch_vcpu_put()
4030 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_get_lapic()  argument
4033 	if (vcpu->arch.apicv_active)  in kvm_vcpu_ioctl_get_lapic()
4034 		kvm_x86_ops.sync_pir_to_irr(vcpu);  in kvm_vcpu_ioctl_get_lapic()
4036 	return kvm_apic_get_state(vcpu, s);  in kvm_vcpu_ioctl_get_lapic()
4039 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_set_lapic()  argument
4044 	r = kvm_apic_set_state(vcpu, s);  in kvm_vcpu_ioctl_set_lapic()
4047 	update_cr8_intercept(vcpu);  in kvm_vcpu_ioctl_set_lapic()
4052 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)  in kvm_cpu_accept_dm_intr()  argument
4060 	if (kvm_cpu_has_extint(vcpu))  in kvm_cpu_accept_dm_intr()
4064 	return (!lapic_in_kernel(vcpu) ||  in kvm_cpu_accept_dm_intr()
4065 		kvm_apic_accept_pic_intr(vcpu));  in kvm_cpu_accept_dm_intr()
4068 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)  in kvm_vcpu_ready_for_interrupt_injection()  argument
4070 	return kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
4071 		kvm_cpu_accept_dm_intr(vcpu);  in kvm_vcpu_ready_for_interrupt_injection()
4074 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_interrupt()  argument
4080 	if (!irqchip_in_kernel(vcpu->kvm)) {  in kvm_vcpu_ioctl_interrupt()
4081 		kvm_queue_interrupt(vcpu, irq->irq, false);  in kvm_vcpu_ioctl_interrupt()
4082 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
4090 	if (pic_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_interrupt()
4093 	if (vcpu->arch.pending_external_vector != -1)  in kvm_vcpu_ioctl_interrupt()
4096 	vcpu->arch.pending_external_vector = irq->irq;  in kvm_vcpu_ioctl_interrupt()
4097 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
4101 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)  in kvm_vcpu_ioctl_nmi()  argument
4103 	kvm_inject_nmi(vcpu);  in kvm_vcpu_ioctl_nmi()
4108 static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)  in kvm_vcpu_ioctl_smi()  argument
4110 	kvm_make_request(KVM_REQ_SMI, vcpu);  in kvm_vcpu_ioctl_smi()
4115 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,  in vcpu_ioctl_tpr_access_reporting()  argument
4120 	vcpu->arch.tpr_access_reporting = !!tac->enabled;  in vcpu_ioctl_tpr_access_reporting()
4124 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_setup_mce()  argument
4136 	vcpu->arch.mcg_cap = mcg_cap;  in kvm_vcpu_ioctl_x86_setup_mce()
4139 		vcpu->arch.mcg_ctl = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
4142 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
4144 	kvm_x86_ops.setup_mce(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
4149 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_mce()  argument
4152 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_ioctl_x86_set_mce()
4154 	u64 *banks = vcpu->arch.mce_banks;  in kvm_vcpu_ioctl_x86_set_mce()
4163 	    vcpu->arch.mcg_ctl != ~(u64)0)  in kvm_vcpu_ioctl_x86_set_mce()
4173 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||  in kvm_vcpu_ioctl_x86_set_mce()
4174 		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {  in kvm_vcpu_ioctl_x86_set_mce()
4175 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_mce()
4182 		vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_ioctl_x86_set_mce()
4184 		kvm_queue_exception(vcpu, MC_VECTOR);  in kvm_vcpu_ioctl_x86_set_mce()
4197 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_vcpu_events()  argument
4200 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4213 	if (!vcpu->kvm->arch.exception_payload_enabled &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4214 	    vcpu->arch.exception.pending && vcpu->arch.exception.has_payload)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4215 		kvm_deliver_exception_payload(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4223 	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4227 		events->exception.injected = vcpu->arch.exception.injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4228 		events->exception.pending = vcpu->arch.exception.pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4234 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4236 				vcpu->arch.exception.pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4238 	events->exception.nr = vcpu->arch.exception.nr;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4239 	events->exception.has_error_code = vcpu->arch.exception.has_error_code;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4240 	events->exception.error_code = vcpu->arch.exception.error_code;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4241 	events->exception_has_payload = vcpu->arch.exception.has_payload;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4242 	events->exception_payload = vcpu->arch.exception.payload;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4245 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4246 	events->interrupt.nr = vcpu->arch.interrupt.nr;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4248 	events->interrupt.shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4250 	events->nmi.injected = vcpu->arch.nmi_injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4251 	events->nmi.pending = vcpu->arch.nmi_pending != 0;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4252 	events->nmi.masked = kvm_x86_ops.get_nmi_mask(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4257 	events->smi.smm = is_smm(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4258 	events->smi.pending = vcpu->arch.smi_pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4260 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4261 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4266 	if (vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
4272 static void kvm_smm_changed(struct kvm_vcpu *vcpu);
4274 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()  argument
4285 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4303 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4306 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4307 	vcpu->arch.exception.injected = events->exception.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4308 	vcpu->arch.exception.pending = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4309 	vcpu->arch.exception.nr = events->exception.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4310 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4311 	vcpu->arch.exception.error_code = events->exception.error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4312 	vcpu->arch.exception.has_payload = events->exception_has_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4313 	vcpu->arch.exception.payload = events->exception_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4315 	vcpu->arch.interrupt.injected = events->interrupt.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4316 	vcpu->arch.interrupt.nr = events->interrupt.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4317 	vcpu->arch.interrupt.soft = events->interrupt.soft;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4319 		kvm_x86_ops.set_interrupt_shadow(vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4322 	vcpu->arch.nmi_injected = events->nmi.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4324 		vcpu->arch.nmi_pending = events->nmi.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4325 	kvm_x86_ops.set_nmi_mask(vcpu, events->nmi.masked);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4328 	    lapic_in_kernel(vcpu))  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4329 		vcpu->arch.apic->sipi_vector = events->sipi_vector;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4332 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4334 				vcpu->arch.hflags |= HF_SMM_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4336 				vcpu->arch.hflags &= ~HF_SMM_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4337 			kvm_smm_changed(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4340 		vcpu->arch.smi_pending = events->smi.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4344 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4346 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4349 		if (lapic_in_kernel(vcpu)) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4351 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4353 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4357 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
4362 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_debugregs()  argument
4367 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_get_debugregs()
4368 	kvm_get_dr(vcpu, 6, &val);  in kvm_vcpu_ioctl_x86_get_debugregs()
4370 	dbgregs->dr7 = vcpu->arch.dr7;  in kvm_vcpu_ioctl_x86_get_debugregs()
4375 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_debugregs()  argument
4386 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_set_debugregs()
4387 	kvm_update_dr0123(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
4388 	vcpu->arch.dr6 = dbgregs->dr6;  in kvm_vcpu_ioctl_x86_set_debugregs()
4389 	vcpu->arch.dr7 = dbgregs->dr7;  in kvm_vcpu_ioctl_x86_set_debugregs()
4390 	kvm_update_dr7(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
4397 static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)  in fill_xsave()  argument
4399 	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;  in fill_xsave()
4410 	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;  in fill_xsave()
4428 				memcpy(dest + offset, &vcpu->arch.pkru,  in fill_xsave()
4429 				       sizeof(vcpu->arch.pkru));  in fill_xsave()
4439 static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)  in load_xsave()  argument
4441 	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;  in load_xsave()
4471 				memcpy(&vcpu->arch.pkru, src + offset,  in load_xsave()
4472 				       sizeof(vcpu->arch.pkru));  in load_xsave()
4481 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave()  argument
4486 		fill_xsave((u8 *) guest_xsave->region, vcpu);  in kvm_vcpu_ioctl_x86_get_xsave()
4489 			&vcpu->arch.guest_fpu->state.fxsave,  in kvm_vcpu_ioctl_x86_get_xsave()
4498 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xsave()  argument
4513 		load_xsave(vcpu, (u8 *)guest_xsave->region);  in kvm_vcpu_ioctl_x86_set_xsave()
4518 		memcpy(&vcpu->arch.guest_fpu->state.fxsave,  in kvm_vcpu_ioctl_x86_set_xsave()
4524 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xcrs()  argument
4535 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;  in kvm_vcpu_ioctl_x86_get_xcrs()
4538 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xcrs()  argument
4552 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,  in kvm_vcpu_ioctl_x86_set_xcrs()
4567 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)  in kvm_set_guest_paused()  argument
4569 	if (!vcpu->arch.pv_time_enabled)  in kvm_set_guest_paused()
4571 	vcpu->arch.pvclock_set_guest_stopped_request = true;  in kvm_set_guest_paused()
4572 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_guest_paused()
4576 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_enable_cap()  argument
4593 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_enable_cap()
4595 		return kvm_hv_activate_synic(vcpu, cap->cap ==  in kvm_vcpu_ioctl_enable_cap()
4600 		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);  in kvm_vcpu_ioctl_enable_cap()
4612 		return kvm_x86_ops.enable_direct_tlbflush(vcpu);  in kvm_vcpu_ioctl_enable_cap()
4615 		vcpu->arch.pv_cpuid.enforce = cap->args[0];  in kvm_vcpu_ioctl_enable_cap()
4616 		if (vcpu->arch.pv_cpuid.enforce)  in kvm_vcpu_ioctl_enable_cap()
4617 			kvm_update_pv_runtime(vcpu);  in kvm_vcpu_ioctl_enable_cap()
4629 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_ioctl()  local
4639 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl()
4645 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
4653 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
4664 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
4672 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
4681 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);  in kvm_arch_vcpu_ioctl()
4685 		r = kvm_vcpu_ioctl_nmi(vcpu);  in kvm_arch_vcpu_ioctl()
4689 		r = kvm_vcpu_ioctl_smi(vcpu);  in kvm_arch_vcpu_ioctl()
4699 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
4709 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
4720 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
4731 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
4732 		r = msr_io(vcpu, argp, do_get_msr, 1);  in kvm_arch_vcpu_ioctl()
4733 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
4737 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
4738 		r = msr_io(vcpu, argp, do_set_msr, 0);  in kvm_arch_vcpu_ioctl()
4739 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
4748 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);  in kvm_arch_vcpu_ioctl()
4762 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
4767 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
4768 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);  in kvm_arch_vcpu_ioctl()
4769 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
4778 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);  in kvm_arch_vcpu_ioctl()
4787 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);  in kvm_arch_vcpu_ioctl()
4793 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
4808 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
4814 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
4831 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
4840 		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
4855 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
4864 		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
4880 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
4896 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))  in kvm_arch_vcpu_ioctl()
4902 		r = vcpu->arch.virtual_tsc_khz;  in kvm_arch_vcpu_ioctl()
4906 		r = kvm_set_guest_paused(vcpu);  in kvm_arch_vcpu_ioctl()
4915 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);  in kvm_arch_vcpu_ioctl()
4931 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,  in kvm_arch_vcpu_ioctl()
4975 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
4976 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);  in kvm_arch_vcpu_ioctl()
4977 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
4988 		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5005 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl()
5009 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)  in kvm_arch_vcpu_fault()  argument
5796 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,  in vcpu_mmio_write()  argument
5804 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_write()
5805 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))  in vcpu_mmio_write()
5806 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_write()
5817 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)  in vcpu_mmio_read()  argument
5824 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_read()
5825 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,  in vcpu_mmio_read()
5827 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_read()
5839 static void kvm_set_segment(struct kvm_vcpu *vcpu,  in kvm_set_segment()  argument
5842 	kvm_x86_ops.set_segment(vcpu, var, seg);  in kvm_set_segment()
5845 void kvm_get_segment(struct kvm_vcpu *vcpu,  in kvm_get_segment()  argument
5848 	kvm_x86_ops.get_segment(vcpu, var, seg);  in kvm_get_segment()
5851 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,  in translate_nested_gpa()  argument
5856 	BUG_ON(!mmu_is_nested(vcpu));  in translate_nested_gpa()
5860 	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);  in translate_nested_gpa()
5865 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_read()  argument
5868 	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_read()
5869 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);  in kvm_mmu_gva_to_gpa_read()
5872  gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_fetch()  argument
5875 	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_fetch()
5877 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);  in kvm_mmu_gva_to_gpa_fetch()
5880 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_write()  argument
5883 	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_write()
5885 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);  in kvm_mmu_gva_to_gpa_write()
5889 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_system()  argument
5892 	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);  in kvm_mmu_gva_to_gpa_system()
5896 				      struct kvm_vcpu *vcpu, u32 access,  in kvm_read_guest_virt_helper()  argument
5903 		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,  in kvm_read_guest_virt_helper()
5911 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,  in kvm_read_guest_virt_helper()
5931 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_fetch_guest_virt()  local
5932 	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_fetch_guest_virt()
5937 	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,  in kvm_fetch_guest_virt()
5945 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,  in kvm_fetch_guest_virt()
5953 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,  in kvm_read_guest_virt()  argument
5957 	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_read_guest_virt()
5966 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,  in kvm_read_guest_virt()
5975 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_std()  local
5978 	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)  in emulator_read_std()
5981 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);  in emulator_read_std()
5987 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_read_guest_phys_system()  local
5988 	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);  in kvm_read_guest_phys_system()
5994 				      struct kvm_vcpu *vcpu, u32 access,  in kvm_write_guest_virt_helper()  argument
6001 		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,  in kvm_write_guest_virt_helper()
6010 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);  in kvm_write_guest_virt_helper()
6028 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_write_std()  local
6031 	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)  in emulator_write_std()
6034 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in emulator_write_std()
6038 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,  in kvm_write_guest_virt_system()  argument
6042 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_write_guest_virt_system()
6044 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in kvm_write_guest_virt_system()
6049 int handle_ud(struct kvm_vcpu *vcpu)  in handle_ud()  argument
6056 	if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, NULL, 0)))  in handle_ud()
6060 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),  in handle_ud()
6063 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));  in handle_ud()
6067 	return kvm_emulate_instruction(vcpu, emul_type);  in handle_ud()
6071 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_is_mmio_gpa()  argument
6078 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {  in vcpu_is_mmio_gpa()
6086 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_mmio_gva_to_gpa()  argument
6090 	u32 access = ((kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)  in vcpu_mmio_gva_to_gpa()
6098 	if (vcpu_match_mmio_gva(vcpu, gva)  in vcpu_mmio_gva_to_gpa()
6099 	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,  in vcpu_mmio_gva_to_gpa()
6100 				 vcpu->arch.mmio_access, 0, access)) {  in vcpu_mmio_gva_to_gpa()
6101 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |  in vcpu_mmio_gva_to_gpa()
6107 	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);  in vcpu_mmio_gva_to_gpa()
6112 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);  in vcpu_mmio_gva_to_gpa()
6115 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,  in emulator_write_phys()  argument
6120 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);  in emulator_write_phys()
6123 	kvm_page_track_write(vcpu, gpa, val, bytes);  in emulator_write_phys()
6128 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
6130 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
6132 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
6134 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
6139 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)  in read_prepare()  argument
6141 	if (vcpu->mmio_read_completed) {  in read_prepare()
6143 			       vcpu->mmio_fragments[0].gpa, val);  in read_prepare()
6144 		vcpu->mmio_read_completed = 0;  in read_prepare()
6151 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_emulate()  argument
6154 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);  in read_emulate()
6157 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_emulate()  argument
6160 	return emulator_write_phys(vcpu, gpa, val, bytes);  in write_emulate()
6163 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)  in write_mmio()  argument
6166 	return vcpu_mmio_write(vcpu, gpa, bytes, val);  in write_mmio()
6169 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_exit_mmio()  argument
6176 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_exit_mmio()  argument
6179 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];  in write_exit_mmio()
6181 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in write_exit_mmio()
6202 				       struct kvm_vcpu *vcpu,  in emulator_read_write_onepage()  argument
6209 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in emulator_read_write_onepage()
6221 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);  in emulator_read_write_onepage()
6223 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);  in emulator_read_write_onepage()
6228 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))  in emulator_read_write_onepage()
6234 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);  in emulator_read_write_onepage()
6242 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);  in emulator_read_write_onepage()
6243 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];  in emulator_read_write_onepage()
6256 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_write()  local
6261 		  ops->read_write_prepare(vcpu, val, bytes))  in emulator_read_write()
6264 	vcpu->mmio_nr_fragments = 0;  in emulator_read_write()
6272 						 vcpu, ops);  in emulator_read_write()
6284 					 vcpu, ops);  in emulator_read_write()
6288 	if (!vcpu->mmio_nr_fragments)  in emulator_read_write()
6291 	gpa = vcpu->mmio_fragments[0].gpa;  in emulator_read_write()
6293 	vcpu->mmio_needed = 1;  in emulator_read_write()
6294 	vcpu->mmio_cur_fragment = 0;  in emulator_read_write()
6296 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);  in emulator_read_write()
6297 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;  in emulator_read_write()
6298 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in emulator_read_write()
6299 	vcpu->run->mmio.phys_addr = gpa;  in emulator_read_write()
6301 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);  in emulator_read_write()
6342 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_cmpxchg_emulated()  local
6352 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);  in emulator_cmpxchg_emulated()
6370 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))  in emulator_cmpxchg_emulated()
6392 	kvm_vcpu_unmap(vcpu, &map, true);  in emulator_cmpxchg_emulated()
6397 	kvm_page_track_write(vcpu, gpa, new, bytes);  in emulator_cmpxchg_emulated()
6407 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)  in kernel_pio()  argument
6411 	for (i = 0; i < vcpu->arch.pio.count; i++) {  in kernel_pio()
6412 		if (vcpu->arch.pio.in)  in kernel_pio()
6413 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,  in kernel_pio()
6414 					    vcpu->arch.pio.size, pd);  in kernel_pio()
6416 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,  in kernel_pio()
6417 					     vcpu->arch.pio.port, vcpu->arch.pio.size,  in kernel_pio()
6421 		pd += vcpu->arch.pio.size;  in kernel_pio()
6426 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in_out()  argument
6430 	vcpu->arch.pio.port = port;  in emulator_pio_in_out()
6431 	vcpu->arch.pio.in = in;  in emulator_pio_in_out()
6432 	vcpu->arch.pio.count  = count;  in emulator_pio_in_out()
6433 	vcpu->arch.pio.size = size;  in emulator_pio_in_out()
6435 	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {  in emulator_pio_in_out()
6436 		vcpu->arch.pio.count = 0;  in emulator_pio_in_out()
6440 	vcpu->run->exit_reason = KVM_EXIT_IO;  in emulator_pio_in_out()
6441 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;  in emulator_pio_in_out()
6442 	vcpu->run->io.size = size;  in emulator_pio_in_out()
6443 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;  in emulator_pio_in_out()
6444 	vcpu->run->io.count = count;  in emulator_pio_in_out()
6445 	vcpu->run->io.port = port;  in emulator_pio_in_out()
6450 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in()  argument
6455 	if (vcpu->arch.pio.count)  in emulator_pio_in()
6458 	memset(vcpu->arch.pio_data, 0, size * count);  in emulator_pio_in()
6460 	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);  in emulator_pio_in()
6463 		memcpy(val, vcpu->arch.pio_data, size * count);  in emulator_pio_in()
6464 		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);  in emulator_pio_in()
6465 		vcpu->arch.pio.count = 0;  in emulator_pio_in()
6480 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_out()  argument
6484 	memcpy(vcpu->arch.pio_data, val, size * count);  in emulator_pio_out()
6485 	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);  in emulator_pio_out()
6486 	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);  in emulator_pio_out()
6496 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)  in get_segment_base()  argument
6498 	return kvm_x86_ops.get_segment_base(vcpu, seg);  in get_segment_base()
6506 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd_noskip()  argument
6508 	if (!need_emulate_wbinvd(vcpu))  in kvm_emulate_wbinvd_noskip()
6514 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
6515 		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,  in kvm_emulate_wbinvd_noskip()
6518 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
6524 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd()  argument
6526 	kvm_emulate_wbinvd_noskip(vcpu);  in kvm_emulate_wbinvd()
6527 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_wbinvd()
6558 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_cr()  local
6563 		value = kvm_read_cr0(vcpu);  in emulator_get_cr()
6566 		value = vcpu->arch.cr2;  in emulator_get_cr()
6569 		value = kvm_read_cr3(vcpu);  in emulator_get_cr()
6572 		value = kvm_read_cr4(vcpu);  in emulator_get_cr()
6575 		value = kvm_get_cr8(vcpu);  in emulator_get_cr()
6587 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_cr()  local
6592 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));  in emulator_set_cr()
6595 		vcpu->arch.cr2 = val;  in emulator_set_cr()
6598 		res = kvm_set_cr3(vcpu, val);  in emulator_set_cr()
6601 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));  in emulator_set_cr()
6604 		res = kvm_set_cr8(vcpu, val);  in emulator_set_cr()
6685 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_segment()  local
6707 	kvm_set_segment(vcpu, &var, seg);  in emulator_set_segment()
6714 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_msr()  local
6717 	r = kvm_get_msr(vcpu, msr_index, pdata);  in emulator_get_msr()
6719 	if (r && kvm_get_msr_user_space(vcpu, msr_index, r)) {  in emulator_get_msr()
6730 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_msr()  local
6733 	r = kvm_set_msr(vcpu, msr_index, data);  in emulator_set_msr()
6735 	if (r && kvm_set_msr_user_space(vcpu, msr_index, data, r)) {  in emulator_set_msr()
6745 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_smbase()  local
6747 	return vcpu->arch.smbase;  in emulator_get_smbase()
6752 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_smbase()  local
6754 	vcpu->arch.smbase = smbase;  in emulator_set_smbase()
6892 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)  in toggle_interruptibility()  argument
6894 	u32 int_shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);  in toggle_interruptibility()
6905 		kvm_x86_ops.set_interrupt_shadow(vcpu, mask);  in toggle_interruptibility()
6907 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in toggle_interruptibility()
6911 static bool inject_emulated_exception(struct kvm_vcpu *vcpu)  in inject_emulated_exception()  argument
6913 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in inject_emulated_exception()
6915 		return kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);  in inject_emulated_exception()
6918 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,  in inject_emulated_exception()
6921 		kvm_queue_exception(vcpu, ctxt->exception.vector);  in inject_emulated_exception()
6925 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)  in alloc_emulate_ctxt()  argument
6935 	ctxt->vcpu = vcpu;  in alloc_emulate_ctxt()
6937 	vcpu->arch.emulate_ctxt = ctxt;  in alloc_emulate_ctxt()
6942 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)  in init_emulate_ctxt()  argument
6944 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in init_emulate_ctxt()
6947 	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);  in init_emulate_ctxt()
6950 	ctxt->eflags = kvm_get_rflags(vcpu);  in init_emulate_ctxt()
6953 	ctxt->eip = kvm_rip_read(vcpu);  in init_emulate_ctxt()
6954 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :  in init_emulate_ctxt()
6956 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :  in init_emulate_ctxt()
6964 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in init_emulate_ctxt()
6967 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)  in kvm_inject_realmode_interrupt()  argument
6969 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_inject_realmode_interrupt()
6972 	init_emulate_ctxt(vcpu);  in kvm_inject_realmode_interrupt()
6980 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_inject_realmode_interrupt()
6983 		kvm_rip_write(vcpu, ctxt->eip);  in kvm_inject_realmode_interrupt()
6984 		kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_inject_realmode_interrupt()
6989 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)  in handle_emulation_failure()  argument
6991 	++vcpu->stat.insn_emulation_fail;  in handle_emulation_failure()
6992 	trace_kvm_emulate_insn_failed(vcpu);  in handle_emulation_failure()
6995 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in handle_emulation_failure()
7000 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in handle_emulation_failure()
7001 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in handle_emulation_failure()
7002 		vcpu->run->internal.ndata = 0;  in handle_emulation_failure()
7006 	kvm_queue_exception(vcpu, UD_VECTOR);  in handle_emulation_failure()
7008 	if (!is_guest_mode(vcpu) && kvm_x86_ops.get_cpl(vcpu) == 0) {  in handle_emulation_failure()
7009 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in handle_emulation_failure()
7010 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in handle_emulation_failure()
7011 		vcpu->run->internal.ndata = 0;  in handle_emulation_failure()
7018 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in reexecute_instruction()  argument
7028 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in reexecute_instruction()
7032 	if (!vcpu->arch.mmu->direct_map) {  in reexecute_instruction()
7037 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in reexecute_instruction()
7053 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
7065 	if (vcpu->arch.mmu->direct_map) {  in reexecute_instruction()
7068 		spin_lock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
7069 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;  in reexecute_instruction()
7070 		spin_unlock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
7073 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
7083 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
7096 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in retry_instruction()  local
7099 	last_retry_eip = vcpu->arch.last_retry_eip;  in retry_instruction()
7100 	last_retry_addr = vcpu->arch.last_retry_addr;  in retry_instruction()
7115 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;  in retry_instruction()
7120 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in retry_instruction()
7130 	vcpu->arch.last_retry_eip = ctxt->eip;  in retry_instruction()
7131 	vcpu->arch.last_retry_addr = cr2_or_gpa;  in retry_instruction()
7133 	if (!vcpu->arch.mmu->direct_map)  in retry_instruction()
7134 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in retry_instruction()
7136 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in retry_instruction()
7141 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
7142 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
7144 static void kvm_smm_changed(struct kvm_vcpu *vcpu)  in kvm_smm_changed()  argument
7146 	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {  in kvm_smm_changed()
7148 		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);  in kvm_smm_changed()
7151 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_smm_changed()
7154 	kvm_mmu_reset_context(vcpu);  in kvm_smm_changed()
7172 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)  in kvm_vcpu_do_singlestep()  argument
7174 	struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_do_singlestep()
7176 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {  in kvm_vcpu_do_singlestep()
7178 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);  in kvm_vcpu_do_singlestep()
7183 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);  in kvm_vcpu_do_singlestep()
7187 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)  in kvm_skip_emulated_instruction()  argument
7189 	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);  in kvm_skip_emulated_instruction()
7192 	r = kvm_x86_ops.skip_emulated_instruction(vcpu);  in kvm_skip_emulated_instruction()
7205 		r = kvm_vcpu_do_singlestep(vcpu);  in kvm_skip_emulated_instruction()
7210 static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)  in kvm_vcpu_check_breakpoint()  argument
7212 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&  in kvm_vcpu_check_breakpoint()
7213 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {  in kvm_vcpu_check_breakpoint()
7214 		struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_check_breakpoint()
7215 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_breakpoint()
7217 					   vcpu->arch.guest_debug_dr7,  in kvm_vcpu_check_breakpoint()
7218 					   vcpu->arch.eff_db);  in kvm_vcpu_check_breakpoint()
7230 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&  in kvm_vcpu_check_breakpoint()
7231 	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {  in kvm_vcpu_check_breakpoint()
7232 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_breakpoint()
7234 					   vcpu->arch.dr7,  in kvm_vcpu_check_breakpoint()
7235 					   vcpu->arch.db);  in kvm_vcpu_check_breakpoint()
7238 			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);  in kvm_vcpu_check_breakpoint()
7278 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in x86_emulate_instruction()  argument
7282 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_emulate_instruction()
7286 	if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, insn, insn_len)))  in x86_emulate_instruction()
7289 	vcpu->arch.l1tf_flush_l1d = true;  in x86_emulate_instruction()
7295 	write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;  in x86_emulate_instruction()
7296 	vcpu->arch.write_fault_to_shadow_pgtable = false;  in x86_emulate_instruction()
7297 	kvm_clear_exception_queue(vcpu);  in x86_emulate_instruction()
7300 		init_emulate_ctxt(vcpu);  in x86_emulate_instruction()
7309 		    kvm_vcpu_check_breakpoint(vcpu, &r))  in x86_emulate_instruction()
7321 		trace_kvm_emulate_insn_start(vcpu);  in x86_emulate_instruction()
7322 		++vcpu->stat.insn_emulation;  in x86_emulate_instruction()
7326 				kvm_queue_exception(vcpu, UD_VECTOR);  in x86_emulate_instruction()
7329 			if (reexecute_instruction(vcpu, cr2_or_gpa,  in x86_emulate_instruction()
7340 				inject_emulated_exception(vcpu);  in x86_emulate_instruction()
7343 			return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
7349 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in x86_emulate_instruction()
7359 		kvm_rip_write(vcpu, ctxt->_eip);  in x86_emulate_instruction()
7361 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);  in x86_emulate_instruction()
7370 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {  in x86_emulate_instruction()
7371 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in x86_emulate_instruction()
7381 		if (vcpu->arch.mmu->direct_map) {  in x86_emulate_instruction()
7396 		if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,  in x86_emulate_instruction()
7400 		return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
7405 		if (inject_emulated_exception(vcpu))  in x86_emulate_instruction()
7407 	} else if (vcpu->arch.pio.count) {  in x86_emulate_instruction()
7408 		if (!vcpu->arch.pio.in) {  in x86_emulate_instruction()
7410 			vcpu->arch.pio.count = 0;  in x86_emulate_instruction()
7413 			vcpu->arch.complete_userspace_io = complete_emulated_pio;  in x86_emulate_instruction()
7416 	} else if (vcpu->mmio_needed) {  in x86_emulate_instruction()
7417 		++vcpu->stat.mmio_exits;  in x86_emulate_instruction()
7419 		if (!vcpu->mmio_is_write)  in x86_emulate_instruction()
7422 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in x86_emulate_instruction()
7429 		unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);  in x86_emulate_instruction()
7430 		toggle_interruptibility(vcpu, ctxt->interruptibility);  in x86_emulate_instruction()
7431 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in x86_emulate_instruction()
7434 			kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
7435 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))  in x86_emulate_instruction()
7436 				r = kvm_vcpu_do_singlestep(vcpu);  in x86_emulate_instruction()
7438 				kvm_x86_ops.update_emulated_instruction(vcpu);  in x86_emulate_instruction()
7439 			__kvm_set_rflags(vcpu, ctxt->eflags);  in x86_emulate_instruction()
7449 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in x86_emulate_instruction()
7451 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;  in x86_emulate_instruction()
7456 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)  in kvm_emulate_instruction()  argument
7458 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);  in kvm_emulate_instruction()
7462 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,  in kvm_emulate_instruction_from_buffer()  argument
7465 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);  in kvm_emulate_instruction_from_buffer()
7469 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)  in complete_fast_pio_out_port_0x7e()  argument
7471 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out_port_0x7e()
7475 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)  in complete_fast_pio_out()  argument
7477 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out()
7479 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))  in complete_fast_pio_out()
7482 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_out()
7485 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_out()  argument
7488 	unsigned long val = kvm_rax_read(vcpu);  in kvm_fast_pio_out()
7489 	int ret = emulator_pio_out(vcpu, size, port, &val, 1);  in kvm_fast_pio_out()
7499 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {  in kvm_fast_pio_out()
7500 		vcpu->arch.complete_userspace_io =  in kvm_fast_pio_out()
7502 		kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio_out()
7504 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_out()
7505 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;  in kvm_fast_pio_out()
7510 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)  in complete_fast_pio_in()  argument
7515 	BUG_ON(vcpu->arch.pio.count != 1);  in complete_fast_pio_in()
7517 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {  in complete_fast_pio_in()
7518 		vcpu->arch.pio.count = 0;  in complete_fast_pio_in()
7523 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;  in complete_fast_pio_in()
7529 	emulator_pio_in(vcpu, vcpu->arch.pio.size, vcpu->arch.pio.port, &val, 1);  in complete_fast_pio_in()
7530 	kvm_rax_write(vcpu, val);  in complete_fast_pio_in()
7532 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_in()
7535 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_in()  argument
7542 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;  in kvm_fast_pio_in()
7544 	ret = emulator_pio_in(vcpu, size, port, &val, 1);  in kvm_fast_pio_in()
7546 		kvm_rax_write(vcpu, val);  in kvm_fast_pio_in()
7550 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_in()
7551 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;  in kvm_fast_pio_in()
7556 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)  in kvm_fast_pio()  argument
7561 		ret = kvm_fast_pio_in(vcpu, size, port);  in kvm_fast_pio()
7563 		ret = kvm_fast_pio_out(vcpu, size, port);  in kvm_fast_pio()
7564 	return ret && kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio()
7592 	struct kvm_vcpu *vcpu;  in kvm_hyperv_tsc_notifier()  local
7613 		kvm_for_each_vcpu(cpu, vcpu, kvm)  in kvm_hyperv_tsc_notifier()
7614 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_hyperv_tsc_notifier()
7616 		kvm_for_each_vcpu(cpu, vcpu, kvm)  in kvm_hyperv_tsc_notifier()
7617 			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);  in kvm_hyperv_tsc_notifier()
7628 	struct kvm_vcpu *vcpu;  in __kvmclock_cpufreq_notifier()  local
7674 		kvm_for_each_vcpu(i, vcpu, kvm) {  in __kvmclock_cpufreq_notifier()
7675 			if (vcpu->cpu != cpu)  in __kvmclock_cpufreq_notifier()
7677 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in __kvmclock_cpufreq_notifier()
7678 			if (vcpu->cpu != raw_smp_processor_id())  in __kvmclock_cpufreq_notifier()
7784 	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);  in kvm_handle_intel_pt_intr()  local
7786 	kvm_make_request(KVM_REQ_PMI, vcpu);  in kvm_handle_intel_pt_intr()
7788 			(unsigned long *)&vcpu->arch.pmu.global_status);  in kvm_handle_intel_pt_intr()
7803 	struct kvm_vcpu *vcpu;  in pvclock_gtod_update_fn()  local
7808 		kvm_for_each_vcpu(i, vcpu, kvm)  in pvclock_gtod_update_fn()
7809 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in pvclock_gtod_update_fn()
7956 int kvm_vcpu_halt(struct kvm_vcpu *vcpu)  in kvm_vcpu_halt()  argument
7958 	++vcpu->stat.halt_exits;  in kvm_vcpu_halt()
7959 	if (lapic_in_kernel(vcpu)) {  in kvm_vcpu_halt()
7960 		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;  in kvm_vcpu_halt()
7963 		vcpu->run->exit_reason = KVM_EXIT_HLT;  in kvm_vcpu_halt()
7969 int kvm_emulate_halt(struct kvm_vcpu *vcpu)  in kvm_emulate_halt()  argument
7971 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_halt()
7976 	return kvm_vcpu_halt(vcpu) && ret;  in kvm_emulate_halt()
7981 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,  in kvm_pv_clock_pairing()  argument
7997 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);  in kvm_pv_clock_pairing()
8002 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,  in kvm_pv_clock_pairing()
8055 		target = map->phys_map[dest_id]->vcpu;  in kvm_sched_yield()
8063 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)  in kvm_emulate_hypercall()  argument
8068 	if (kvm_hv_hypercall_enabled(vcpu->kvm))  in kvm_emulate_hypercall()
8069 		return kvm_hv_hypercall(vcpu);  in kvm_emulate_hypercall()
8071 	nr = kvm_rax_read(vcpu);  in kvm_emulate_hypercall()
8072 	a0 = kvm_rbx_read(vcpu);  in kvm_emulate_hypercall()
8073 	a1 = kvm_rcx_read(vcpu);  in kvm_emulate_hypercall()
8074 	a2 = kvm_rdx_read(vcpu);  in kvm_emulate_hypercall()
8075 	a3 = kvm_rsi_read(vcpu);  in kvm_emulate_hypercall()
8079 	op_64_bit = is_64_bit_mode(vcpu);  in kvm_emulate_hypercall()
8088 	if (kvm_x86_ops.get_cpl(vcpu) != 0) {  in kvm_emulate_hypercall()
8100 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))  in kvm_emulate_hypercall()
8103 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);  in kvm_emulate_hypercall()
8104 		kvm_sched_yield(vcpu->kvm, a1);  in kvm_emulate_hypercall()
8109 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);  in kvm_emulate_hypercall()
8113 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))  in kvm_emulate_hypercall()
8116 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);  in kvm_emulate_hypercall()
8119 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))  in kvm_emulate_hypercall()
8122 		kvm_sched_yield(vcpu->kvm, a0);  in kvm_emulate_hypercall()
8132 	kvm_rax_write(vcpu, ret);  in kvm_emulate_hypercall()
8134 	++vcpu->stat.hypercalls;  in kvm_emulate_hypercall()
8135 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_hypercall()
8141 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_fix_hypercall()  local
8143 	unsigned long rip = kvm_rip_read(vcpu);  in emulator_fix_hypercall()
8145 	kvm_x86_ops.patch_hypercall(vcpu, instruction);  in emulator_fix_hypercall()
8151 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)  in dm_request_for_irq_injection()  argument
8153 	return vcpu->run->request_interrupt_window &&  in dm_request_for_irq_injection()
8154 		likely(!pic_in_kernel(vcpu->kvm));  in dm_request_for_irq_injection()
8157 static void post_kvm_run_save(struct kvm_vcpu *vcpu)  in post_kvm_run_save()  argument
8159 	struct kvm_run *kvm_run = vcpu->run;  in post_kvm_run_save()
8161 	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;  in post_kvm_run_save()
8162 	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;  in post_kvm_run_save()
8163 	kvm_run->cr8 = kvm_get_cr8(vcpu);  in post_kvm_run_save()
8164 	kvm_run->apic_base = kvm_get_apic_base(vcpu);  in post_kvm_run_save()
8166 		pic_in_kernel(vcpu->kvm) ||  in post_kvm_run_save()
8167 		kvm_vcpu_ready_for_interrupt_injection(vcpu);  in post_kvm_run_save()
8170 static void update_cr8_intercept(struct kvm_vcpu *vcpu)  in update_cr8_intercept()  argument
8177 	if (!lapic_in_kernel(vcpu))  in update_cr8_intercept()
8180 	if (vcpu->arch.apicv_active)  in update_cr8_intercept()
8183 	if (!vcpu->arch.apic->vapic_addr)  in update_cr8_intercept()
8184 		max_irr = kvm_lapic_find_highest_irr(vcpu);  in update_cr8_intercept()
8191 	tpr = kvm_lapic_get_cr8(vcpu);  in update_cr8_intercept()
8193 	kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr);  in update_cr8_intercept()
8196 static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit)  in inject_pending_event()  argument
8203 	if (vcpu->arch.exception.injected) {  in inject_pending_event()
8204 		kvm_x86_ops.queue_exception(vcpu);  in inject_pending_event()
8221 	else if (!vcpu->arch.exception.pending) {  in inject_pending_event()
8222 		if (vcpu->arch.nmi_injected) {  in inject_pending_event()
8223 			kvm_x86_ops.set_nmi(vcpu);  in inject_pending_event()
8225 		} else if (vcpu->arch.interrupt.injected) {  in inject_pending_event()
8226 			kvm_x86_ops.set_irq(vcpu);  in inject_pending_event()
8231 	WARN_ON_ONCE(vcpu->arch.exception.injected &&  in inject_pending_event()
8232 		     vcpu->arch.exception.pending);  in inject_pending_event()
8240 	if (is_guest_mode(vcpu)) {  in inject_pending_event()
8241 		r = kvm_x86_ops.nested_ops->check_events(vcpu);  in inject_pending_event()
8247 	if (vcpu->arch.exception.pending) {  in inject_pending_event()
8248 		trace_kvm_inj_exception(vcpu->arch.exception.nr,  in inject_pending_event()
8249 					vcpu->arch.exception.has_error_code,  in inject_pending_event()
8250 					vcpu->arch.exception.error_code);  in inject_pending_event()
8252 		vcpu->arch.exception.pending = false;  in inject_pending_event()
8253 		vcpu->arch.exception.injected = true;  in inject_pending_event()
8255 		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)  in inject_pending_event()
8256 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |  in inject_pending_event()
8259 		if (vcpu->arch.exception.nr == DB_VECTOR) {  in inject_pending_event()
8260 			kvm_deliver_exception_payload(vcpu);  in inject_pending_event()
8261 			if (vcpu->arch.dr7 & DR7_GD) {  in inject_pending_event()
8262 				vcpu->arch.dr7 &= ~DR7_GD;  in inject_pending_event()
8263 				kvm_update_dr7(vcpu);  in inject_pending_event()
8267 		kvm_x86_ops.queue_exception(vcpu);  in inject_pending_event()
8282 	if (vcpu->arch.smi_pending) {  in inject_pending_event()
8283 		r = can_inject ? kvm_x86_ops.smi_allowed(vcpu, true) : -EBUSY;  in inject_pending_event()
8287 			vcpu->arch.smi_pending = false;  in inject_pending_event()
8288 			++vcpu->arch.smi_count;  in inject_pending_event()
8289 			enter_smm(vcpu);  in inject_pending_event()
8292 			kvm_x86_ops.enable_smi_window(vcpu);  in inject_pending_event()
8295 	if (vcpu->arch.nmi_pending) {  in inject_pending_event()
8296 		r = can_inject ? kvm_x86_ops.nmi_allowed(vcpu, true) : -EBUSY;  in inject_pending_event()
8300 			--vcpu->arch.nmi_pending;  in inject_pending_event()
8301 			vcpu->arch.nmi_injected = true;  in inject_pending_event()
8302 			kvm_x86_ops.set_nmi(vcpu);  in inject_pending_event()
8304 			WARN_ON(kvm_x86_ops.nmi_allowed(vcpu, true) < 0);  in inject_pending_event()
8306 		if (vcpu->arch.nmi_pending)  in inject_pending_event()
8307 			kvm_x86_ops.enable_nmi_window(vcpu);  in inject_pending_event()
8310 	if (kvm_cpu_has_injectable_intr(vcpu)) {  in inject_pending_event()
8311 		r = can_inject ? kvm_x86_ops.interrupt_allowed(vcpu, true) : -EBUSY;  in inject_pending_event()
8315 			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);  in inject_pending_event()
8316 			kvm_x86_ops.set_irq(vcpu);  in inject_pending_event()
8317 			WARN_ON(kvm_x86_ops.interrupt_allowed(vcpu, true) < 0);  in inject_pending_event()
8319 		if (kvm_cpu_has_injectable_intr(vcpu))  in inject_pending_event()
8320 			kvm_x86_ops.enable_irq_window(vcpu);  in inject_pending_event()
8323 	if (is_guest_mode(vcpu) &&  in inject_pending_event()
8325 	    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))  in inject_pending_event()
8328 	WARN_ON(vcpu->arch.exception.pending);  in inject_pending_event()
8336 static void process_nmi(struct kvm_vcpu *vcpu)  in process_nmi()  argument
8345 	if (kvm_x86_ops.get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)  in process_nmi()
8348 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);  in process_nmi()
8349 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);  in process_nmi()
8350 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in process_nmi()
8367 static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)  in enter_smm_save_seg_32()  argument
8372 	kvm_get_segment(vcpu, &seg, n);  in enter_smm_save_seg_32()
8386 static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)  in enter_smm_save_seg_64()  argument
8392 	kvm_get_segment(vcpu, &seg, n);  in enter_smm_save_seg_64()
8403 static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)  in enter_smm_save_state_32()  argument
8410 	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));  in enter_smm_save_state_32()
8411 	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));  in enter_smm_save_state_32()
8412 	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));  in enter_smm_save_state_32()
8413 	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));  in enter_smm_save_state_32()
8416 		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));  in enter_smm_save_state_32()
8418 	kvm_get_dr(vcpu, 6, &val);  in enter_smm_save_state_32()
8420 	kvm_get_dr(vcpu, 7, &val);  in enter_smm_save_state_32()
8423 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);  in enter_smm_save_state_32()
8429 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);  in enter_smm_save_state_32()
8435 	kvm_x86_ops.get_gdt(vcpu, &dt);  in enter_smm_save_state_32()
8439 	kvm_x86_ops.get_idt(vcpu, &dt);  in enter_smm_save_state_32()
8444 		enter_smm_save_seg_32(vcpu, buf, i);  in enter_smm_save_state_32()
8446 	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));  in enter_smm_save_state_32()
8450 	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);  in enter_smm_save_state_32()
8454 static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)  in enter_smm_save_state_64()  argument
8462 		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));  in enter_smm_save_state_64()
8464 	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));  in enter_smm_save_state_64()
8465 	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));  in enter_smm_save_state_64()
8467 	kvm_get_dr(vcpu, 6, &val);  in enter_smm_save_state_64()
8469 	kvm_get_dr(vcpu, 7, &val);  in enter_smm_save_state_64()
8472 	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));  in enter_smm_save_state_64()
8473 	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));  in enter_smm_save_state_64()
8474 	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));  in enter_smm_save_state_64()
8476 	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);  in enter_smm_save_state_64()
8481 	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);  in enter_smm_save_state_64()
8483 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);  in enter_smm_save_state_64()
8489 	kvm_x86_ops.get_idt(vcpu, &dt);  in enter_smm_save_state_64()
8493 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);  in enter_smm_save_state_64()
8499 	kvm_x86_ops.get_gdt(vcpu, &dt);  in enter_smm_save_state_64()
8504 		enter_smm_save_seg_64(vcpu, buf, i);  in enter_smm_save_state_64()
8508 static void enter_smm(struct kvm_vcpu *vcpu)  in enter_smm()  argument
8515 	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);  in enter_smm()
8518 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))  in enter_smm()
8519 		enter_smm_save_state_64(vcpu, buf);  in enter_smm()
8522 		enter_smm_save_state_32(vcpu, buf);  in enter_smm()
8529 	kvm_x86_ops.pre_enter_smm(vcpu, buf);  in enter_smm()
8531 	vcpu->arch.hflags |= HF_SMM_MASK;  in enter_smm()
8532 	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));  in enter_smm()
8534 	if (kvm_x86_ops.get_nmi_mask(vcpu))  in enter_smm()
8535 		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;  in enter_smm()
8537 		kvm_x86_ops.set_nmi_mask(vcpu, true);  in enter_smm()
8539 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);  in enter_smm()
8540 	kvm_rip_write(vcpu, 0x8000);  in enter_smm()
8542 	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);  in enter_smm()
8543 	kvm_x86_ops.set_cr0(vcpu, cr0);  in enter_smm()
8544 	vcpu->arch.cr0 = cr0;  in enter_smm()
8546 	kvm_x86_ops.set_cr4(vcpu, 0);  in enter_smm()
8550 	kvm_x86_ops.set_idt(vcpu, &dt);  in enter_smm()
8552 	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);  in enter_smm()
8554 	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;  in enter_smm()
8555 	cs.base = vcpu->arch.smbase;  in enter_smm()
8572 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);  in enter_smm()
8573 	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);  in enter_smm()
8574 	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);  in enter_smm()
8575 	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);  in enter_smm()
8576 	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);  in enter_smm()
8577 	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);  in enter_smm()
8580 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))  in enter_smm()
8581 		kvm_x86_ops.set_efer(vcpu, 0);  in enter_smm()
8584 	kvm_update_cpuid_runtime(vcpu);  in enter_smm()
8585 	kvm_mmu_reset_context(vcpu);  in enter_smm()
8588 static void process_smi(struct kvm_vcpu *vcpu)  in process_smi()  argument
8590 	vcpu->arch.smi_pending = true;  in process_smi()
8591 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in process_smi()
8612 void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in kvm_vcpu_update_apicv()  argument
8614 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_update_apicv()
8617 	vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);  in kvm_vcpu_update_apicv()
8618 	kvm_apic_update_apicv(vcpu);  in kvm_vcpu_update_apicv()
8619 	kvm_x86_ops.refresh_apicv_exec_ctrl(vcpu);  in kvm_vcpu_update_apicv()
8671 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)  in vcpu_scan_ioapic()  argument
8673 	if (!kvm_apic_present(vcpu))  in vcpu_scan_ioapic()
8676 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);  in vcpu_scan_ioapic()
8678 	if (irqchip_split(vcpu->kvm))  in vcpu_scan_ioapic()
8679 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
8681 		if (vcpu->arch.apicv_active)  in vcpu_scan_ioapic()
8682 			kvm_x86_ops.sync_pir_to_irr(vcpu);  in vcpu_scan_ioapic()
8683 		if (ioapic_in_kernel(vcpu->kvm))  in vcpu_scan_ioapic()
8684 			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
8687 	if (is_guest_mode(vcpu))  in vcpu_scan_ioapic()
8688 		vcpu->arch.load_eoi_exitmap_pending = true;  in vcpu_scan_ioapic()
8690 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);  in vcpu_scan_ioapic()
8693 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)  in vcpu_load_eoi_exitmap()  argument
8697 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))  in vcpu_load_eoi_exitmap()
8700 	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,  in vcpu_load_eoi_exitmap()
8701 		  vcpu_to_synic(vcpu)->vec_bitmap, 256);  in vcpu_load_eoi_exitmap()
8702 	kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap);  in vcpu_load_eoi_exitmap()
8719 void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)  in kvm_vcpu_reload_apic_access_page()  argument
8721 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_reload_apic_access_page()
8727 	kvm_x86_ops.set_apic_access_page_addr(vcpu);  in kvm_vcpu_reload_apic_access_page()
8730 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)  in __kvm_request_immediate_exit()  argument
8732 	smp_send_reschedule(vcpu->cpu);  in __kvm_request_immediate_exit()
8741 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)  in vcpu_enter_guest()  argument
8745 		dm_request_for_irq_injection(vcpu) &&  in vcpu_enter_guest()
8746 		kvm_cpu_accept_dm_intr(vcpu);  in vcpu_enter_guest()
8751 	if (kvm_request_pending(vcpu)) {  in vcpu_enter_guest()
8752 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {  in vcpu_enter_guest()
8753 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {  in vcpu_enter_guest()
8758 		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))  in vcpu_enter_guest()
8759 			kvm_mmu_unload(vcpu);  in vcpu_enter_guest()
8760 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))  in vcpu_enter_guest()
8761 			__kvm_migrate_timers(vcpu);  in vcpu_enter_guest()
8762 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
8763 			kvm_gen_update_masterclock(vcpu->kvm);  in vcpu_enter_guest()
8764 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
8765 			kvm_gen_kvmclock_update(vcpu);  in vcpu_enter_guest()
8766 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {  in vcpu_enter_guest()
8767 			r = kvm_guest_time_update(vcpu);  in vcpu_enter_guest()
8771 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))  in vcpu_enter_guest()
8772 			kvm_mmu_sync_roots(vcpu);  in vcpu_enter_guest()
8773 		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))  in vcpu_enter_guest()
8774 			kvm_mmu_load_pgd(vcpu);  in vcpu_enter_guest()
8775 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {  in vcpu_enter_guest()
8776 			kvm_vcpu_flush_tlb_all(vcpu);  in vcpu_enter_guest()
8779 			kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in vcpu_enter_guest()
8781 		if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))  in vcpu_enter_guest()
8782 			kvm_vcpu_flush_tlb_current(vcpu);  in vcpu_enter_guest()
8783 		if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu))  in vcpu_enter_guest()
8784 			kvm_vcpu_flush_tlb_guest(vcpu);  in vcpu_enter_guest()
8786 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {  in vcpu_enter_guest()
8787 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;  in vcpu_enter_guest()
8791 		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
8792 			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;  in vcpu_enter_guest()
8793 			vcpu->mmio_needed = 0;  in vcpu_enter_guest()
8797 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {  in vcpu_enter_guest()
8799 			vcpu->arch.apf.halted = true;  in vcpu_enter_guest()
8803 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))  in vcpu_enter_guest()
8804 			record_steal_time(vcpu);  in vcpu_enter_guest()
8805 		if (kvm_check_request(KVM_REQ_SMI, vcpu))  in vcpu_enter_guest()
8806 			process_smi(vcpu);  in vcpu_enter_guest()
8807 		if (kvm_check_request(KVM_REQ_NMI, vcpu))  in vcpu_enter_guest()
8808 			process_nmi(vcpu);  in vcpu_enter_guest()
8809 		if (kvm_check_request(KVM_REQ_PMU, vcpu))  in vcpu_enter_guest()
8810 			kvm_pmu_handle_event(vcpu);  in vcpu_enter_guest()
8811 		if (kvm_check_request(KVM_REQ_PMI, vcpu))  in vcpu_enter_guest()
8812 			kvm_pmu_deliver_pmi(vcpu);  in vcpu_enter_guest()
8813 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {  in vcpu_enter_guest()
8814 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);  in vcpu_enter_guest()
8815 			if (test_bit(vcpu->arch.pending_ioapic_eoi,  in vcpu_enter_guest()
8816 				     vcpu->arch.ioapic_handled_vectors)) {  in vcpu_enter_guest()
8817 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;  in vcpu_enter_guest()
8818 				vcpu->run->eoi.vector =  in vcpu_enter_guest()
8819 						vcpu->arch.pending_ioapic_eoi;  in vcpu_enter_guest()
8824 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))  in vcpu_enter_guest()
8825 			vcpu_scan_ioapic(vcpu);  in vcpu_enter_guest()
8826 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))  in vcpu_enter_guest()
8827 			vcpu_load_eoi_exitmap(vcpu);  in vcpu_enter_guest()
8828 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))  in vcpu_enter_guest()
8829 			kvm_vcpu_reload_apic_access_page(vcpu);  in vcpu_enter_guest()
8830 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {  in vcpu_enter_guest()
8831 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
8832 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;  in vcpu_enter_guest()
8836 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {  in vcpu_enter_guest()
8837 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
8838 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;  in vcpu_enter_guest()
8842 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {  in vcpu_enter_guest()
8843 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;  in vcpu_enter_guest()
8844 			vcpu->run->hyperv = vcpu->arch.hyperv.exit;  in vcpu_enter_guest()
8854 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))  in vcpu_enter_guest()
8855 			kvm_hv_process_stimers(vcpu);  in vcpu_enter_guest()
8856 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))  in vcpu_enter_guest()
8857 			kvm_vcpu_update_apicv(vcpu);  in vcpu_enter_guest()
8858 		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))  in vcpu_enter_guest()
8859 			kvm_check_async_pf_completion(vcpu);  in vcpu_enter_guest()
8860 		if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))  in vcpu_enter_guest()
8861 			kvm_x86_ops.msr_filter_changed(vcpu);  in vcpu_enter_guest()
8864 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {  in vcpu_enter_guest()
8865 		++vcpu->stat.req_event;  in vcpu_enter_guest()
8866 		kvm_apic_accept_events(vcpu);  in vcpu_enter_guest()
8867 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {  in vcpu_enter_guest()
8872 		inject_pending_event(vcpu, &req_immediate_exit);  in vcpu_enter_guest()
8874 			kvm_x86_ops.enable_irq_window(vcpu);  in vcpu_enter_guest()
8876 		if (kvm_lapic_enabled(vcpu)) {  in vcpu_enter_guest()
8877 			update_cr8_intercept(vcpu);  in vcpu_enter_guest()
8878 			kvm_lapic_sync_to_vapic(vcpu);  in vcpu_enter_guest()
8882 	r = kvm_mmu_reload(vcpu);  in vcpu_enter_guest()
8889 	kvm_x86_ops.prepare_guest_switch(vcpu);  in vcpu_enter_guest()
8897 	vcpu->mode = IN_GUEST_MODE;  in vcpu_enter_guest()
8899 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);  in vcpu_enter_guest()
8919 	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)  in vcpu_enter_guest()
8920 		kvm_x86_ops.sync_pir_to_irr(vcpu);  in vcpu_enter_guest()
8922 	if (kvm_vcpu_exit_request(vcpu)) {  in vcpu_enter_guest()
8923 		vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
8927 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);  in vcpu_enter_guest()
8933 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
8934 		kvm_x86_ops.request_immediate_exit(vcpu);  in vcpu_enter_guest()
8937 	trace_kvm_entry(vcpu);  in vcpu_enter_guest()
8943 	if (unlikely(vcpu->arch.switch_db_regs)) {  in vcpu_enter_guest()
8945 		set_debugreg(vcpu->arch.eff_db[0], 0);  in vcpu_enter_guest()
8946 		set_debugreg(vcpu->arch.eff_db[1], 1);  in vcpu_enter_guest()
8947 		set_debugreg(vcpu->arch.eff_db[2], 2);  in vcpu_enter_guest()
8948 		set_debugreg(vcpu->arch.eff_db[3], 3);  in vcpu_enter_guest()
8949 		set_debugreg(vcpu->arch.dr6, 6);  in vcpu_enter_guest()
8950 		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;  in vcpu_enter_guest()
8953 	exit_fastpath = kvm_x86_ops.run(vcpu);  in vcpu_enter_guest()
8961 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {  in vcpu_enter_guest()
8962 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);  in vcpu_enter_guest()
8963 		kvm_x86_ops.sync_dirty_debug_regs(vcpu);  in vcpu_enter_guest()
8964 		kvm_update_dr0123(vcpu);  in vcpu_enter_guest()
8965 		kvm_update_dr7(vcpu);  in vcpu_enter_guest()
8966 		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;  in vcpu_enter_guest()
8979 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;  in vcpu_enter_guest()
8980 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());  in vcpu_enter_guest()
8982 	vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
8985 	kvm_x86_ops.handle_exit_irqoff(vcpu);  in vcpu_enter_guest()
8994 	kvm_before_interrupt(vcpu);  in vcpu_enter_guest()
8996 	++vcpu->stat.exits;  in vcpu_enter_guest()
8998 	kvm_after_interrupt(vcpu);  in vcpu_enter_guest()
9000 	if (lapic_in_kernel(vcpu)) {  in vcpu_enter_guest()
9001 		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;  in vcpu_enter_guest()
9003 			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);  in vcpu_enter_guest()
9004 			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;  in vcpu_enter_guest()
9011 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);  in vcpu_enter_guest()
9017 		unsigned long rip = kvm_rip_read(vcpu);  in vcpu_enter_guest()
9021 	if (unlikely(vcpu->arch.tsc_always_catchup))  in vcpu_enter_guest()
9022 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in vcpu_enter_guest()
9024 	if (vcpu->arch.apic_attention)  in vcpu_enter_guest()
9025 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
9027 	r = kvm_x86_ops.handle_exit(vcpu, exit_fastpath);  in vcpu_enter_guest()
9032 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
9033 	kvm_x86_ops.cancel_injection(vcpu);  in vcpu_enter_guest()
9034 	if (unlikely(vcpu->arch.apic_attention))  in vcpu_enter_guest()
9035 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
9040 static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)  in vcpu_block()  argument
9042 	if (!kvm_arch_vcpu_runnable(vcpu) &&  in vcpu_block()
9043 	    (!kvm_x86_ops.pre_block || kvm_x86_ops.pre_block(vcpu) == 0)) {  in vcpu_block()
9044 		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);  in vcpu_block()
9045 		kvm_vcpu_block(vcpu);  in vcpu_block()
9046 		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);  in vcpu_block()
9049 			kvm_x86_ops.post_block(vcpu);  in vcpu_block()
9051 		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))  in vcpu_block()
9055 	kvm_apic_accept_events(vcpu);  in vcpu_block()
9056 	switch(vcpu->arch.mp_state) {  in vcpu_block()
9058 		vcpu->arch.pv.pv_unhalted = false;  in vcpu_block()
9059 		vcpu->arch.mp_state =  in vcpu_block()
9063 		vcpu->arch.apf.halted = false;  in vcpu_block()
9073 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)  in kvm_vcpu_running()  argument
9075 	if (is_guest_mode(vcpu))  in kvm_vcpu_running()
9076 		kvm_x86_ops.nested_ops->check_events(vcpu);  in kvm_vcpu_running()
9078 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&  in kvm_vcpu_running()
9079 		!vcpu->arch.apf.halted);  in kvm_vcpu_running()
9082 static int vcpu_run(struct kvm_vcpu *vcpu)  in vcpu_run()  argument
9085 	struct kvm *kvm = vcpu->kvm;  in vcpu_run()
9087 	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);  in vcpu_run()
9088 	vcpu->arch.l1tf_flush_l1d = true;  in vcpu_run()
9091 		if (kvm_vcpu_running(vcpu)) {  in vcpu_run()
9092 			r = vcpu_enter_guest(vcpu);  in vcpu_run()
9094 			r = vcpu_block(kvm, vcpu);  in vcpu_run()
9100 		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);  in vcpu_run()
9101 		if (kvm_cpu_has_pending_timer(vcpu))  in vcpu_run()
9102 			kvm_inject_pending_timer_irqs(vcpu);  in vcpu_run()
9104 		if (dm_request_for_irq_injection(vcpu) &&  in vcpu_run()
9105 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {  in vcpu_run()
9107 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;  in vcpu_run()
9108 			++vcpu->stat.request_irq_exits;  in vcpu_run()
9113 			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);  in vcpu_run()
9114 			r = xfer_to_guest_mode_handle_work(vcpu);  in vcpu_run()
9117 			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);  in vcpu_run()
9121 	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);  in vcpu_run()
9126 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)  in complete_emulated_io()  argument
9130 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);  in complete_emulated_io()
9131 	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);  in complete_emulated_io()
9132 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);  in complete_emulated_io()
9136 static int complete_emulated_pio(struct kvm_vcpu *vcpu)  in complete_emulated_pio()  argument
9138 	BUG_ON(!vcpu->arch.pio.count);  in complete_emulated_pio()
9140 	return complete_emulated_io(vcpu);  in complete_emulated_pio()
9161 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_emulated_mmio()  argument
9163 	struct kvm_run *run = vcpu->run;  in complete_emulated_mmio()
9167 	BUG_ON(!vcpu->mmio_needed);  in complete_emulated_mmio()
9170 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_emulated_mmio()
9172 	if (!vcpu->mmio_is_write)  in complete_emulated_mmio()
9178 		vcpu->mmio_cur_fragment++;  in complete_emulated_mmio()
9186 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_emulated_mmio()
9187 		vcpu->mmio_needed = 0;  in complete_emulated_mmio()
9190 		if (vcpu->mmio_is_write)  in complete_emulated_mmio()
9192 		vcpu->mmio_read_completed = 1;  in complete_emulated_mmio()
9193 		return complete_emulated_io(vcpu);  in complete_emulated_mmio()
9198 	if (vcpu->mmio_is_write)  in complete_emulated_mmio()
9201 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_emulated_mmio()
9202 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in complete_emulated_mmio()
9220 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_load_guest_fpu()  argument
9224 	kvm_save_current_fpu(vcpu->arch.user_fpu);  in kvm_load_guest_fpu()
9227 	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,  in kvm_load_guest_fpu()
9237 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_put_guest_fpu()  argument
9241 	kvm_save_current_fpu(vcpu->arch.guest_fpu);  in kvm_put_guest_fpu()
9243 	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);  in kvm_put_guest_fpu()
9248 	++vcpu->stat.fpu_reload;  in kvm_put_guest_fpu()
9252 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_ioctl_run()  argument
9254 	struct kvm_run *kvm_run = vcpu->run;  in kvm_arch_vcpu_ioctl_run()
9257 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_run()
9258 	kvm_sigset_activate(vcpu);  in kvm_arch_vcpu_ioctl_run()
9259 	kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
9261 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {  in kvm_arch_vcpu_ioctl_run()
9266 		kvm_vcpu_block(vcpu);  in kvm_arch_vcpu_ioctl_run()
9267 		kvm_apic_accept_events(vcpu);  in kvm_arch_vcpu_ioctl_run()
9268 		kvm_clear_request(KVM_REQ_UNHALT, vcpu);  in kvm_arch_vcpu_ioctl_run()
9273 			++vcpu->stat.signal_exits;  in kvm_arch_vcpu_ioctl_run()
9284 		r = sync_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
9290 	if (!lapic_in_kernel(vcpu)) {  in kvm_arch_vcpu_ioctl_run()
9291 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {  in kvm_arch_vcpu_ioctl_run()
9297 	if (unlikely(vcpu->arch.complete_userspace_io)) {  in kvm_arch_vcpu_ioctl_run()
9298 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;  in kvm_arch_vcpu_ioctl_run()
9299 		vcpu->arch.complete_userspace_io = NULL;  in kvm_arch_vcpu_ioctl_run()
9300 		r = cui(vcpu);  in kvm_arch_vcpu_ioctl_run()
9304 		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);  in kvm_arch_vcpu_ioctl_run()
9309 		r = vcpu_run(vcpu);  in kvm_arch_vcpu_ioctl_run()
9312 	kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
9314 		store_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
9315 	post_kvm_run_save(vcpu);  in kvm_arch_vcpu_ioctl_run()
9316 	kvm_sigset_deactivate(vcpu);  in kvm_arch_vcpu_ioctl_run()
9318 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_run()
9322 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __get_regs()  argument
9324 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {  in __get_regs()
9332 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);  in __get_regs()
9333 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __get_regs()
9335 	regs->rax = kvm_rax_read(vcpu);  in __get_regs()
9336 	regs->rbx = kvm_rbx_read(vcpu);  in __get_regs()
9337 	regs->rcx = kvm_rcx_read(vcpu);  in __get_regs()
9338 	regs->rdx = kvm_rdx_read(vcpu);  in __get_regs()
9339 	regs->rsi = kvm_rsi_read(vcpu);  in __get_regs()
9340 	regs->rdi = kvm_rdi_read(vcpu);  in __get_regs()
9341 	regs->rsp = kvm_rsp_read(vcpu);  in __get_regs()
9342 	regs->rbp = kvm_rbp_read(vcpu);  in __get_regs()
9344 	regs->r8 = kvm_r8_read(vcpu);  in __get_regs()
9345 	regs->r9 = kvm_r9_read(vcpu);  in __get_regs()
9346 	regs->r10 = kvm_r10_read(vcpu);  in __get_regs()
9347 	regs->r11 = kvm_r11_read(vcpu);  in __get_regs()
9348 	regs->r12 = kvm_r12_read(vcpu);  in __get_regs()
9349 	regs->r13 = kvm_r13_read(vcpu);  in __get_regs()
9350 	regs->r14 = kvm_r14_read(vcpu);  in __get_regs()
9351 	regs->r15 = kvm_r15_read(vcpu);  in __get_regs()
9354 	regs->rip = kvm_rip_read(vcpu);  in __get_regs()
9355 	regs->rflags = kvm_get_rflags(vcpu);  in __get_regs()
9358 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_get_regs()  argument
9360 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
9361 	__get_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_get_regs()
9362 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
9366 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __set_regs()  argument
9368 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;  in __set_regs()
9369 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __set_regs()
9371 	kvm_rax_write(vcpu, regs->rax);  in __set_regs()
9372 	kvm_rbx_write(vcpu, regs->rbx);  in __set_regs()
9373 	kvm_rcx_write(vcpu, regs->rcx);  in __set_regs()
9374 	kvm_rdx_write(vcpu, regs->rdx);  in __set_regs()
9375 	kvm_rsi_write(vcpu, regs->rsi);  in __set_regs()
9376 	kvm_rdi_write(vcpu, regs->rdi);  in __set_regs()
9377 	kvm_rsp_write(vcpu, regs->rsp);  in __set_regs()
9378 	kvm_rbp_write(vcpu, regs->rbp);  in __set_regs()
9380 	kvm_r8_write(vcpu, regs->r8);  in __set_regs()
9381 	kvm_r9_write(vcpu, regs->r9);  in __set_regs()
9382 	kvm_r10_write(vcpu, regs->r10);  in __set_regs()
9383 	kvm_r11_write(vcpu, regs->r11);  in __set_regs()
9384 	kvm_r12_write(vcpu, regs->r12);  in __set_regs()
9385 	kvm_r13_write(vcpu, regs->r13);  in __set_regs()
9386 	kvm_r14_write(vcpu, regs->r14);  in __set_regs()
9387 	kvm_r15_write(vcpu, regs->r15);  in __set_regs()
9390 	kvm_rip_write(vcpu, regs->rip);  in __set_regs()
9391 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);  in __set_regs()
9393 	vcpu->arch.exception.pending = false;  in __set_regs()
9395 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_regs()
9398 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_set_regs()  argument
9400 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
9401 	__set_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_set_regs()
9402 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
9406 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)  in kvm_get_cs_db_l_bits()  argument
9410 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_get_cs_db_l_bits()
9416 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs()  argument
9420 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __get_sregs()
9421 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __get_sregs()
9422 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __get_sregs()
9423 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __get_sregs()
9424 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __get_sregs()
9425 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __get_sregs()
9427 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __get_sregs()
9428 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __get_sregs()
9430 	kvm_x86_ops.get_idt(vcpu, &dt);  in __get_sregs()
9433 	kvm_x86_ops.get_gdt(vcpu, &dt);  in __get_sregs()
9437 	sregs->cr0 = kvm_read_cr0(vcpu);  in __get_sregs()
9438 	sregs->cr2 = vcpu->arch.cr2;  in __get_sregs()
9439 	sregs->cr3 = kvm_read_cr3(vcpu);  in __get_sregs()
9440 	sregs->cr4 = kvm_read_cr4(vcpu);  in __get_sregs()
9441 	sregs->cr8 = kvm_get_cr8(vcpu);  in __get_sregs()
9442 	sregs->efer = vcpu->arch.efer;  in __get_sregs()
9443 	sregs->apic_base = kvm_get_apic_base(vcpu);  in __get_sregs()
9447 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)  in __get_sregs()
9448 		set_bit(vcpu->arch.interrupt.nr,  in __get_sregs()
9452 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_sregs()  argument
9455 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
9456 	__get_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_get_sregs()
9457 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
9461 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_mpstate()  argument
9464 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
9466 		kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
9468 	kvm_apic_accept_events(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
9469 	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&  in kvm_arch_vcpu_ioctl_get_mpstate()
9470 					vcpu->arch.pv.pv_unhalted)  in kvm_arch_vcpu_ioctl_get_mpstate()
9473 		mp_state->mp_state = vcpu->arch.mp_state;  in kvm_arch_vcpu_ioctl_get_mpstate()
9476 		kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
9477 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
9481 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_mpstate()  argument
9486 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
9488 	if (!lapic_in_kernel(vcpu) &&  in kvm_arch_vcpu_ioctl_set_mpstate()
9497 	if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&  in kvm_arch_vcpu_ioctl_set_mpstate()
9503 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;  in kvm_arch_vcpu_ioctl_set_mpstate()
9504 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);  in kvm_arch_vcpu_ioctl_set_mpstate()
9506 		vcpu->arch.mp_state = mp_state->mp_state;  in kvm_arch_vcpu_ioctl_set_mpstate()
9507 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
9511 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
9515 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,  in kvm_task_switch()  argument
9518 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_task_switch()
9521 	init_emulate_ctxt(vcpu);  in kvm_task_switch()
9526 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in kvm_task_switch()
9527 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in kvm_task_switch()
9528 		vcpu->run->internal.ndata = 0;  in kvm_task_switch()
9532 	kvm_rip_write(vcpu, ctxt->eip);  in kvm_task_switch()
9533 	kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_task_switch()
9538 static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in kvm_valid_sregs()  argument
9558 	return kvm_valid_cr4(vcpu, sregs->cr4);  in kvm_valid_sregs()
9561 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __set_sregs()  argument
9570 	if (kvm_valid_sregs(vcpu, sregs))  in __set_sregs()
9575 	if (kvm_set_apic_base(vcpu, &apic_base_msr))  in __set_sregs()
9580 	kvm_x86_ops.set_idt(vcpu, &dt);  in __set_sregs()
9583 	kvm_x86_ops.set_gdt(vcpu, &dt);  in __set_sregs()
9585 	vcpu->arch.cr2 = sregs->cr2;  in __set_sregs()
9586 	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;  in __set_sregs()
9587 	vcpu->arch.cr3 = sregs->cr3;  in __set_sregs()
9588 	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);  in __set_sregs()
9590 	kvm_set_cr8(vcpu, sregs->cr8);  in __set_sregs()
9592 	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;  in __set_sregs()
9593 	kvm_x86_ops.set_efer(vcpu, sregs->efer);  in __set_sregs()
9595 	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;  in __set_sregs()
9596 	kvm_x86_ops.set_cr0(vcpu, sregs->cr0);  in __set_sregs()
9597 	vcpu->arch.cr0 = sregs->cr0;  in __set_sregs()
9599 	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;  in __set_sregs()
9600 	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &  in __set_sregs()
9602 	kvm_x86_ops.set_cr4(vcpu, sregs->cr4);  in __set_sregs()
9604 		kvm_update_cpuid_runtime(vcpu);  in __set_sregs()
9606 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in __set_sregs()
9607 	if (is_pae_paging(vcpu)) {  in __set_sregs()
9608 		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));  in __set_sregs()
9611 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in __set_sregs()
9614 		kvm_mmu_reset_context(vcpu);  in __set_sregs()
9620 		kvm_queue_interrupt(vcpu, pending_vec, false);  in __set_sregs()
9624 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __set_sregs()
9625 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __set_sregs()
9626 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __set_sregs()
9627 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __set_sregs()
9628 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __set_sregs()
9629 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __set_sregs()
9631 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __set_sregs()
9632 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __set_sregs()
9634 	update_cr8_intercept(vcpu);  in __set_sregs()
9637 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&  in __set_sregs()
9639 	    !is_protmode(vcpu))  in __set_sregs()
9640 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in __set_sregs()
9642 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_sregs()
9649 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_sregs()  argument
9654 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
9655 	ret = __set_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_set_sregs()
9656 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
9660 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_guest_debug()  argument
9666 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9670 		if (vcpu->arch.exception.pending)  in kvm_arch_vcpu_ioctl_set_guest_debug()
9673 			kvm_queue_exception(vcpu, DB_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9675 			kvm_queue_exception(vcpu, BP_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9682 	rflags = kvm_get_rflags(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9684 	vcpu->guest_debug = dbg->control;  in kvm_arch_vcpu_ioctl_set_guest_debug()
9685 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))  in kvm_arch_vcpu_ioctl_set_guest_debug()
9686 		vcpu->guest_debug = 0;  in kvm_arch_vcpu_ioctl_set_guest_debug()
9688 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {  in kvm_arch_vcpu_ioctl_set_guest_debug()
9690 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
9691 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];  in kvm_arch_vcpu_ioctl_set_guest_debug()
9694 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
9696 	kvm_update_dr7(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9698 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_arch_vcpu_ioctl_set_guest_debug()
9699 		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +  in kvm_arch_vcpu_ioctl_set_guest_debug()
9700 			get_segment_base(vcpu, VCPU_SREG_CS);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9706 	kvm_set_rflags(vcpu, rflags);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9708 	kvm_x86_ops.update_exception_bitmap(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9713 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
9720 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_translate()  argument
9727 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_translate()
9729 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl_translate()
9730 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);  in kvm_arch_vcpu_ioctl_translate()
9731 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl_translate()
9737 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_translate()
9741 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_get_fpu()  argument
9745 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
9747 	fxsave = &vcpu->arch.guest_fpu->state.fxsave;  in kvm_arch_vcpu_ioctl_get_fpu()
9757 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
9761 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_set_fpu()  argument
9765 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
9767 	fxsave = &vcpu->arch.guest_fpu->state.fxsave;  in kvm_arch_vcpu_ioctl_set_fpu()
9778 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
9782 static void store_regs(struct kvm_vcpu *vcpu)  in store_regs()  argument
9786 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)  in store_regs()
9787 		__get_regs(vcpu, &vcpu->run->s.regs.regs);  in store_regs()
9789 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)  in store_regs()
9790 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);  in store_regs()
9792 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)  in store_regs()
9794 				vcpu, &vcpu->run->s.regs.events);  in store_regs()
9797 static int sync_regs(struct kvm_vcpu *vcpu)  in sync_regs()  argument
9799 	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)  in sync_regs()
9802 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {  in sync_regs()
9803 		__set_regs(vcpu, &vcpu->run->s.regs.regs);  in sync_regs()
9804 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;  in sync_regs()
9806 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {  in sync_regs()
9807 		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))  in sync_regs()
9809 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;  in sync_regs()
9811 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {  in sync_regs()
9813 				vcpu, &vcpu->run->s.regs.events))  in sync_regs()
9815 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;  in sync_regs()
9821 static void fx_init(struct kvm_vcpu *vcpu)  in fx_init()  argument
9823 	fpstate_init(&vcpu->arch.guest_fpu->state);  in fx_init()
9825 		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =  in fx_init()
9831 	vcpu->arch.xcr0 = XFEATURE_MASK_FP;  in fx_init()
9833 	vcpu->arch.cr0 |= X86_CR0_ET;  in fx_init()
9845 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_create()  argument
9850 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))  in kvm_arch_vcpu_create()
9851 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_vcpu_create()
9853 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;  in kvm_arch_vcpu_create()
9855 	kvm_set_tsc_khz(vcpu, max_tsc_khz);  in kvm_arch_vcpu_create()
9857 	r = kvm_mmu_create(vcpu);  in kvm_arch_vcpu_create()
9861 	if (irqchip_in_kernel(vcpu->kvm)) {  in kvm_arch_vcpu_create()
9862 		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);  in kvm_arch_vcpu_create()
9865 		if (kvm_apicv_activated(vcpu->kvm))  in kvm_arch_vcpu_create()
9866 			vcpu->arch.apicv_active = true;  in kvm_arch_vcpu_create()
9875 	vcpu->arch.pio_data = page_address(page);  in kvm_arch_vcpu_create()
9877 	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,  in kvm_arch_vcpu_create()
9879 	if (!vcpu->arch.mce_banks)  in kvm_arch_vcpu_create()
9881 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;  in kvm_arch_vcpu_create()
9883 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,  in kvm_arch_vcpu_create()
9887 	if (!alloc_emulate_ctxt(vcpu))  in kvm_arch_vcpu_create()
9890 	vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,  in kvm_arch_vcpu_create()
9892 	if (!vcpu->arch.user_fpu) {  in kvm_arch_vcpu_create()
9897 	vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,  in kvm_arch_vcpu_create()
9899 	if (!vcpu->arch.guest_fpu) {  in kvm_arch_vcpu_create()
9903 	fx_init(vcpu);  in kvm_arch_vcpu_create()
9905 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);  in kvm_arch_vcpu_create()
9907 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;  in kvm_arch_vcpu_create()
9909 	kvm_async_pf_hash_reset(vcpu);  in kvm_arch_vcpu_create()
9910 	kvm_pmu_init(vcpu);  in kvm_arch_vcpu_create()
9912 	vcpu->arch.pending_external_vector = -1;  in kvm_arch_vcpu_create()
9913 	vcpu->arch.preempted_in_kernel = false;  in kvm_arch_vcpu_create()
9915 	kvm_hv_vcpu_init(vcpu);  in kvm_arch_vcpu_create()
9917 	r = kvm_x86_ops.vcpu_create(vcpu);  in kvm_arch_vcpu_create()
9921 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();  in kvm_arch_vcpu_create()
9922 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;  in kvm_arch_vcpu_create()
9923 	kvm_vcpu_mtrr_init(vcpu);  in kvm_arch_vcpu_create()
9924 	vcpu_load(vcpu);  in kvm_arch_vcpu_create()
9925 	kvm_vcpu_reset(vcpu, false);  in kvm_arch_vcpu_create()
9926 	kvm_init_mmu(vcpu, false);  in kvm_arch_vcpu_create()
9927 	vcpu_put(vcpu);  in kvm_arch_vcpu_create()
9931 	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);  in kvm_arch_vcpu_create()
9933 	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);  in kvm_arch_vcpu_create()
9935 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_create()
9937 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_create()
9939 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_create()
9941 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_create()
9943 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_create()
9945 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_create()
9949 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_postcreate()  argument
9951 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_vcpu_postcreate()
9953 	kvm_hv_vcpu_postcreate(vcpu);  in kvm_arch_vcpu_postcreate()
9955 	if (mutex_lock_killable(&vcpu->mutex))  in kvm_arch_vcpu_postcreate()
9957 	vcpu_load(vcpu);  in kvm_arch_vcpu_postcreate()
9958 	kvm_synchronize_tsc(vcpu, 0);  in kvm_arch_vcpu_postcreate()
9959 	vcpu_put(vcpu);  in kvm_arch_vcpu_postcreate()
9962 	vcpu->arch.msr_kvm_poll_control = 1;  in kvm_arch_vcpu_postcreate()
9964 	mutex_unlock(&vcpu->mutex);  in kvm_arch_vcpu_postcreate()
9966 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)  in kvm_arch_vcpu_postcreate()
9971 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_destroy()  argument
9973 	struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache;  in kvm_arch_vcpu_destroy()
9978 	kvmclock_reset(vcpu);  in kvm_arch_vcpu_destroy()
9980 	kvm_x86_ops.vcpu_free(vcpu);  in kvm_arch_vcpu_destroy()
9982 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_destroy()
9983 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_destroy()
9984 	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);  in kvm_arch_vcpu_destroy()
9985 	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);  in kvm_arch_vcpu_destroy()
9987 	kvm_hv_vcpu_uninit(vcpu);  in kvm_arch_vcpu_destroy()
9988 	kvm_pmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
9989 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_destroy()
9990 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_destroy()
9991 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_destroy()
9992 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
9993 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_destroy()
9994 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_destroy()
9995 	kvfree(vcpu->arch.cpuid_entries);  in kvm_arch_vcpu_destroy()
9996 	if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_destroy()
10000 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)  in kvm_vcpu_reset()  argument
10002 	kvm_lapic_reset(vcpu, init_event);  in kvm_vcpu_reset()
10004 	vcpu->arch.hflags = 0;  in kvm_vcpu_reset()
10006 	vcpu->arch.smi_pending = 0;  in kvm_vcpu_reset()
10007 	vcpu->arch.smi_count = 0;  in kvm_vcpu_reset()
10008 	atomic_set(&vcpu->arch.nmi_queued, 0);  in kvm_vcpu_reset()
10009 	vcpu->arch.nmi_pending = 0;  in kvm_vcpu_reset()
10010 	vcpu->arch.nmi_injected = false;  in kvm_vcpu_reset()
10011 	kvm_clear_interrupt_queue(vcpu);  in kvm_vcpu_reset()
10012 	kvm_clear_exception_queue(vcpu);  in kvm_vcpu_reset()
10014 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));  in kvm_vcpu_reset()
10015 	kvm_update_dr0123(vcpu);  in kvm_vcpu_reset()
10016 	vcpu->arch.dr6 = DR6_INIT;  in kvm_vcpu_reset()
10017 	vcpu->arch.dr7 = DR7_FIXED_1;  in kvm_vcpu_reset()
10018 	kvm_update_dr7(vcpu);  in kvm_vcpu_reset()
10020 	vcpu->arch.cr2 = 0;  in kvm_vcpu_reset()
10022 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_reset()
10023 	vcpu->arch.apf.msr_en_val = 0;  in kvm_vcpu_reset()
10024 	vcpu->arch.apf.msr_int_val = 0;  in kvm_vcpu_reset()
10025 	vcpu->arch.st.msr_val = 0;  in kvm_vcpu_reset()
10027 	kvmclock_reset(vcpu);  in kvm_vcpu_reset()
10029 	kvm_clear_async_pf_completion_queue(vcpu);  in kvm_vcpu_reset()
10030 	kvm_async_pf_hash_reset(vcpu);  in kvm_vcpu_reset()
10031 	vcpu->arch.apf.halted = false;  in kvm_vcpu_reset()
10041 			kvm_put_guest_fpu(vcpu);  in kvm_vcpu_reset()
10042 		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,  in kvm_vcpu_reset()
10046 		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,  in kvm_vcpu_reset()
10051 			kvm_load_guest_fpu(vcpu);  in kvm_vcpu_reset()
10055 		kvm_pmu_reset(vcpu);  in kvm_vcpu_reset()
10056 		vcpu->arch.smbase = 0x30000;  in kvm_vcpu_reset()
10058 		vcpu->arch.msr_misc_features_enables = 0;  in kvm_vcpu_reset()
10060 		vcpu->arch.xcr0 = XFEATURE_MASK_FP;  in kvm_vcpu_reset()
10063 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));  in kvm_vcpu_reset()
10064 	vcpu->arch.regs_avail = ~0;  in kvm_vcpu_reset()
10065 	vcpu->arch.regs_dirty = ~0;  in kvm_vcpu_reset()
10067 	vcpu->arch.ia32_xss = 0;  in kvm_vcpu_reset()
10069 	kvm_x86_ops.vcpu_reset(vcpu, init_event);  in kvm_vcpu_reset()
10072 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)  in kvm_vcpu_deliver_sipi_vector()  argument
10076 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
10079 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
10080 	kvm_rip_write(vcpu, 0);  in kvm_vcpu_deliver_sipi_vector()
10086 	struct kvm_vcpu *vcpu;  in kvm_arch_hardware_enable()  local
10101 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
10102 			if (!stable && vcpu->cpu == smp_processor_id())  in kvm_arch_hardware_enable()
10103 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
10104 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {  in kvm_arch_hardware_enable()
10106 				if (vcpu->arch.last_host_tsc > max_tsc)  in kvm_arch_hardware_enable()
10107 					max_tsc = vcpu->arch.last_host_tsc;  in kvm_arch_hardware_enable()
10154 			kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
10155 				vcpu->arch.tsc_offset_adjustment += delta_cyc;  in kvm_arch_hardware_enable()
10156 				vcpu->arch.last_host_tsc = local_tsc;  in kvm_arch_hardware_enable()
10157 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
10240 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_reset_bsp()  argument
10242 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;  in kvm_vcpu_is_reset_bsp()
10246 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_bsp()  argument
10248 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;  in kvm_vcpu_is_bsp()
10254 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_sched_in()  argument
10256 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);  in kvm_arch_sched_in()
10258 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_arch_sched_in()
10261 		kvm_make_request(KVM_REQ_PMU, vcpu);  in kvm_arch_sched_in()
10263 	kvm_x86_ops.sched_in(vcpu, cpu);  in kvm_arch_sched_in()
10315 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)  in kvm_unload_vcpu_mmu()  argument
10317 	vcpu_load(vcpu);  in kvm_unload_vcpu_mmu()
10318 	kvm_mmu_unload(vcpu);  in kvm_unload_vcpu_mmu()
10319 	vcpu_put(vcpu);  in kvm_unload_vcpu_mmu()
10325 	struct kvm_vcpu *vcpu;  in kvm_free_vcpus()  local
10330 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_free_vcpus()
10331 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_free_vcpus()
10332 		kvm_unload_vcpu_mmu(vcpu);  in kvm_free_vcpus()
10334 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_free_vcpus()
10335 		kvm_vcpu_destroy(vcpu);  in kvm_free_vcpus()
10531 	struct kvm_vcpu *vcpu;  in kvm_arch_memslots_updated()  local
10541 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_memslots_updated()
10542 		kvm_vcpu_kick(vcpu);  in kvm_arch_memslots_updated()
10672 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)  in kvm_guest_apic_has_interrupt()  argument
10674 	return (is_guest_mode(vcpu) &&  in kvm_guest_apic_has_interrupt()
10676 			kvm_x86_ops.guest_apic_has_interrupt(vcpu));  in kvm_guest_apic_has_interrupt()
10679 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)  in kvm_vcpu_has_events()  argument
10681 	if (!list_empty_careful(&vcpu->async_pf.done))  in kvm_vcpu_has_events()
10684 	if (kvm_apic_has_events(vcpu))  in kvm_vcpu_has_events()
10687 	if (vcpu->arch.pv.pv_unhalted)  in kvm_vcpu_has_events()
10690 	if (vcpu->arch.exception.pending)  in kvm_vcpu_has_events()
10693 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_vcpu_has_events()
10694 	    (vcpu->arch.nmi_pending &&  in kvm_vcpu_has_events()
10695 	     kvm_x86_ops.nmi_allowed(vcpu, false)))  in kvm_vcpu_has_events()
10698 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_vcpu_has_events()
10699 	    (vcpu->arch.smi_pending &&  in kvm_vcpu_has_events()
10700 	     kvm_x86_ops.smi_allowed(vcpu, false)))  in kvm_vcpu_has_events()
10703 	if (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_has_events()
10704 	    (kvm_cpu_has_interrupt(vcpu) ||  in kvm_vcpu_has_events()
10705 	    kvm_guest_apic_has_interrupt(vcpu)))  in kvm_vcpu_has_events()
10708 	if (kvm_hv_has_stimer_pending(vcpu))  in kvm_vcpu_has_events()
10711 	if (is_guest_mode(vcpu) &&  in kvm_vcpu_has_events()
10713 	    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))  in kvm_vcpu_has_events()
10719 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_runnable()  argument
10721 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);  in kvm_arch_vcpu_runnable()
10724 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_dy_runnable()  argument
10726 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))  in kvm_arch_dy_runnable()
10729 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_arch_dy_runnable()
10730 		kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_arch_dy_runnable()
10731 		 kvm_test_request(KVM_REQ_EVENT, vcpu))  in kvm_arch_dy_runnable()
10734 	if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu))  in kvm_arch_dy_runnable()
10740 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_in_kernel()  argument
10742 	return vcpu->arch.preempted_in_kernel;  in kvm_arch_vcpu_in_kernel()
10745 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_should_kick()  argument
10747 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;  in kvm_arch_vcpu_should_kick()
10750 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)  in kvm_arch_interrupt_allowed()  argument
10752 	return kvm_x86_ops.interrupt_allowed(vcpu, false);  in kvm_arch_interrupt_allowed()
10755 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)  in kvm_get_linear_rip()  argument
10757 	if (is_64_bit_mode(vcpu))  in kvm_get_linear_rip()
10758 		return kvm_rip_read(vcpu);  in kvm_get_linear_rip()
10759 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +  in kvm_get_linear_rip()
10760 		     kvm_rip_read(vcpu));  in kvm_get_linear_rip()
10764 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)  in kvm_is_linear_rip()  argument
10766 	return kvm_get_linear_rip(vcpu) == linear_rip;  in kvm_is_linear_rip()
10770 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)  in kvm_get_rflags()  argument
10774 	rflags = kvm_x86_ops.get_rflags(vcpu);  in kvm_get_rflags()
10775 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_get_rflags()
10781 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in __kvm_set_rflags()  argument
10783 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&  in __kvm_set_rflags()
10784 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))  in __kvm_set_rflags()
10786 	kvm_x86_ops.set_rflags(vcpu, rflags);  in __kvm_set_rflags()
10789 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in kvm_set_rflags()  argument
10791 	__kvm_set_rflags(vcpu, rflags);  in kvm_set_rflags()
10792 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_set_rflags()
10796 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)  in kvm_arch_async_page_ready()  argument
10800 	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||  in kvm_arch_async_page_ready()
10804 	r = kvm_mmu_reload(vcpu);  in kvm_arch_async_page_ready()
10808 	if (!vcpu->arch.mmu->direct_map &&  in kvm_arch_async_page_ready()
10809 	      work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))  in kvm_arch_async_page_ready()
10812 	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);  in kvm_arch_async_page_ready()
10827 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_add_async_pf_gfn()  argument
10831 	while (vcpu->arch.apf.gfns[key] != ~0)  in kvm_add_async_pf_gfn()
10834 	vcpu->arch.apf.gfns[key] = gfn;  in kvm_add_async_pf_gfn()
10837 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_async_pf_gfn_slot()  argument
10843 		     (vcpu->arch.apf.gfns[key] != gfn &&  in kvm_async_pf_gfn_slot()
10844 		      vcpu->arch.apf.gfns[key] != ~0); i++)  in kvm_async_pf_gfn_slot()
10850 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_find_async_pf_gfn()  argument
10852 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;  in kvm_find_async_pf_gfn()
10855 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_del_async_pf_gfn()  argument
10859 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);  in kvm_del_async_pf_gfn()
10861 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))  in kvm_del_async_pf_gfn()
10865 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_del_async_pf_gfn()
10868 			if (vcpu->arch.apf.gfns[j] == ~0)  in kvm_del_async_pf_gfn()
10870 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);  in kvm_del_async_pf_gfn()
10877 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];  in kvm_del_async_pf_gfn()
10882 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)  in apf_put_user_notpresent()  argument
10886 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,  in apf_put_user_notpresent()
10890 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)  in apf_put_user_ready()  argument
10894 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_put_user_ready()
10898 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)  in apf_pageready_slot_free()  argument
10903 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_pageready_slot_free()
10910 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_deliver_async_pf()  argument
10912 	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))  in kvm_can_deliver_async_pf()
10915 	if (!kvm_pv_async_pf_enabled(vcpu) ||  in kvm_can_deliver_async_pf()
10916 	    (vcpu->arch.apf.send_user_only && kvm_x86_ops.get_cpl(vcpu) == 0))  in kvm_can_deliver_async_pf()
10922 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_do_async_pf()  argument
10924 	if (unlikely(!lapic_in_kernel(vcpu) ||  in kvm_can_do_async_pf()
10925 		     kvm_event_needs_reinjection(vcpu) ||  in kvm_can_do_async_pf()
10926 		     vcpu->arch.exception.pending))  in kvm_can_do_async_pf()
10929 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))  in kvm_can_do_async_pf()
10936 	return kvm_arch_interrupt_allowed(vcpu);  in kvm_can_do_async_pf()
10939 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_not_present()  argument
10945 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_not_present()
10947 	if (kvm_can_deliver_async_pf(vcpu) &&  in kvm_arch_async_page_not_present()
10948 	    !apf_put_user_notpresent(vcpu)) {  in kvm_arch_async_page_not_present()
10955 		kvm_inject_page_fault(vcpu, &fault);  in kvm_arch_async_page_not_present()
10966 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);  in kvm_arch_async_page_not_present()
10971 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_present()  argument
10976 		.vector = vcpu->arch.apf.vec  in kvm_arch_async_page_present()
10982 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_present()
10986 	    kvm_pv_async_pf_enabled(vcpu) &&  in kvm_arch_async_page_present()
10987 	    !apf_put_user_ready(vcpu, work->arch.token)) {  in kvm_arch_async_page_present()
10988 		vcpu->arch.apf.pageready_pending = true;  in kvm_arch_async_page_present()
10989 		kvm_apic_set_irq(vcpu, &irq, NULL);  in kvm_arch_async_page_present()
10992 	vcpu->arch.apf.halted = false;  in kvm_arch_async_page_present()
10993 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_async_page_present()
10996 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)  in kvm_arch_async_page_present_queued()  argument
10998 	kvm_make_request(KVM_REQ_APF_READY, vcpu);  in kvm_arch_async_page_present_queued()
10999 	if (!vcpu->arch.apf.pageready_pending)  in kvm_arch_async_page_present_queued()
11000 		kvm_vcpu_kick(vcpu);  in kvm_arch_async_page_present_queued()
11003 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)  in kvm_arch_can_dequeue_async_page_present()  argument
11005 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_arch_can_dequeue_async_page_present()
11008 		return apf_pageready_slot_free(vcpu);  in kvm_arch_can_dequeue_async_page_present()
11105 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)  in kvm_arch_no_poll()  argument
11107 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;  in kvm_arch_no_poll()
11138 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)  in kvm_fixup_and_inject_pf_error()  argument
11145 	    vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, &fault) != UNMAPPED_GVA) {  in kvm_fixup_and_inject_pf_error()
11157 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);  in kvm_fixup_and_inject_pf_error()
11166 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,  in kvm_handle_memory_failure()  argument
11170 		kvm_inject_emulated_page_fault(vcpu, e);  in kvm_handle_memory_failure()
11181 	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in kvm_handle_memory_failure()
11182 	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in kvm_handle_memory_failure()
11183 	vcpu->run->internal.ndata = 0;  in kvm_handle_memory_failure()
11189 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)  in kvm_handle_invpcid()  argument
11201 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);  in kvm_handle_invpcid()
11203 		return kvm_handle_memory_failure(vcpu, r, &e);  in kvm_handle_invpcid()
11206 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
11210 	pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);  in kvm_handle_invpcid()
11215 		    is_noncanonical_address(operand.gla, vcpu)) {  in kvm_handle_invpcid()
11216 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
11219 		kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);  in kvm_handle_invpcid()
11220 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
11224 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
11228 		if (kvm_get_active_pcid(vcpu) == operand.pcid) {  in kvm_handle_invpcid()
11229 			kvm_mmu_sync_roots(vcpu);  in kvm_handle_invpcid()
11230 			kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_handle_invpcid()
11234 			if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].pgd)  in kvm_handle_invpcid()
11238 		kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free);  in kvm_handle_invpcid()
11245 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
11257 		kvm_mmu_unload(vcpu);  in kvm_handle_invpcid()
11258 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()