303 	svm->vmcb->save.efer = efer | EFER_SVME;  in svm_set_efer()
304 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);  in svm_set_efer()
319 	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)  in svm_get_interrupt_shadow()
329 		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;  in svm_set_interrupt_shadow()
331 		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;  in svm_set_interrupt_shadow()
339 	if (nrips && svm->vmcb->control.next_rip != 0) {  in skip_emulated_instruction()
341 		svm->next_rip = svm->vmcb->control.next_rip;  in skip_emulated_instruction()
376 		svm->int3_rip = rip + svm->vmcb->save.cs.base;  in svm_queue_exception()
380 	svm->vmcb->control.event_inj = nr  in svm_queue_exception()
384 	svm->vmcb->control.event_inj_err = error_code;  in svm_queue_exception()
769 	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;  in svm_enable_lbrv()
780 	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;  in svm_disable_lbrv()
794 			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;  in disable_nmi_singlestep()
796 			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;  in disable_nmi_singlestep()
803 	struct vmcb_control_area *control = &svm->vmcb->control;  in grow_ple_window()
812 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);  in grow_ple_window()
821 	struct vmcb_control_area *control = &svm->vmcb->control;  in shrink_ple_window()
830 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);  in shrink_ple_window()
1072 		g_tsc_offset = svm->vmcb->control.tsc_offset -  in svm_write_l1_tsc_offset()
1078 				   svm->vmcb->control.tsc_offset - g_tsc_offset,  in svm_write_l1_tsc_offset()
1081 	svm->vmcb->control.tsc_offset = offset + g_tsc_offset;  in svm_write_l1_tsc_offset()
1083 	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);  in svm_write_l1_tsc_offset()
1084 	return svm->vmcb->control.tsc_offset;  in svm_write_l1_tsc_offset()
1104 	struct vmcb_control_area *control = &svm->vmcb->control;  in init_vmcb()
1105 	struct vmcb_save_area *save = &svm->vmcb->save;  in init_vmcb()
1243 		svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;  in init_vmcb()
1249 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;  in init_vmcb()
1253 		svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;  in init_vmcb()
1257 	vmcb_mark_all_dirty(svm->vmcb);  in init_vmcb()
1319 	svm->vmcb = page_address(vmcb_page);  in svm_create_vcpu()
1335 static void svm_clear_current_vmcb(struct vmcb *vmcb)  in svm_clear_current_vmcb()  argument
1340 		cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);  in svm_clear_current_vmcb()
1352 	svm_clear_current_vmcb(svm->vmcb);  in svm_free_vcpu()
1368 		vmcb_mark_all_dirty(svm->vmcb);  in svm_vcpu_load()
1392 	if (sd->current_vmcb != svm->vmcb) {  in svm_vcpu_load()
1393 		sd->current_vmcb = svm->vmcb;  in svm_vcpu_load()
1424 	unsigned long rflags = svm->vmcb->save.rflags;  in svm_get_rflags()
1446 	to_svm(vcpu)->vmcb->save.rflags = rflags;  in svm_set_rflags()
1473 	control = &svm->vmcb->control;  in svm_set_vintr()
1478 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);  in svm_set_vintr()
1487 	svm->vmcb->control.int_ctl &= mask;  in svm_clear_vintr()
1491 		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=  in svm_clear_vintr()
1493 		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & ~mask;  in svm_clear_vintr()
1496 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);  in svm_clear_vintr()
1501 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;  in svm_seg()
1588 		var->dpl = to_svm(vcpu)->vmcb->save.cpl;  in svm_get_segment()
1595 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;  in svm_get_cpl()
1604 	dt->size = svm->vmcb->save.idtr.limit;  in svm_get_idt()
1605 	dt->address = svm->vmcb->save.idtr.base;  in svm_get_idt()
1612 	svm->vmcb->save.idtr.limit = dt->size;  in svm_set_idt()
1613 	svm->vmcb->save.idtr.base = dt->address ;  in svm_set_idt()
1614 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);  in svm_set_idt()
1621 	dt->size = svm->vmcb->save.gdtr.limit;  in svm_get_gdt()
1622 	dt->address = svm->vmcb->save.gdtr.base;  in svm_get_gdt()
1629 	svm->vmcb->save.gdtr.limit = dt->size;  in svm_set_gdt()
1630 	svm->vmcb->save.gdtr.base = dt->address ;  in svm_set_gdt()
1631 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);  in svm_set_gdt()
1637 	u64 *hcr0 = &svm->vmcb->save.cr0;  in update_cr0_intercept()
1642 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);  in update_cr0_intercept()
1661 			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;  in svm_set_cr0()
1666 			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);  in svm_set_cr0()
1682 	svm->vmcb->save.cr0 = cr0;  in svm_set_cr0()
1683 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);  in svm_set_cr0()
1690 	unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;  in svm_set_cr4()
1702 	to_svm(vcpu)->vmcb->save.cr4 = cr4;  in svm_set_cr4()
1703 	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);  in svm_set_cr4()
1733 		svm->vmcb->save.cpl = (var->dpl & 3);  in svm_set_segment()
1735 	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);  in svm_set_segment()
1755 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;  in new_asid()
1759 	svm->vmcb->control.asid = sd->next_asid++;  in new_asid()
1761 	vmcb_mark_dirty(svm->vmcb, VMCB_ASID);  in new_asid()
1766 	struct vmcb *vmcb = svm->vmcb;  in svm_set_dr6()  local
1768 	if (unlikely(value != vmcb->save.dr6)) {  in svm_set_dr6()
1769 		vmcb->save.dr6 = value;  in svm_set_dr6()
1770 		vmcb_mark_dirty(vmcb, VMCB_DR);  in svm_set_dr6()
1786 	vcpu->arch.dr6 = svm->vmcb->save.dr6;  in svm_sync_dirty_debug_regs()
1787 	vcpu->arch.dr7 = svm->vmcb->save.dr7;  in svm_sync_dirty_debug_regs()
1796 	svm->vmcb->save.dr7 = value;  in svm_set_dr7()
1797 	vmcb_mark_dirty(svm->vmcb, VMCB_DR);  in svm_set_dr7()
1802 	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);  in pf_interception()
1803 	u64 error_code = svm->vmcb->control.exit_info_1;  in pf_interception()
1807 			svm->vmcb->control.insn_bytes : NULL,  in pf_interception()
1808 			svm->vmcb->control.insn_len);  in pf_interception()
1813 	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);  in npf_interception()
1814 	u64 error_code = svm->vmcb->control.exit_info_1;  in npf_interception()
1819 			svm->vmcb->control.insn_bytes : NULL,  in npf_interception()
1820 			svm->vmcb->control.insn_len);  in npf_interception()
1831 		u32 payload = (svm->vmcb->save.dr6 ^ DR6_RTM) & ~DR6_FIXED_1;  in db_interception()
1845 		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;  in db_interception()
1846 		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;  in db_interception()
1848 			svm->vmcb->save.cs.base + svm->vmcb->save.rip;  in db_interception()
1861 	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;  in bp_interception()
1880 	u32 error_code = svm->vmcb->control.exit_info_1;  in gp_interception()
1987 	clear_page(svm->vmcb);  in shutdown_interception()
1997 	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */  in io_interception()
2009 	svm->next_rip = svm->vmcb->control.exit_info_2;  in io_interception()
2042 	struct vmcb *nested_vmcb;  in vmload_interception()
2049 	ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);  in vmload_interception()
2060 	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);  in vmload_interception()
2068 	struct vmcb *nested_vmcb;  in vmsave_interception()
2075 	ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);  in vmsave_interception()
2086 	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);  in vmsave_interception()
2160 	trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),  in invlpga_interception()
2171 	trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));  in skinit_interception()
2204 	int int_type = svm->vmcb->control.exit_int_info &  in task_switch_interception()
2206 	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;  in task_switch_interception()
2208 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;  in task_switch_interception()
2210 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;  in task_switch_interception()
2214 	tss_selector = (u16)svm->vmcb->control.exit_info_1;  in task_switch_interception()
2216 	if (svm->vmcb->control.exit_info_2 &  in task_switch_interception()
2219 	else if (svm->vmcb->control.exit_info_2 &  in task_switch_interception()
2233 			if (svm->vmcb->control.exit_info_2 &  in task_switch_interception()
2237 					(u32)svm->vmcb->control.exit_info_2;  in task_switch_interception()
2290 	kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);  in invlpg_interception()
2329 		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;  in check_selective_cr0_intercepted()
2347 	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))  in cr_interception()
2350 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;  in cr_interception()
2351 	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)  in cr_interception()
2354 		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;  in cr_interception()
2430 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;  in dr_interception()
2431 	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;  in dr_interception()
2488 		msr_info->data = svm->vmcb->save.star;  in svm_get_msr()
2492 		msr_info->data = svm->vmcb->save.lstar;  in svm_get_msr()
2495 		msr_info->data = svm->vmcb->save.cstar;  in svm_get_msr()
2498 		msr_info->data = svm->vmcb->save.kernel_gs_base;  in svm_get_msr()
2501 		msr_info->data = svm->vmcb->save.sfmask;  in svm_get_msr()
2505 		msr_info->data = svm->vmcb->save.sysenter_cs;  in svm_get_msr()
2524 		msr_info->data = svm->vmcb->save.dbgctl;  in svm_get_msr()
2527 		msr_info->data = svm->vmcb->save.br_from;  in svm_get_msr()
2530 		msr_info->data = svm->vmcb->save.br_to;  in svm_get_msr()
2533 		msr_info->data = svm->vmcb->save.last_excp_from;  in svm_get_msr()
2536 		msr_info->data = svm->vmcb->save.last_excp_to;  in svm_get_msr()
2628 		svm->vmcb->save.g_pat = data;  in svm_set_msr()
2629 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);  in svm_set_msr()
2685 		svm->vmcb->save.star = data;  in svm_set_msr()
2689 		svm->vmcb->save.lstar = data;  in svm_set_msr()
2692 		svm->vmcb->save.cstar = data;  in svm_set_msr()
2695 		svm->vmcb->save.kernel_gs_base = data;  in svm_set_msr()
2698 		svm->vmcb->save.sfmask = data;  in svm_set_msr()
2702 		svm->vmcb->save.sysenter_cs = data;  in svm_set_msr()
2706 		svm->vmcb->save.sysenter_eip = data;  in svm_set_msr()
2710 		svm->vmcb->save.sysenter_esp = data;  in svm_set_msr()
2733 		svm->vmcb->save.dbgctl = data;  in svm_set_msr()
2734 		vmcb_mark_dirty(svm->vmcb, VMCB_LBR);  in svm_set_msr()
2783 	if (svm->vmcb->control.exit_info_1)  in msr_interception()
2850 	type = svm->vmcb->control.exit_info_2;  in invpcid_interception()
2851 	gva = svm->vmcb->control.exit_info_1;  in invpcid_interception()
2933 	struct vmcb_control_area *control = &svm->vmcb->control;  in dump_vmcb()
2934 	struct vmcb_save_area *save = &svm->vmcb->save;  in dump_vmcb()
3051 	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;  in svm_get_exit_info()
3067 	u32 exit_code = svm->vmcb->control.exit_code;  in handle_exit()
3072 		vcpu->arch.cr0 = svm->vmcb->save.cr0;  in handle_exit()
3074 		vcpu->arch.cr3 = svm->vmcb->save.cr3;  in handle_exit()
3090 	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {  in handle_exit()
3093 			= svm->vmcb->control.exit_code;  in handle_exit()
3099 	if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&  in handle_exit()
3105 		       __func__, svm->vmcb->control.exit_int_info,  in handle_exit()
3163 	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;  in svm_inject_nmi()
3178 	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |  in svm_set_irq()
3201 	struct vmcb *vmcb = svm->vmcb;  in svm_nmi_blocked()  local
3210 	ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||  in svm_nmi_blocked()
3252 	struct vmcb *vmcb = svm->vmcb;  in svm_interrupt_blocked()  local
3272 	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);  in svm_interrupt_blocked()
3335 	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);  in enable_nmi_window()
3360 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;  in svm_flush_tlb()
3369 	invlpga(gva, svm->vmcb->control.asid);  in svm_flush_tlb_gva()
3384 		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;  in sync_cr8_to_lapic()
3399 	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;  in sync_lapic_to_cr8()
3400 	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;  in sync_lapic_to_cr8()
3407 	u32 exitintinfo = svm->vmcb->control.exit_int_info;  in svm_complete_interrupts()
3453 			u32 err = svm->vmcb->control.exit_int_info_err;  in svm_complete_interrupts()
3470 	struct vmcb_control_area *control = &svm->vmcb->control;  in svm_cancel_injection()
3480 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&  in svm_exit_handlers_fastpath()
3481 	    to_svm(vcpu)->vmcb->control.exit_info_1)  in svm_exit_handlers_fastpath()
3547 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];  in svm_vcpu_run()
3548 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];  in svm_vcpu_run()
3549 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];  in svm_vcpu_run()
3557 	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {  in svm_vcpu_run()
3571 	svm->vmcb->save.cr2 = vcpu->arch.cr2;  in svm_vcpu_run()
3619 	vcpu->arch.cr2 = svm->vmcb->save.cr2;  in svm_vcpu_run()
3620 	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;  in svm_vcpu_run()
3621 	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;  in svm_vcpu_run()
3622 	vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;  in svm_vcpu_run()
3624 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))  in svm_vcpu_run()
3632 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))  in svm_vcpu_run()
3643 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;  in svm_vcpu_run()
3644 	vmcb_mark_all_clean(svm->vmcb);  in svm_vcpu_run()
3647 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)  in svm_vcpu_run()
3660 	if (unlikely(svm->vmcb->control.exit_code ==  in svm_vcpu_run()
3680 		svm->vmcb->control.nested_cr3 = cr3;  in svm_load_mmu_pgd()
3681 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);  in svm_load_mmu_pgd()
3689 	svm->vmcb->save.cr3 = cr3;  in svm_load_mmu_pgd()
3690 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);  in svm_load_mmu_pgd()
3863 	struct vmcb *vmcb = svm->vmcb;  in svm_check_intercept()  local
3914 			vmcb->control.exit_info_1 = 1;  in svm_check_intercept()
3916 			vmcb->control.exit_info_1 = 0;  in svm_check_intercept()
3953 		vmcb->control.exit_info_1 = exit_info;  in svm_check_intercept()
3954 		vmcb->control.exit_info_2 = info->next_rip;  in svm_check_intercept()
3964 		vmcb->control.next_rip  = info->next_rip;  in svm_check_intercept()
3965 	vmcb->control.exit_code = icpt_info.exit_code;  in svm_check_intercept()
4026 		svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];  in svm_pre_enter_smm()
4027 		svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];  in svm_pre_enter_smm()
4028 		svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];  in svm_pre_enter_smm()
4165 		   (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT));  in svm_apic_init_signal_blocked()