97 #define emul_to_vcpu(ctxt) \  argument
98 	((struct kvm_vcpu *)(ctxt)->vcpu)
342 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
7227 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,  in kvm_fetch_guest_virt()  argument
7231 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_fetch_guest_virt()
7272 static int emulator_read_std(struct x86_emulate_ctxt *ctxt,  in emulator_read_std()  argument
7276 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_std()
7287 static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,  in kvm_read_guest_phys_system()  argument
7290 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_read_guest_phys_system()
7326 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,  in emulator_write_std()  argument
7330 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_write_std()
7524 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in emulator_read_write_onepage()  local
7533 	if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&  in emulator_read_write_onepage()
7534 	    (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {  in emulator_read_write_onepage()
7535 		gpa = ctxt->gpa_val;  in emulator_read_write_onepage()
7565 static int emulator_read_write(struct x86_emulate_ctxt *ctxt,  in emulator_read_write()  argument
7571 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_write()
7592 		if (ctxt->mode != X86EMUL_MODE_PROT64)  in emulator_read_write()
7619 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,  in emulator_read_emulated()  argument
7625 	return emulator_read_write(ctxt, addr, val, bytes,  in emulator_read_emulated()
7629 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,  in emulator_write_emulated()  argument
7635 	return emulator_read_write(ctxt, addr, (void *)val, bytes,  in emulator_write_emulated()
7642 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,  in emulator_cmpxchg_emulated()  argument
7649 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_cmpxchg_emulated()
7712 	return emulator_write_emulated(ctxt, addr, new, bytes, exception);  in emulator_cmpxchg_emulated()
7785 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,  in emulator_pio_in_emulated()  argument
7789 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_pio_in_emulated()
7813 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,  in emulator_pio_out_emulated()  argument
7817 	return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);  in emulator_pio_out_emulated()
7825 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)  in emulator_invlpg()  argument
7827 	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);  in emulator_invlpg()
7857 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)  in emulator_wbinvd()  argument
7859 	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));  in emulator_wbinvd()
7862 static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,  in emulator_get_dr()  argument
7865 	kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);  in emulator_get_dr()
7868 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,  in emulator_set_dr()  argument
7872 	return kvm_set_dr(emul_to_vcpu(ctxt), dr, value);  in emulator_set_dr()
7880 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)  in emulator_get_cr()  argument
7882 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_cr()
7909 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)  in emulator_set_cr()  argument
7911 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_cr()
7938 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)  in emulator_get_cpl()  argument
7940 	return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt));  in emulator_get_cpl()
7943 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)  in emulator_get_gdt()  argument
7945 	static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);  in emulator_get_gdt()
7948 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)  in emulator_get_idt()  argument
7950 	static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);  in emulator_get_idt()
7953 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)  in emulator_set_gdt()  argument
7955 	static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt);  in emulator_set_gdt()
7958 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)  in emulator_set_idt()  argument
7960 	static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt);  in emulator_set_idt()
7964 	struct x86_emulate_ctxt *ctxt, int seg)  in emulator_get_cached_segment_base()  argument
7966 	return get_segment_base(emul_to_vcpu(ctxt), seg);  in emulator_get_cached_segment_base()
7969 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,  in emulator_get_segment()  argument
7975 	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);  in emulator_get_segment()
8005 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,  in emulator_set_segment()  argument
8009 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_segment()
8035 static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt,  in emulator_get_msr_with_filter()  argument
8038 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_msr_with_filter()
8058 static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt,  in emulator_set_msr_with_filter()  argument
8061 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_msr_with_filter()
8081 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,  in emulator_get_msr()  argument
8084 	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);  in emulator_get_msr()
8087 static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,  in emulator_set_msr()  argument
8090 	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);  in emulator_set_msr()
8093 static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)  in emulator_get_smbase()  argument
8095 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_smbase()
8100 static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)  in emulator_set_smbase()  argument
8102 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_smbase()
8107 static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,  in emulator_check_pmc()  argument
8110 	if (kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc))  in emulator_check_pmc()
8115 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,  in emulator_read_pmc()  argument
8118 	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);  in emulator_read_pmc()
8121 static void emulator_halt(struct x86_emulate_ctxt *ctxt)  in emulator_halt()  argument
8123 	emul_to_vcpu(ctxt)->arch.halt_request = 1;  in emulator_halt()
8126 static int emulator_intercept(struct x86_emulate_ctxt *ctxt,  in emulator_intercept()  argument
8130 	return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage,  in emulator_intercept()
8131 					    &ctxt->exception);  in emulator_intercept()
8134 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,  in emulator_get_cpuid()  argument
8138 	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only);  in emulator_get_cpuid()
8141 static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)  in emulator_guest_has_long_mode()  argument
8143 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);  in emulator_guest_has_long_mode()
8146 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)  in emulator_guest_has_movbe()  argument
8148 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);  in emulator_guest_has_movbe()
8151 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)  in emulator_guest_has_fxsr()  argument
8153 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);  in emulator_guest_has_fxsr()
8156 static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt)  in emulator_guest_has_rdpid()  argument
8158 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID);  in emulator_guest_has_rdpid()
8161 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)  in emulator_read_gpr()  argument
8163 	return kvm_register_read_raw(emul_to_vcpu(ctxt), reg);  in emulator_read_gpr()
8166 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)  in emulator_write_gpr()  argument
8168 	kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val);  in emulator_write_gpr()
8171 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)  in emulator_set_nmi_mask()  argument
8173 	static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked);  in emulator_set_nmi_mask()
8176 static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)  in emulator_get_hflags()  argument
8178 	return emul_to_vcpu(ctxt)->arch.hflags;  in emulator_get_hflags()
8181 static void emulator_exiting_smm(struct x86_emulate_ctxt *ctxt)  in emulator_exiting_smm()  argument
8183 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_exiting_smm()
8188 static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt,  in emulator_leave_smm()  argument
8191 	return static_call(kvm_x86_leave_smm)(emul_to_vcpu(ctxt), smstate);  in emulator_leave_smm()
8194 static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt)  in emulator_triple_fault()  argument
8196 	kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));  in emulator_triple_fault()
8199 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)  in emulator_set_xcr()  argument
8201 	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);  in emulator_set_xcr()
8204 static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt)  in emulator_vm_bugged()  argument
8206 	struct kvm *kvm = emul_to_vcpu(ctxt)->kvm;  in emulator_vm_bugged()
8284 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in inject_emulated_exception()  local
8286 	if (ctxt->exception.vector == PF_VECTOR)  in inject_emulated_exception()
8287 		kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);  in inject_emulated_exception()
8288 	else if (ctxt->exception.error_code_valid)  in inject_emulated_exception()
8289 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,  in inject_emulated_exception()
8290 				      ctxt->exception.error_code);  in inject_emulated_exception()
8292 		kvm_queue_exception(vcpu, ctxt->exception.vector);  in inject_emulated_exception()
8297 	struct x86_emulate_ctxt *ctxt;  in alloc_emulate_ctxt()  local
8299 	ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT);  in alloc_emulate_ctxt()
8300 	if (!ctxt) {  in alloc_emulate_ctxt()
8305 	ctxt->vcpu = vcpu;  in alloc_emulate_ctxt()
8306 	ctxt->ops = &emulate_ops;  in alloc_emulate_ctxt()
8307 	vcpu->arch.emulate_ctxt = ctxt;  in alloc_emulate_ctxt()
8309 	return ctxt;  in alloc_emulate_ctxt()
8314 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in init_emulate_ctxt()  local
8319 	ctxt->gpa_available = false;  in init_emulate_ctxt()
8320 	ctxt->eflags = kvm_get_rflags(vcpu);  in init_emulate_ctxt()
8321 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;  in init_emulate_ctxt()
8323 	ctxt->eip = kvm_rip_read(vcpu);  in init_emulate_ctxt()
8324 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :  in init_emulate_ctxt()
8325 		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :  in init_emulate_ctxt()
8333 	ctxt->interruptibility = 0;  in init_emulate_ctxt()
8334 	ctxt->have_exception = false;  in init_emulate_ctxt()
8335 	ctxt->exception.vector = -1;  in init_emulate_ctxt()
8336 	ctxt->perm_ok = false;  in init_emulate_ctxt()
8338 	init_decode_cache(ctxt);  in init_emulate_ctxt()
8344 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_inject_realmode_interrupt()  local
8349 	ctxt->op_bytes = 2;  in kvm_inject_realmode_interrupt()
8350 	ctxt->ad_bytes = 2;  in kvm_inject_realmode_interrupt()
8351 	ctxt->_eip = ctxt->eip + inc_eip;  in kvm_inject_realmode_interrupt()
8352 	ret = emulate_int_real(ctxt, irq);  in kvm_inject_realmode_interrupt()
8357 		ctxt->eip = ctxt->_eip;  in kvm_inject_realmode_interrupt()
8358 		kvm_rip_write(vcpu, ctxt->eip);  in kvm_inject_realmode_interrupt()
8359 		kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_inject_realmode_interrupt()
8417 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in prepare_emulation_ctxt_failure_exit()  local
8419 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,  in prepare_emulation_ctxt_failure_exit()
8420 				       ctxt->fetch.end - ctxt->fetch.data);  in prepare_emulation_ctxt_failure_exit()
8539 static bool retry_instruction(struct x86_emulate_ctxt *ctxt,  in retry_instruction()  argument
8542 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in retry_instruction()
8570 	if (x86_page_table_writing_insn(ctxt))  in retry_instruction()
8573 	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)  in retry_instruction()
8576 	vcpu->arch.last_retry_eip = ctxt->eip;  in retry_instruction()
8743 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)  in is_vmware_backdoor_opcode()  argument
8745 	switch (ctxt->opcode_len) {  in is_vmware_backdoor_opcode()
8747 		switch (ctxt->b) {  in is_vmware_backdoor_opcode()
8764 		switch (ctxt->b) {  in is_vmware_backdoor_opcode()
8787 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_decode_emulated_instruction()  local
8792 	r = x86_decode_insn(ctxt, insn, insn_len, emulation_type);  in x86_decode_emulated_instruction()
8805 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_emulate_instruction()  local
8844 			if (ctxt->have_exception) {  in x86_emulate_instruction()
8849 				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||  in x86_emulate_instruction()
8850 					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);  in x86_emulate_instruction()
8859 	    !is_vmware_backdoor_opcode(ctxt)) {  in x86_emulate_instruction()
8871 		if (ctxt->mode != X86EMUL_MODE_PROT64)  in x86_emulate_instruction()
8872 			ctxt->eip = (u32)ctxt->_eip;  in x86_emulate_instruction()
8874 			ctxt->eip = ctxt->_eip;  in x86_emulate_instruction()
8881 		kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
8882 		if (ctxt->eflags & X86_EFLAGS_RF)  in x86_emulate_instruction()
8883 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);  in x86_emulate_instruction()
8887 	if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))  in x86_emulate_instruction()
8894 		emulator_invalidate_register_cache(ctxt);  in x86_emulate_instruction()
8900 		ctxt->exception.address = cr2_or_gpa;  in x86_emulate_instruction()
8904 			ctxt->gpa_available = true;  in x86_emulate_instruction()
8905 			ctxt->gpa_val = cr2_or_gpa;  in x86_emulate_instruction()
8909 		ctxt->exception.address = 0;  in x86_emulate_instruction()
8912 	r = x86_emulate_insn(ctxt);  in x86_emulate_instruction()
8925 	if (ctxt->have_exception) {  in x86_emulate_instruction()
8955 		toggle_interruptibility(vcpu, ctxt->interruptibility);  in x86_emulate_instruction()
8963 		if (!ctxt->have_exception ||  in x86_emulate_instruction()
8964 		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {  in x86_emulate_instruction()
8966 			if (ctxt->is_branch)  in x86_emulate_instruction()
8968 			kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
8969 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))  in x86_emulate_instruction()
8972 			__kvm_set_rflags(vcpu, ctxt->eflags);  in x86_emulate_instruction()
8981 		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))  in x86_emulate_instruction()
9739 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)  in emulator_fix_hypercall()  argument
9741 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_fix_hypercall()
9750 		ctxt->exception.error_code_valid = false;  in emulator_fix_hypercall()
9751 		ctxt->exception.vector = UD_VECTOR;  in emulator_fix_hypercall()
9752 		ctxt->have_exception = true;  in emulator_fix_hypercall()
9758 	return emulator_write_emulated(ctxt, rip, instruction, 3,  in emulator_fix_hypercall()
9759 		&ctxt->exception);  in emulator_fix_hypercall()
11440 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_task_switch()  local
11445 	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,  in kvm_task_switch()
11454 	kvm_rip_write(vcpu, ctxt->eip);  in kvm_task_switch()
11455 	kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_task_switch()