177 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
179 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
180 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req,
182 bool kvm_make_cpus_request_mask(struct kvm *kvm, unsigned int req,
219 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
221 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
223 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
255 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
256 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
257 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
258 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
310 	struct kvm *kvm;  member
625 		   struct kvm *kvm, int irq_source_id, int level,
689 struct kvm {  struct
817 static inline void kvm_vm_dead(struct kvm *kvm)  in kvm_vm_dead()  argument
819 	kvm->vm_dead = true;  in kvm_vm_dead()
820 	kvm_make_all_cpus_request(kvm, KVM_REQ_VM_DEAD);  in kvm_vm_dead()
823 static inline void kvm_vm_bugged(struct kvm *kvm)  in kvm_vm_bugged()  argument
825 	kvm->vm_bugged = true;  in kvm_vm_bugged()
826 	kvm_vm_dead(kvm);  in kvm_vm_bugged()
830 #define KVM_BUG(cond, kvm, fmt...)				\  argument
834 	if (WARN_ONCE(__ret && !(kvm)->vm_bugged, fmt))		\
835 		kvm_vm_bugged(kvm);				\
839 #define KVM_BUG_ON(cond, kvm)					\  argument
843 	if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged))		\
844 		kvm_vm_bugged(kvm);				\
854 	vcpu->____srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_vcpu_srcu_read_lock()
859 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->____srcu_idx);  in kvm_vcpu_srcu_read_unlock()
867 static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)  in kvm_dirty_log_manual_protect_and_init_set()  argument
869 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);  in kvm_dirty_log_manual_protect_and_init_set()
872 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)  in kvm_get_bus()  argument
874 	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,  in kvm_get_bus()
875 				      lockdep_is_held(&kvm->slots_lock) ||  in kvm_get_bus()
876 				      !refcount_read(&kvm->users_count));  in kvm_get_bus()
879 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)  in kvm_get_vcpu()  argument
881 	int num_vcpus = atomic_read(&kvm->online_vcpus);  in kvm_get_vcpu()
886 	return xa_load(&kvm->vcpu_array, i);  in kvm_get_vcpu()
889 #define kvm_for_each_vcpu(idx, vcpup, kvm)		   \  argument
890 	xa_for_each_range(&kvm->vcpu_array, idx, vcpup, 0, \
891 			  (atomic_read(&kvm->online_vcpus) - 1))
893 static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id)  in kvm_get_vcpu_by_id()  argument
901 		vcpu = kvm_get_vcpu(kvm, id);  in kvm_get_vcpu_by_id()
904 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_get_vcpu_by_id()
910 void kvm_destroy_vcpus(struct kvm *kvm);
916 void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm);
917 void kvm_arch_post_irq_routing_update(struct kvm *kvm);
919 static inline void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm)  in kvm_arch_post_irq_ack_notifier_list_update()  argument
922 static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm)  in kvm_arch_post_irq_routing_update()  argument
944 void kvm_get_kvm(struct kvm *kvm);
945 bool kvm_get_kvm_safe(struct kvm *kvm);
946 void kvm_put_kvm(struct kvm *kvm);
948 void kvm_put_kvm_no_destroy(struct kvm *kvm);
950 static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)  in __kvm_memslots()  argument
953 	return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu,  in __kvm_memslots()
954 			lockdep_is_held(&kvm->slots_lock) ||  in __kvm_memslots()
955 			!refcount_read(&kvm->users_count));  in __kvm_memslots()
958 static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)  in kvm_memslots()  argument
960 	return __kvm_memslots(kvm, 0);  in kvm_memslots()
967 	return __kvm_memslots(vcpu->kvm, as_id);  in kvm_vcpu_memslots()
1106 int kvm_set_memory_region(struct kvm *kvm,
1108 int __kvm_set_memory_region(struct kvm *kvm,
1110 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot);
1111 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen);
1112 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1116 void kvm_arch_commit_memory_region(struct kvm *kvm,
1121 void kvm_arch_flush_shadow_all(struct kvm *kvm);
1123 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1129 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
1130 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
1131 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable);
1138 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
1139 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1153 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1155 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
1156 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1158 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1161 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1163 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1165 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1167 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1170 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1173 #define __kvm_get_guest(kvm, gfn, offset, v)				\  argument
1175 	unsigned long __addr = gfn_to_hva(kvm, gfn);			\
1184 #define kvm_get_guest(kvm, gpa, v)					\  argument
1187 	struct kvm *__kvm = kvm;					\
1193 #define __kvm_put_guest(kvm, gfn, offset, v)				\  argument
1195 	unsigned long __addr = gfn_to_hva(kvm, gfn);			\
1202 		mark_page_dirty(kvm, gfn);				\
1206 #define kvm_put_guest(kvm, gpa, v)					\  argument
1209 	struct kvm *__kvm = kvm;					\
1215 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
1216 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
1217 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
1220 void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn);
1221 void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
1279 int kvm_gpc_activate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc,
1301 bool kvm_gfn_to_pfn_cache_check(struct kvm *kvm, struct gfn_to_pfn_cache *gpc,
1322 int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc,
1335 void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc);
1346 void kvm_gpc_deactivate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc);
1360 void kvm_flush_remote_tlbs(struct kvm *kvm);
1370 void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
1372 void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
1381 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
1383 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1387 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot);
1390 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
1393 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log);
1394 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
1398 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1400 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
1434 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id);
1440 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state);
1459 int kvm_arch_post_init_vm(struct kvm *kvm);
1460 void kvm_arch_pre_destroy_vm(struct kvm *kvm);
1461 int kvm_arch_create_vm_debugfs(struct kvm *kvm);
1468 static inline struct kvm *kvm_arch_alloc_vm(void)  in kvm_arch_alloc_vm()
1470 	return kzalloc(sizeof(struct kvm), GFP_KERNEL);  in kvm_arch_alloc_vm()
1474 static inline void __kvm_arch_free_vm(struct kvm *kvm)  in __kvm_arch_free_vm()  argument
1476 	kvfree(kvm);  in __kvm_arch_free_vm()
1480 static inline void kvm_arch_free_vm(struct kvm *kvm)  in kvm_arch_free_vm()  argument
1482 	__kvm_arch_free_vm(kvm);  in kvm_arch_free_vm()
1487 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)  in kvm_arch_flush_remote_tlb()  argument
1494 void kvm_arch_register_noncoherent_dma(struct kvm *kvm);
1495 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm);
1496 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm);
1498 static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm)  in kvm_arch_register_noncoherent_dma()  argument
1502 static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)  in kvm_arch_unregister_noncoherent_dma()  argument
1506 static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)  in kvm_arch_has_noncoherent_dma()  argument
1512 void kvm_arch_start_assignment(struct kvm *kvm);
1513 void kvm_arch_end_assignment(struct kvm *kvm);
1514 bool kvm_arch_has_assigned_device(struct kvm *kvm);
1516 static inline void kvm_arch_start_assignment(struct kvm *kvm)  in kvm_arch_start_assignment()  argument
1520 static inline void kvm_arch_end_assignment(struct kvm *kvm)  in kvm_arch_end_assignment()  argument
1524 static __always_inline bool kvm_arch_has_assigned_device(struct kvm *kvm)  in kvm_arch_has_assigned_device()  argument
1559 bool kvm_arch_intc_initialized(struct kvm *kvm);
1561 static inline bool kvm_arch_intc_initialized(struct kvm *kvm)  in kvm_arch_intc_initialized()  argument
1577 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
1578 void kvm_arch_destroy_vm(struct kvm *kvm);
1579 void kvm_arch_sync_events(struct kvm *kvm);
1592 int kvm_irq_map_gsi(struct kvm *kvm,
1594 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin);
1596 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
1598 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
1601 			       struct kvm *kvm, int irq_source_id,
1603 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin);
1604 void kvm_notify_acked_gsi(struct kvm *kvm, int gsi);
1605 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
1606 void kvm_register_irq_ack_notifier(struct kvm *kvm,
1608 void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
1610 int kvm_request_irq_source_id(struct kvm *kvm);
1611 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
1612 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args);
1702 static inline int memslot_id(struct kvm *kvm, gfn_t gfn)  in memslot_id()  argument
1704 	return gfn_to_memslot(kvm, gfn)->id;  in memslot_id()
1730 static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa)  in kvm_is_error_gpa()  argument
1732 	unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa));  in kvm_is_error_gpa()
1743 	struct kvm *kvm;  member
1922 static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq)  in mmu_invalidate_retry()  argument
1924 	if (unlikely(kvm->mmu_invalidate_in_progress))  in mmu_invalidate_retry()
1939 	if (kvm->mmu_invalidate_seq != mmu_seq)  in mmu_invalidate_retry()
1944 static inline int mmu_invalidate_retry_hva(struct kvm *kvm,  in mmu_invalidate_retry_hva()  argument
1948 	lockdep_assert_held(&kvm->mmu_lock);  in mmu_invalidate_retry_hva()
1955 	if (unlikely(kvm->mmu_invalidate_in_progress) &&  in mmu_invalidate_retry_hva()
1956 	    hva >= kvm->mmu_invalidate_range_start &&  in mmu_invalidate_retry_hva()
1957 	    hva < kvm->mmu_invalidate_range_end)  in mmu_invalidate_retry_hva()
1959 	if (kvm->mmu_invalidate_seq != mmu_seq)  in mmu_invalidate_retry_hva()
1969 bool kvm_arch_can_set_irq_routing(struct kvm *kvm);
1970 int kvm_set_irq_routing(struct kvm *kvm,
1974 int kvm_set_routing_entry(struct kvm *kvm,
1977 void kvm_free_irq_routing(struct kvm *kvm);
1981 static inline void kvm_free_irq_routing(struct kvm *kvm) {}  in kvm_free_irq_routing()  argument
1985 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
1989 void kvm_eventfd_init(struct kvm *kvm);
1990 int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);
1993 int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
1994 void kvm_irqfd_release(struct kvm *kvm);
1995 void kvm_irq_routing_update(struct kvm *);
1997 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)  in kvm_irqfd()  argument
2002 static inline void kvm_irqfd_release(struct kvm *kvm) {}  in kvm_irqfd_release()  argument
2007 static inline void kvm_eventfd_init(struct kvm *kvm) {}  in kvm_eventfd_init()  argument
2009 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)  in kvm_irqfd()  argument
2014 static inline void kvm_irqfd_release(struct kvm *kvm) {}  in kvm_irqfd_release()  argument
2017 static inline void kvm_irq_routing_update(struct kvm *kvm)  in kvm_irq_routing_update()  argument
2022 static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)  in kvm_ioeventfd()  argument
2029 void kvm_arch_irq_routing_update(struct kvm *kvm);
2094 	struct kvm *kvm;  member
2192 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
2234 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
2237 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm);
2248 typedef int (*kvm_vm_thread_fn_t)(struct kvm *kvm, uintptr_t data);
2250 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,