156 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
162 __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,  in kvm_arch_mmu_notifier_invalidate_range()  argument
167 __weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)  in kvm_arch_guest_memory_reclaimed()  argument
300 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,  in kvm_make_vcpus_request_mask()  argument
314 		vcpu = kvm_get_vcpu(kvm, i);  in kvm_make_vcpus_request_mask()
326 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req,  in kvm_make_all_cpus_request_except()  argument
340 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_make_all_cpus_request_except()
352 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)  in kvm_make_all_cpus_request()  argument
354 	return kvm_make_all_cpus_request_except(kvm, req, NULL);  in kvm_make_all_cpus_request()
359 void kvm_flush_remote_tlbs(struct kvm *kvm)  in kvm_flush_remote_tlbs()  argument
361 	++kvm->stat.generic.remote_tlb_flush_requests;  in kvm_flush_remote_tlbs()
374 	if (!kvm_arch_flush_remote_tlb(kvm)  in kvm_flush_remote_tlbs()
375 	    || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))  in kvm_flush_remote_tlbs()
376 		++kvm->stat.generic.remote_tlb_flush;  in kvm_flush_remote_tlbs()
381 static void kvm_flush_shadow_all(struct kvm *kvm)  in kvm_flush_shadow_all()  argument
383 	kvm_arch_flush_shadow_all(kvm);  in kvm_flush_shadow_all()
384 	kvm_arch_guest_memory_reclaimed(kvm);  in kvm_flush_shadow_all()
469 static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)  in kvm_vcpu_init()  argument
473 	vcpu->kvm = kvm;  in kvm_vcpu_init()
509 void kvm_destroy_vcpus(struct kvm *kvm)  in kvm_destroy_vcpus()  argument
514 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_destroy_vcpus()
516 		xa_erase(&kvm->vcpu_array, i);  in kvm_destroy_vcpus()
519 	atomic_set(&kvm->online_vcpus, 0);  in kvm_destroy_vcpus()
524 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)  in mmu_notifier_to_kvm()
526 	return container_of(mn, struct kvm, mmu_notifier);  in mmu_notifier_to_kvm()
533 	struct kvm *kvm = mmu_notifier_to_kvm(mn);  in kvm_mmu_notifier_invalidate_range()  local
536 	idx = srcu_read_lock(&kvm->srcu);  in kvm_mmu_notifier_invalidate_range()
537 	kvm_arch_mmu_notifier_invalidate_range(kvm, start, end);  in kvm_mmu_notifier_invalidate_range()
538 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_mmu_notifier_invalidate_range()
541 typedef bool (*hva_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
543 typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start,
546 typedef void (*on_unlock_fn_t)(struct kvm *kvm);
578 static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,  in __kvm_handle_hva_range()  argument
595 	idx = srcu_read_lock(&kvm->srcu);  in __kvm_handle_hva_range()
600 		slots = __kvm_memslots(kvm, i);  in __kvm_handle_hva_range()
629 				KVM_MMU_LOCK(kvm);  in __kvm_handle_hva_range()
631 					range->on_lock(kvm, range->start, range->end);  in __kvm_handle_hva_range()
635 			ret |= range->handler(kvm, &gfn_range);  in __kvm_handle_hva_range()
640 		kvm_flush_remote_tlbs(kvm);  in __kvm_handle_hva_range()
643 		KVM_MMU_UNLOCK(kvm);  in __kvm_handle_hva_range()
645 			range->on_unlock(kvm);  in __kvm_handle_hva_range()
648 	srcu_read_unlock(&kvm->srcu, idx);  in __kvm_handle_hva_range()
660 	struct kvm *kvm = mmu_notifier_to_kvm(mn);  in kvm_handle_hva_range()  local
672 	return __kvm_handle_hva_range(kvm, &range);  in kvm_handle_hva_range()
680 	struct kvm *kvm = mmu_notifier_to_kvm(mn);  in kvm_handle_hva_range_no_flush()  local
692 	return __kvm_handle_hva_range(kvm, &range);  in kvm_handle_hva_range_no_flush()
699 	struct kvm *kvm = mmu_notifier_to_kvm(mn);  in kvm_mmu_notifier_change_pte()  local
711 	WARN_ON_ONCE(!READ_ONCE(kvm->mn_active_invalidate_count));  in kvm_mmu_notifier_change_pte()
712 	if (!READ_ONCE(kvm->mmu_invalidate_in_progress))  in kvm_mmu_notifier_change_pte()
718 void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,  in kvm_mmu_invalidate_begin()  argument
726 	kvm->mmu_invalidate_in_progress++;  in kvm_mmu_invalidate_begin()
727 	if (likely(kvm->mmu_invalidate_in_progress == 1)) {  in kvm_mmu_invalidate_begin()
728 		kvm->mmu_invalidate_range_start = start;  in kvm_mmu_invalidate_begin()
729 		kvm->mmu_invalidate_range_end = end;  in kvm_mmu_invalidate_begin()
740 		kvm->mmu_invalidate_range_start =  in kvm_mmu_invalidate_begin()
741 			min(kvm->mmu_invalidate_range_start, start);  in kvm_mmu_invalidate_begin()
742 		kvm->mmu_invalidate_range_end =  in kvm_mmu_invalidate_begin()
743 			max(kvm->mmu_invalidate_range_end, end);  in kvm_mmu_invalidate_begin()
750 	struct kvm *kvm = mmu_notifier_to_kvm(mn);  in kvm_mmu_notifier_invalidate_range_start()  local
772 	spin_lock(&kvm->mn_invalidate_lock);  in kvm_mmu_notifier_invalidate_range_start()
773 	kvm->mn_active_invalidate_count++;  in kvm_mmu_notifier_invalidate_range_start()
774 	spin_unlock(&kvm->mn_invalidate_lock);  in kvm_mmu_notifier_invalidate_range_start()
786 	gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end,  in kvm_mmu_notifier_invalidate_range_start()
789 	__kvm_handle_hva_range(kvm, &hva_range);  in kvm_mmu_notifier_invalidate_range_start()
794 void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,  in kvm_mmu_invalidate_end()  argument
802 	kvm->mmu_invalidate_seq++;  in kvm_mmu_invalidate_end()
809 	kvm->mmu_invalidate_in_progress--;  in kvm_mmu_invalidate_end()
815 	struct kvm *kvm = mmu_notifier_to_kvm(mn);  in kvm_mmu_notifier_invalidate_range_end()  local
828 	__kvm_handle_hva_range(kvm, &hva_range);  in kvm_mmu_notifier_invalidate_range_end()
831 	spin_lock(&kvm->mn_invalidate_lock);  in kvm_mmu_notifier_invalidate_range_end()
832 	wake = (--kvm->mn_active_invalidate_count == 0);  in kvm_mmu_notifier_invalidate_range_end()
833 	spin_unlock(&kvm->mn_invalidate_lock);  in kvm_mmu_notifier_invalidate_range_end()
840 		rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);  in kvm_mmu_notifier_invalidate_range_end()
842 	BUG_ON(kvm->mmu_invalidate_in_progress < 0);  in kvm_mmu_notifier_invalidate_range_end()
891 	struct kvm *kvm = mmu_notifier_to_kvm(mn);  in kvm_mmu_notifier_release()  local
894 	idx = srcu_read_lock(&kvm->srcu);  in kvm_mmu_notifier_release()
895 	kvm_flush_shadow_all(kvm);  in kvm_mmu_notifier_release()
896 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_mmu_notifier_release()
910 static int kvm_init_mmu_notifier(struct kvm *kvm)  in kvm_init_mmu_notifier()  argument
912 	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;  in kvm_init_mmu_notifier()
913 	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);  in kvm_init_mmu_notifier()
918 static int kvm_init_mmu_notifier(struct kvm *kvm)  in kvm_init_mmu_notifier()  argument
930 	struct kvm *kvm = container_of(bl, struct kvm, pm_notifier);  in kvm_pm_notifier_call()  local
932 	return kvm_arch_pm_notifier(kvm, state);  in kvm_pm_notifier_call()
935 static void kvm_init_pm_notifier(struct kvm *kvm)  in kvm_init_pm_notifier()  argument
937 	kvm->pm_notifier.notifier_call = kvm_pm_notifier_call;  in kvm_init_pm_notifier()
939 	kvm->pm_notifier.priority = INT_MAX;  in kvm_init_pm_notifier()
940 	register_pm_notifier(&kvm->pm_notifier);  in kvm_init_pm_notifier()
943 static void kvm_destroy_pm_notifier(struct kvm *kvm)  in kvm_destroy_pm_notifier()  argument
945 	unregister_pm_notifier(&kvm->pm_notifier);  in kvm_destroy_pm_notifier()
948 static void kvm_init_pm_notifier(struct kvm *kvm)  in kvm_init_pm_notifier()  argument
952 static void kvm_destroy_pm_notifier(struct kvm *kvm)  in kvm_destroy_pm_notifier()  argument
967 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)  in kvm_free_memslot()  argument
971 	kvm_arch_free_memslot(kvm, slot);  in kvm_free_memslot()
976 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)  in kvm_free_memslots()  argument
992 		kvm_free_memslot(kvm, memslot);  in kvm_free_memslots()
1008 static void kvm_destroy_vm_debugfs(struct kvm *kvm)  in kvm_destroy_vm_debugfs()  argument
1014 	if (IS_ERR(kvm->debugfs_dentry))  in kvm_destroy_vm_debugfs()
1017 	debugfs_remove_recursive(kvm->debugfs_dentry);  in kvm_destroy_vm_debugfs()
1019 	if (kvm->debugfs_stat_data) {  in kvm_destroy_vm_debugfs()
1021 			kfree(kvm->debugfs_stat_data[i]);  in kvm_destroy_vm_debugfs()
1022 		kfree(kvm->debugfs_stat_data);  in kvm_destroy_vm_debugfs()
1026 static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname)  in kvm_create_vm_debugfs()  argument
1054 	kvm->debugfs_dentry = dent;  in kvm_create_vm_debugfs()
1055 	kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,  in kvm_create_vm_debugfs()
1056 					 sizeof(*kvm->debugfs_stat_data),  in kvm_create_vm_debugfs()
1058 	if (!kvm->debugfs_stat_data)  in kvm_create_vm_debugfs()
1067 		stat_data->kvm = kvm;  in kvm_create_vm_debugfs()
1070 		kvm->debugfs_stat_data[i] = stat_data;  in kvm_create_vm_debugfs()
1072 				    kvm->debugfs_dentry, stat_data,  in kvm_create_vm_debugfs()
1082 		stat_data->kvm = kvm;  in kvm_create_vm_debugfs()
1085 		kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data;  in kvm_create_vm_debugfs()
1087 				    kvm->debugfs_dentry, stat_data,  in kvm_create_vm_debugfs()
1091 	ret = kvm_arch_create_vm_debugfs(kvm);  in kvm_create_vm_debugfs()
1097 	kvm_destroy_vm_debugfs(kvm);  in kvm_create_vm_debugfs()
1105 int __weak kvm_arch_post_init_vm(struct kvm *kvm)  in kvm_arch_post_init_vm()  argument
1114 void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)  in kvm_arch_pre_destroy_vm()  argument
1124 int __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)  in kvm_arch_create_vm_debugfs()  argument
1129 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)  in kvm_create_vm()
1131 	struct kvm *kvm = kvm_arch_alloc_vm();  in kvm_create_vm()  local
1136 	if (!kvm)  in kvm_create_vm()
1142 	KVM_MMU_LOCK_INIT(kvm);  in kvm_create_vm()
1144 	kvm->mm = current->mm;  in kvm_create_vm()
1145 	kvm_eventfd_init(kvm);  in kvm_create_vm()
1146 	mutex_init(&kvm->lock);  in kvm_create_vm()
1147 	mutex_init(&kvm->irq_lock);  in kvm_create_vm()
1148 	mutex_init(&kvm->slots_lock);  in kvm_create_vm()
1149 	mutex_init(&kvm->slots_arch_lock);  in kvm_create_vm()
1150 	spin_lock_init(&kvm->mn_invalidate_lock);  in kvm_create_vm()
1151 	rcuwait_init(&kvm->mn_memslots_update_rcuwait);  in kvm_create_vm()
1152 	xa_init(&kvm->vcpu_array);  in kvm_create_vm()
1154 	INIT_LIST_HEAD(&kvm->gpc_list);  in kvm_create_vm()
1155 	spin_lock_init(&kvm->gpc_lock);  in kvm_create_vm()
1157 	INIT_LIST_HEAD(&kvm->devices);  in kvm_create_vm()
1158 	kvm->max_vcpus = KVM_MAX_VCPUS;  in kvm_create_vm()
1166 	kvm->debugfs_dentry = ERR_PTR(-ENOENT);  in kvm_create_vm()
1168 	snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d",  in kvm_create_vm()
1171 	if (init_srcu_struct(&kvm->srcu))  in kvm_create_vm()
1173 	if (init_srcu_struct(&kvm->irq_srcu))  in kvm_create_vm()
1176 	refcount_set(&kvm->users_count, 1);  in kvm_create_vm()
1179 			slots = &kvm->__memslots[i][j];  in kvm_create_vm()
1191 		rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]);  in kvm_create_vm()
1195 		rcu_assign_pointer(kvm->buses[i],  in kvm_create_vm()
1197 		if (!kvm->buses[i])  in kvm_create_vm()
1201 	r = kvm_arch_init_vm(kvm, type);  in kvm_create_vm()
1210 	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);  in kvm_create_vm()
1213 	r = kvm_init_mmu_notifier(kvm);  in kvm_create_vm()
1217 	r = kvm_coalesced_mmio_init(kvm);  in kvm_create_vm()
1221 	r = kvm_create_vm_debugfs(kvm, fdname);  in kvm_create_vm()
1225 	r = kvm_arch_post_init_vm(kvm);  in kvm_create_vm()
1230 	list_add(&kvm->vm_list, &vm_list);  in kvm_create_vm()
1234 	kvm_init_pm_notifier(kvm);  in kvm_create_vm()
1236 	return kvm;  in kvm_create_vm()
1239 	kvm_destroy_vm_debugfs(kvm);  in kvm_create_vm()
1241 	kvm_coalesced_mmio_free(kvm);  in kvm_create_vm()
1244 	if (kvm->mmu_notifier.ops)  in kvm_create_vm()
1245 		mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);  in kvm_create_vm()
1250 	kvm_arch_destroy_vm(kvm);  in kvm_create_vm()
1252 	WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));  in kvm_create_vm()
1254 		kfree(kvm_get_bus(kvm, i));  in kvm_create_vm()
1255 	cleanup_srcu_struct(&kvm->irq_srcu);  in kvm_create_vm()
1257 	cleanup_srcu_struct(&kvm->srcu);  in kvm_create_vm()
1259 	kvm_arch_free_vm(kvm);  in kvm_create_vm()
1265 static void kvm_destroy_devices(struct kvm *kvm)  in kvm_destroy_devices()  argument
1274 	list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {  in kvm_destroy_devices()
1280 static void kvm_destroy_vm(struct kvm *kvm)  in kvm_destroy_vm()  argument
1283 	struct mm_struct *mm = kvm->mm;  in kvm_destroy_vm()
1285 	kvm_destroy_pm_notifier(kvm);  in kvm_destroy_vm()
1286 	kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);  in kvm_destroy_vm()
1287 	kvm_destroy_vm_debugfs(kvm);  in kvm_destroy_vm()
1288 	kvm_arch_sync_events(kvm);  in kvm_destroy_vm()
1290 	list_del(&kvm->vm_list);  in kvm_destroy_vm()
1292 	kvm_arch_pre_destroy_vm(kvm);  in kvm_destroy_vm()
1294 	kvm_free_irq_routing(kvm);  in kvm_destroy_vm()
1296 		struct kvm_io_bus *bus = kvm_get_bus(kvm, i);  in kvm_destroy_vm()
1300 		kvm->buses[i] = NULL;  in kvm_destroy_vm()
1302 	kvm_coalesced_mmio_free(kvm);  in kvm_destroy_vm()
1304 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);  in kvm_destroy_vm()
1313 	WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));  in kvm_destroy_vm()
1314 	kvm->mn_active_invalidate_count = 0;  in kvm_destroy_vm()
1316 	kvm_flush_shadow_all(kvm);  in kvm_destroy_vm()
1318 	kvm_arch_destroy_vm(kvm);  in kvm_destroy_vm()
1319 	kvm_destroy_devices(kvm);  in kvm_destroy_vm()
1321 		kvm_free_memslots(kvm, &kvm->__memslots[i][0]);  in kvm_destroy_vm()
1322 		kvm_free_memslots(kvm, &kvm->__memslots[i][1]);  in kvm_destroy_vm()
1324 	cleanup_srcu_struct(&kvm->irq_srcu);  in kvm_destroy_vm()
1325 	cleanup_srcu_struct(&kvm->srcu);  in kvm_destroy_vm()
1326 	kvm_arch_free_vm(kvm);  in kvm_destroy_vm()
1333 void kvm_get_kvm(struct kvm *kvm)  in kvm_get_kvm()  argument
1335 	refcount_inc(&kvm->users_count);  in kvm_get_kvm()
1343 bool kvm_get_kvm_safe(struct kvm *kvm)  in kvm_get_kvm_safe()  argument
1345 	return refcount_inc_not_zero(&kvm->users_count);  in kvm_get_kvm_safe()
1349 void kvm_put_kvm(struct kvm *kvm)  in kvm_put_kvm()  argument
1351 	if (refcount_dec_and_test(&kvm->users_count))  in kvm_put_kvm()
1352 		kvm_destroy_vm(kvm);  in kvm_put_kvm()
1363 void kvm_put_kvm_no_destroy(struct kvm *kvm)  in kvm_put_kvm_no_destroy()  argument
1365 	WARN_ON(refcount_dec_and_test(&kvm->users_count));  in kvm_put_kvm_no_destroy()
1371 	struct kvm *kvm = filp->private_data;  in kvm_vm_release()  local
1373 	kvm_irqfd_release(kvm);  in kvm_vm_release()
1375 	kvm_put_kvm(kvm);  in kvm_vm_release()
1394 static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id)  in kvm_get_inactive_memslots()  argument
1396 	struct kvm_memslots *active = __kvm_memslots(kvm, as_id);  in kvm_get_inactive_memslots()
1399 	return &kvm->__memslots[as_id][node_idx_inactive];  in kvm_get_inactive_memslots()
1474 static void kvm_replace_memslot(struct kvm *kvm,  in kvm_replace_memslot()  argument
1479 	struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);  in kvm_replace_memslot()
1541 static void kvm_swap_active_memslots(struct kvm *kvm, int as_id)  in kvm_swap_active_memslots()  argument
1543 	struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);  in kvm_swap_active_memslots()
1546 	u64 gen = __kvm_memslots(kvm, as_id)->generation;  in kvm_swap_active_memslots()
1556 	spin_lock(&kvm->mn_invalidate_lock);  in kvm_swap_active_memslots()
1557 	prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait);  in kvm_swap_active_memslots()
1558 	while (kvm->mn_active_invalidate_count) {  in kvm_swap_active_memslots()
1560 		spin_unlock(&kvm->mn_invalidate_lock);  in kvm_swap_active_memslots()
1562 		spin_lock(&kvm->mn_invalidate_lock);  in kvm_swap_active_memslots()
1564 	finish_rcuwait(&kvm->mn_memslots_update_rcuwait);  in kvm_swap_active_memslots()
1565 	rcu_assign_pointer(kvm->memslots[as_id], slots);  in kvm_swap_active_memslots()
1566 	spin_unlock(&kvm->mn_invalidate_lock);  in kvm_swap_active_memslots()
1573 	mutex_unlock(&kvm->slots_arch_lock);  in kvm_swap_active_memslots()
1575 	synchronize_srcu_expedited(&kvm->srcu);  in kvm_swap_active_memslots()
1594 	kvm_arch_memslots_updated(kvm, gen);  in kvm_swap_active_memslots()
1599 static int kvm_prepare_memory_region(struct kvm *kvm,  in kvm_prepare_memory_region()  argument
1618 		else if (!kvm->dirty_ring_size) {  in kvm_prepare_memory_region()
1623 			if (kvm_dirty_log_manual_protect_and_init_set(kvm))  in kvm_prepare_memory_region()
1628 	r = kvm_arch_prepare_memory_region(kvm, old, new, change);  in kvm_prepare_memory_region()
1637 static void kvm_commit_memory_region(struct kvm *kvm,  in kvm_commit_memory_region()  argument
1647 		kvm->nr_memslot_pages -= old->npages;  in kvm_commit_memory_region()
1649 		kvm->nr_memslot_pages += new->npages;  in kvm_commit_memory_region()
1651 	kvm_arch_commit_memory_region(kvm, old, new, change);  in kvm_commit_memory_region()
1659 		kvm_free_memslot(kvm, old);  in kvm_commit_memory_region()
1691 static void kvm_activate_memslot(struct kvm *kvm,  in kvm_activate_memslot()  argument
1697 	kvm_swap_active_memslots(kvm, as_id);  in kvm_activate_memslot()
1700 	kvm_replace_memslot(kvm, old, new);  in kvm_activate_memslot()
1716 static void kvm_invalidate_memslot(struct kvm *kvm,  in kvm_invalidate_memslot()  argument
1727 	kvm_replace_memslot(kvm, old, invalid_slot);  in kvm_invalidate_memslot()
1734 	kvm_swap_active_memslots(kvm, old->as_id);  in kvm_invalidate_memslot()
1742 	kvm_arch_flush_shadow_memslot(kvm, old);  in kvm_invalidate_memslot()
1743 	kvm_arch_guest_memory_reclaimed(kvm);  in kvm_invalidate_memslot()
1746 	mutex_lock(&kvm->slots_arch_lock);  in kvm_invalidate_memslot()
1758 static void kvm_create_memslot(struct kvm *kvm,  in kvm_create_memslot()  argument
1762 	kvm_replace_memslot(kvm, NULL, new);  in kvm_create_memslot()
1763 	kvm_activate_memslot(kvm, NULL, new);  in kvm_create_memslot()
1766 static void kvm_delete_memslot(struct kvm *kvm,  in kvm_delete_memslot()  argument
1774 	kvm_replace_memslot(kvm, old, NULL);  in kvm_delete_memslot()
1775 	kvm_activate_memslot(kvm, invalid_slot, NULL);  in kvm_delete_memslot()
1778 static void kvm_move_memslot(struct kvm *kvm,  in kvm_move_memslot()  argument
1787 	kvm_replace_memslot(kvm, old, new);  in kvm_move_memslot()
1788 	kvm_activate_memslot(kvm, invalid_slot, new);  in kvm_move_memslot()
1791 static void kvm_update_flags_memslot(struct kvm *kvm,  in kvm_update_flags_memslot()  argument
1800 	kvm_replace_memslot(kvm, old, new);  in kvm_update_flags_memslot()
1801 	kvm_activate_memslot(kvm, old, new);  in kvm_update_flags_memslot()
1804 static int kvm_set_memslot(struct kvm *kvm,  in kvm_set_memslot()  argument
1826 	mutex_lock(&kvm->slots_arch_lock);  in kvm_set_memslot()
1844 			mutex_unlock(&kvm->slots_arch_lock);  in kvm_set_memslot()
1847 		kvm_invalidate_memslot(kvm, old, invalid_slot);  in kvm_set_memslot()
1850 	r = kvm_prepare_memory_region(kvm, old, new, change);  in kvm_set_memslot()
1859 			kvm_activate_memslot(kvm, invalid_slot, old);  in kvm_set_memslot()
1862 			mutex_unlock(&kvm->slots_arch_lock);  in kvm_set_memslot()
1875 		kvm_create_memslot(kvm, new);  in kvm_set_memslot()
1877 		kvm_delete_memslot(kvm, old, invalid_slot);  in kvm_set_memslot()
1879 		kvm_move_memslot(kvm, old, new, invalid_slot);  in kvm_set_memslot()
1881 		kvm_update_flags_memslot(kvm, old, new);  in kvm_set_memslot()
1894 	kvm_commit_memory_region(kvm, old, new, change);  in kvm_set_memslot()
1920 int __kvm_set_memory_region(struct kvm *kvm,  in __kvm_set_memory_region()  argument
1957 	slots = __kvm_memslots(kvm, as_id);  in __kvm_set_memory_region()
1969 		if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))  in __kvm_set_memory_region()
1972 		return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE);  in __kvm_set_memory_region()
1985 		if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)  in __kvm_set_memory_region()
2017 	r = kvm_set_memslot(kvm, old, new, change);  in __kvm_set_memory_region()
2024 int kvm_set_memory_region(struct kvm *kvm,  in kvm_set_memory_region()  argument
2029 	mutex_lock(&kvm->slots_lock);  in kvm_set_memory_region()
2030 	r = __kvm_set_memory_region(kvm, mem);  in kvm_set_memory_region()
2031 	mutex_unlock(&kvm->slots_lock);  in kvm_set_memory_region()
2036 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,  in kvm_vm_ioctl_set_memory_region()  argument
2042 	return kvm_set_memory_region(kvm, mem);  in kvm_vm_ioctl_set_memory_region()
2053 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,  in kvm_get_dirty_log()  argument
2062 	if (kvm->dirty_ring_size)  in kvm_get_dirty_log()
2073 	slots = __kvm_memslots(kvm, as_id);  in kvm_get_dirty_log()
2078 	kvm_arch_sync_dirty_log(kvm, *memslot);  in kvm_get_dirty_log()
2116 static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)  in kvm_get_dirty_log_protect()  argument
2127 	if (kvm->dirty_ring_size)  in kvm_get_dirty_log_protect()
2135 	slots = __kvm_memslots(kvm, as_id);  in kvm_get_dirty_log_protect()
2142 	kvm_arch_sync_dirty_log(kvm, memslot);  in kvm_get_dirty_log_protect()
2146 	if (kvm->manual_dirty_log_protect) {  in kvm_get_dirty_log_protect()
2160 		KVM_MMU_LOCK(kvm);  in kvm_get_dirty_log_protect()
2173 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,  in kvm_get_dirty_log_protect()
2176 		KVM_MMU_UNLOCK(kvm);  in kvm_get_dirty_log_protect()
2180 		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);  in kvm_get_dirty_log_protect()
2207 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,  in kvm_vm_ioctl_get_dirty_log()  argument
2212 	mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_get_dirty_log()
2214 	r = kvm_get_dirty_log_protect(kvm, log);  in kvm_vm_ioctl_get_dirty_log()
2216 	mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_get_dirty_log()
2226 static int kvm_clear_dirty_log_protect(struct kvm *kvm,  in kvm_clear_dirty_log_protect()  argument
2239 	if (kvm->dirty_ring_size)  in kvm_clear_dirty_log_protect()
2250 	slots = __kvm_memslots(kvm, as_id);  in kvm_clear_dirty_log_protect()
2264 	kvm_arch_sync_dirty_log(kvm, memslot);  in kvm_clear_dirty_log_protect()
2271 	KVM_MMU_LOCK(kvm);  in kvm_clear_dirty_log_protect()
2290 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,  in kvm_clear_dirty_log_protect()
2294 	KVM_MMU_UNLOCK(kvm);  in kvm_clear_dirty_log_protect()
2297 		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);  in kvm_clear_dirty_log_protect()
2302 static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,  in kvm_vm_ioctl_clear_dirty_log()  argument
2307 	mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_clear_dirty_log()
2309 	r = kvm_clear_dirty_log_protect(kvm, log);  in kvm_vm_ioctl_clear_dirty_log()
2311 	mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_clear_dirty_log()
2316 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)  in gfn_to_memslot()  argument
2318 	return __gfn_to_memslot(kvm_memslots(kvm), gfn);  in gfn_to_memslot()
2355 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_is_visible_gfn()  argument
2357 	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);  in kvm_is_visible_gfn()
2428 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)  in gfn_to_hva()  argument
2430 	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);  in gfn_to_hva()
2459 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)  in gfn_to_hva_prot()  argument
2461 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);  in gfn_to_hva_prot()
2729 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,  in gfn_to_pfn_prot()  argument
2732 	return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,  in gfn_to_pfn_prot()
2755 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)  in gfn_to_pfn()  argument
2757 	return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);  in gfn_to_pfn()
2790 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)  in gfn_to_page()  argument
2795 	pfn = gfn_to_pfn(kvm, gfn);  in gfn_to_page()
2825 	pfn = gfn_to_pfn(vcpu->kvm, gfn);  in kvm_vcpu_map()
2992 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,  in kvm_read_guest_page()  argument
2995 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);  in kvm_read_guest_page()
3010 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)  in kvm_read_guest()  argument
3018 		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);  in kvm_read_guest()
3078 static int __kvm_write_guest_page(struct kvm *kvm,  in __kvm_write_guest_page()  argument
3091 	mark_page_dirty_in_slot(kvm, memslot, gfn);  in __kvm_write_guest_page()
3095 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,  in kvm_write_guest_page()  argument
3098 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);  in kvm_write_guest_page()
3100 	return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len);  in kvm_write_guest_page()
3109 	return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len);  in kvm_vcpu_write_guest_page()
3113 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,  in kvm_write_guest()  argument
3122 		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);  in kvm_write_guest()
3196 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,  in kvm_gfn_to_hva_cache_init()  argument
3199 	struct kvm_memslots *slots = kvm_memslots(kvm);  in kvm_gfn_to_hva_cache_init()
3204 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,  in kvm_write_guest_offset_cached()  argument
3208 	struct kvm_memslots *slots = kvm_memslots(kvm);  in kvm_write_guest_offset_cached()
3224 		return kvm_write_guest(kvm, gpa, data, len);  in kvm_write_guest_offset_cached()
3229 	mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT);  in kvm_write_guest_offset_cached()
3235 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,  in kvm_write_guest_cached()  argument
3238 	return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);  in kvm_write_guest_cached()
3242 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,  in kvm_read_guest_offset_cached()  argument
3246 	struct kvm_memslots *slots = kvm_memslots(kvm);  in kvm_read_guest_offset_cached()
3262 		return kvm_read_guest(kvm, gpa, data, len);  in kvm_read_guest_offset_cached()
3272 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,  in kvm_read_guest_cached()  argument
3275 	return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len);  in kvm_read_guest_cached()
3279 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)  in kvm_clear_guest()  argument
3288 		ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, len);  in kvm_clear_guest()
3299 void mark_page_dirty_in_slot(struct kvm *kvm,  in mark_page_dirty_in_slot()  argument
3306 	if (WARN_ON_ONCE(!vcpu) || WARN_ON_ONCE(vcpu->kvm != kvm))  in mark_page_dirty_in_slot()
3314 		if (kvm->dirty_ring_size)  in mark_page_dirty_in_slot()
3323 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)  in mark_page_dirty()  argument
3327 	memslot = gfn_to_memslot(kvm, gfn);  in mark_page_dirty()
3328 	mark_page_dirty_in_slot(kvm, memslot, gfn);  in mark_page_dirty()
3337 	mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn);  in kvm_vcpu_mark_page_dirty()
3405 	int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_vcpu_check_block()
3418 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_vcpu_check_block()
3483 	struct kvm *kvm = vcpu->kvm;  in kvm_vcpu_max_halt_poll_ns()  local
3485 	if (kvm->override_halt_poll_ns) {  in kvm_vcpu_max_halt_poll_ns()
3493 		return READ_ONCE(kvm->max_halt_poll_ns);  in kvm_vcpu_max_halt_poll_ns()
3725 	struct kvm *kvm = me->kvm;  in kvm_vcpu_on_spin()  local
3727 	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;  in kvm_vcpu_on_spin()
3742 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_vcpu_on_spin()
3763 				kvm->last_boosted_vcpu = i;  in kvm_vcpu_on_spin()
3779 static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff)  in kvm_page_in_dirty_ring()  argument
3784 	     kvm->dirty_ring_size / PAGE_SIZE);  in kvm_page_in_dirty_ring()
3803 		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);  in kvm_vcpu_fault()
3805 	else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff))  in kvm_vcpu_fault()
3825 	if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) ||  in kvm_vcpu_mmap()
3826 	     kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) &&  in kvm_vcpu_mmap()
3838 	kvm_put_kvm(vcpu->kvm);  in kvm_vcpu_release()
3881 					    vcpu->kvm->debugfs_dentry);  in kvm_create_vcpu_debugfs()
3892 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)  in kvm_vm_ioctl_create_vcpu()  argument
3901 	mutex_lock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3902 	if (kvm->created_vcpus >= kvm->max_vcpus) {  in kvm_vm_ioctl_create_vcpu()
3903 		mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3907 	r = kvm_arch_vcpu_precreate(kvm, id);  in kvm_vm_ioctl_create_vcpu()
3909 		mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3913 	kvm->created_vcpus++;  in kvm_vm_ioctl_create_vcpu()
3914 	mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3930 	kvm_vcpu_init(vcpu, kvm, id);  in kvm_vm_ioctl_create_vcpu()
3936 	if (kvm->dirty_ring_size) {  in kvm_vm_ioctl_create_vcpu()
3938 					 id, kvm->dirty_ring_size);  in kvm_vm_ioctl_create_vcpu()
3943 	mutex_lock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3944 	if (kvm_get_vcpu_by_id(kvm, id)) {  in kvm_vm_ioctl_create_vcpu()
3949 	vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);  in kvm_vm_ioctl_create_vcpu()
3950 	r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT);  in kvm_vm_ioctl_create_vcpu()
3956 	kvm_get_kvm(kvm);  in kvm_vm_ioctl_create_vcpu()
3959 		xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx);  in kvm_vm_ioctl_create_vcpu()
3960 		kvm_put_kvm_no_destroy(kvm);  in kvm_vm_ioctl_create_vcpu()
3969 	atomic_inc(&kvm->online_vcpus);  in kvm_vm_ioctl_create_vcpu()
3971 	mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3977 	mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3986 	mutex_lock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
3987 	kvm->created_vcpus--;  in kvm_vm_ioctl_create_vcpu()
3988 	mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_create_vcpu()
4050 	if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)  in kvm_vcpu_ioctl()
4260 	if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)  in kvm_vcpu_compat_ioctl()
4326 	if (dev->kvm->mm != current->mm || dev->kvm->vm_dead)  in kvm_device_ioctl()
4347 	struct kvm *kvm = dev->kvm;  in kvm_device_release()  local
4350 		mutex_lock(&kvm->lock);  in kvm_device_release()
4353 		mutex_unlock(&kvm->lock);  in kvm_device_release()
4356 	kvm_put_kvm(kvm);  in kvm_device_release()
4400 static int kvm_ioctl_create_device(struct kvm *kvm,  in kvm_ioctl_create_device()  argument
4425 	dev->kvm = kvm;  in kvm_ioctl_create_device()
4427 	mutex_lock(&kvm->lock);  in kvm_ioctl_create_device()
4430 		mutex_unlock(&kvm->lock);  in kvm_ioctl_create_device()
4434 	list_add(&dev->vm_node, &kvm->devices);  in kvm_ioctl_create_device()
4435 	mutex_unlock(&kvm->lock);  in kvm_ioctl_create_device()
4440 	kvm_get_kvm(kvm);  in kvm_ioctl_create_device()
4443 		kvm_put_kvm_no_destroy(kvm);  in kvm_ioctl_create_device()
4444 		mutex_lock(&kvm->lock);  in kvm_ioctl_create_device()
4448 		mutex_unlock(&kvm->lock);  in kvm_ioctl_create_device()
4458 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)  in kvm_vm_ioctl_check_extension_generic()  argument
4515 	return kvm_vm_ioctl_check_extension(kvm, arg);  in kvm_vm_ioctl_check_extension_generic()
4518 static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size)  in kvm_vm_ioctl_enable_dirty_log_ring()  argument
4539 	if (kvm->dirty_ring_size)  in kvm_vm_ioctl_enable_dirty_log_ring()
4542 	mutex_lock(&kvm->lock);  in kvm_vm_ioctl_enable_dirty_log_ring()
4544 	if (kvm->created_vcpus) {  in kvm_vm_ioctl_enable_dirty_log_ring()
4548 		kvm->dirty_ring_size = size;  in kvm_vm_ioctl_enable_dirty_log_ring()
4552 	mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_enable_dirty_log_ring()
4556 static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm)  in kvm_vm_ioctl_reset_dirty_pages()  argument
4562 	if (!kvm->dirty_ring_size)  in kvm_vm_ioctl_reset_dirty_pages()
4565 	mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_reset_dirty_pages()
4567 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_vm_ioctl_reset_dirty_pages()
4568 		cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring);  in kvm_vm_ioctl_reset_dirty_pages()
4570 	mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_reset_dirty_pages()
4573 		kvm_flush_remote_tlbs(kvm);  in kvm_vm_ioctl_reset_dirty_pages()
4578 int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,  in kvm_vm_ioctl_enable_cap()  argument
4584 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,  in kvm_vm_ioctl_enable_cap_generic()  argument
4597 		kvm->manual_dirty_log_protect = cap->args[0];  in kvm_vm_ioctl_enable_cap_generic()
4605 		kvm->max_halt_poll_ns = cap->args[0];  in kvm_vm_ioctl_enable_cap_generic()
4614 		kvm->override_halt_poll_ns = true;  in kvm_vm_ioctl_enable_cap_generic()
4620 		if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap))  in kvm_vm_ioctl_enable_cap_generic()
4623 		return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);  in kvm_vm_ioctl_enable_cap_generic()
4625 		return kvm_vm_ioctl_enable_cap(kvm, cap);  in kvm_vm_ioctl_enable_cap_generic()
4632 	struct kvm *kvm = file->private_data;  in kvm_vm_stats_read()  local
4634 	return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header,  in kvm_vm_stats_read()
4635 				&kvm_vm_stats_desc[0], &kvm->stat,  in kvm_vm_stats_read()
4636 				sizeof(kvm->stat), user_buffer, size, offset);  in kvm_vm_stats_read()
4644 static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm)  in kvm_vm_ioctl_get_stats_fd()  argument
4654 			&kvm_vm_stats_fops, kvm, O_RDONLY);  in kvm_vm_ioctl_get_stats_fd()
4668 	struct kvm *kvm = filp->private_data;  in kvm_vm_ioctl()  local
4672 	if (kvm->mm != current->mm || kvm->vm_dead)  in kvm_vm_ioctl()
4676 		r = kvm_vm_ioctl_create_vcpu(kvm, arg);  in kvm_vm_ioctl()
4684 		r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);  in kvm_vm_ioctl()
4695 		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);  in kvm_vm_ioctl()
4704 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);  in kvm_vm_ioctl()
4714 		r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);  in kvm_vm_ioctl()
4725 		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);  in kvm_vm_ioctl()
4734 		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);  in kvm_vm_ioctl()
4744 		r = kvm_irqfd(kvm, &data);  in kvm_vm_ioctl()
4753 		r = kvm_ioeventfd(kvm, &data);  in kvm_vm_ioctl()
4763 		r = kvm_send_userspace_msi(kvm, &msi);  in kvm_vm_ioctl()
4776 		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,  in kvm_vm_ioctl()
4801 		if (!kvm_arch_can_set_irq_routing(kvm))  in kvm_vm_ioctl()
4817 		r = kvm_set_irq_routing(kvm, entries, routing.nr,  in kvm_vm_ioctl()
4830 		r = kvm_ioctl_create_device(kvm, &cd);  in kvm_vm_ioctl()
4842 		r = kvm_vm_ioctl_check_extension_generic(kvm, arg);  in kvm_vm_ioctl()
4845 		r = kvm_vm_ioctl_reset_dirty_pages(kvm);  in kvm_vm_ioctl()
4848 		r = kvm_vm_ioctl_get_stats_fd(kvm);  in kvm_vm_ioctl()
4886 	struct kvm *kvm = filp->private_data;  in kvm_vm_compat_ioctl()  local
4889 	if (kvm->mm != current->mm || kvm->vm_dead)  in kvm_vm_compat_ioctl()
4911 		r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);  in kvm_vm_compat_ioctl()
4927 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);  in kvm_vm_compat_ioctl()
4954 	struct kvm *kvm;  in kvm_dev_ioctl_create_vm()  local
4963 	kvm = kvm_create_vm(type, fdname);  in kvm_dev_ioctl_create_vm()
4964 	if (IS_ERR(kvm)) {  in kvm_dev_ioctl_create_vm()
4965 		r = PTR_ERR(kvm);  in kvm_dev_ioctl_create_vm()
4969 	file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);  in kvm_dev_ioctl_create_vm()
4981 	kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);  in kvm_dev_ioctl_create_vm()
4987 	kvm_put_kvm(kvm);  in kvm_dev_ioctl_create_vm()
5249 	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);  in kvm_io_bus_write()
5269 	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);  in kvm_io_bus_write_cookie()
5320 	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);  in kvm_io_bus_read()
5328 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,  in kvm_io_bus_register_dev()  argument
5335 	bus = kvm_get_bus(kvm, bus_idx);  in kvm_io_bus_register_dev()
5363 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);  in kvm_io_bus_register_dev()
5364 	synchronize_srcu_expedited(&kvm->srcu);  in kvm_io_bus_register_dev()
5370 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,  in kvm_io_bus_unregister_dev()  argument
5376 	lockdep_assert_held(&kvm->slots_lock);  in kvm_io_bus_unregister_dev()
5378 	bus = kvm_get_bus(kvm, bus_idx);  in kvm_io_bus_unregister_dev()
5400 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);  in kvm_io_bus_unregister_dev()
5401 	synchronize_srcu_expedited(&kvm->srcu);  in kvm_io_bus_unregister_dev()
5417 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,  in kvm_io_bus_get_dev()  argument
5424 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvm_io_bus_get_dev()
5426 	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);  in kvm_io_bus_get_dev()
5437 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvm_io_bus_get_dev()
5456 	if (!kvm_get_kvm_safe(stat_data->kvm))  in kvm_debugfs_open()
5463 		kvm_put_kvm(stat_data->kvm);  in kvm_debugfs_open()
5474 	kvm_put_kvm(stat_data->kvm);  in kvm_debugfs_release()
5479 static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)  in kvm_get_stat_per_vm()  argument
5481 	*val = *(u64 *)((void *)(&kvm->stat) + offset);  in kvm_get_stat_per_vm()
5486 static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)  in kvm_clear_stat_per_vm()  argument
5488 	*(u64 *)((void *)(&kvm->stat) + offset) = 0;  in kvm_clear_stat_per_vm()
5493 static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)  in kvm_get_stat_per_vcpu()  argument
5500 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_get_stat_per_vcpu()
5506 static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)  in kvm_clear_stat_per_vcpu()  argument
5511 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_clear_stat_per_vcpu()
5524 		r = kvm_get_stat_per_vm(stat_data->kvm,  in kvm_stat_data_get()
5528 		r = kvm_get_stat_per_vcpu(stat_data->kvm,  in kvm_stat_data_get()
5546 		r = kvm_clear_stat_per_vm(stat_data->kvm,  in kvm_stat_data_clear()
5550 		r = kvm_clear_stat_per_vcpu(stat_data->kvm,  in kvm_stat_data_clear()
5577 	struct kvm *kvm;  in vm_stat_get()  local
5582 	list_for_each_entry(kvm, &vm_list, vm_list) {  in vm_stat_get()
5583 		kvm_get_stat_per_vm(kvm, offset, &tmp_val);  in vm_stat_get()
5593 	struct kvm *kvm;  in vm_stat_clear()  local
5599 	list_for_each_entry(kvm, &vm_list, vm_list) {  in vm_stat_clear()
5600 		kvm_clear_stat_per_vm(kvm, offset);  in vm_stat_clear()
5613 	struct kvm *kvm;  in vcpu_stat_get()  local
5618 	list_for_each_entry(kvm, &vm_list, vm_list) {  in vcpu_stat_get()
5619 		kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);  in vcpu_stat_get()
5629 	struct kvm *kvm;  in vcpu_stat_clear()  local
5635 	list_for_each_entry(kvm, &vm_list, vm_list) {  in vcpu_stat_clear()
5636 		kvm_clear_stat_per_vcpu(kvm, offset);  in vcpu_stat_clear()
5647 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)  in kvm_uevent_notify_change()  argument
5652 	if (!kvm_dev.this_device || !kvm)  in kvm_uevent_notify_change()
5675 		kvm->userspace_pid = task_pid_nr(current);  in kvm_uevent_notify_change()
5679 	add_uevent_var(env, "PID=%d", kvm->userspace_pid);  in kvm_uevent_notify_change()
5681 	if (!IS_ERR(kvm->debugfs_dentry)) {  in kvm_uevent_notify_change()
5685 			tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);  in kvm_uevent_notify_change()
6001 	struct kvm *kvm;  member
6017 	struct kvm *kvm = init_context->kvm;  in kvm_vm_worker_thread()  local
6049 		err = thread_fn(kvm, data);  in kvm_vm_worker_thread()
6073 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,  in kvm_vm_create_worker_thread()  argument
6081 	init_context.kvm = kvm;  in kvm_vm_create_worker_thread()