Lines Matching +full:fails +full:- +full:without +full:- +full:test +full:- +full:cd
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Kernel-based Virtual Machine driver for Linux
5 * This module enables machines with Intel VT-x extensions to run virtual
6 * machines without emulation or binary translation.
77 /* Default doubles per-vcpu halt_poll_ns. */
87 /* Default resets per-vcpu halt_poll_ns . */
95 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
127 * - Prevent a compat task from opening /dev/kvm
128 * - If the open has been done by a 64bit task, and the KVM fd
132 unsigned long arg) { return -EINVAL; } in kvm_no_compat_ioctl()
136 return is_compat_task() ? -ENODEV : 0; in kvm_no_compat_open()
207 preempt_notifier_register(&vcpu->preempt_notifier); in vcpu_load()
217 preempt_notifier_unregister(&vcpu->preempt_notifier); in vcpu_put()
273 cpu = vcpu->cpu; in kvm_make_vcpus_request_mask()
278 if (tmp != NULL && cpu != -1 && cpu != me && in kvm_make_vcpus_request_mask()
315 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); in kvm_flush_remote_tlbs()
319 * mode. Pairs with a memory barrier in arch-specific code. in kvm_flush_remote_tlbs()
320 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest in kvm_flush_remote_tlbs()
322 * - powerpc: smp_mb in kvmppc_prepare_to_enter. in kvm_flush_remote_tlbs()
325 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that in kvm_flush_remote_tlbs()
330 ++kvm->stat.remote_tlb_flush; in kvm_flush_remote_tlbs()
331 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); in kvm_flush_remote_tlbs()
345 gfp_flags |= mc->gfp_zero; in mmu_memory_cache_alloc_obj()
347 if (mc->kmem_cache) in mmu_memory_cache_alloc_obj()
348 return kmem_cache_alloc(mc->kmem_cache, gfp_flags); in mmu_memory_cache_alloc_obj()
357 if (mc->nobjs >= min) in kvm_mmu_topup_memory_cache()
359 while (mc->nobjs < ARRAY_SIZE(mc->objects)) { in kvm_mmu_topup_memory_cache()
362 return mc->nobjs >= min ? 0 : -ENOMEM; in kvm_mmu_topup_memory_cache()
363 mc->objects[mc->nobjs++] = obj; in kvm_mmu_topup_memory_cache()
370 return mc->nobjs; in kvm_mmu_memory_cache_nr_free_objects()
375 while (mc->nobjs) { in kvm_mmu_free_memory_cache()
376 if (mc->kmem_cache) in kvm_mmu_free_memory_cache()
377 kmem_cache_free(mc->kmem_cache, mc->objects[--mc->nobjs]); in kvm_mmu_free_memory_cache()
379 free_page((unsigned long)mc->objects[--mc->nobjs]); in kvm_mmu_free_memory_cache()
387 if (WARN_ON(!mc->nobjs)) in kvm_mmu_memory_cache_alloc()
390 p = mc->objects[--mc->nobjs]; in kvm_mmu_memory_cache_alloc()
398 mutex_init(&vcpu->mutex); in kvm_vcpu_init()
399 vcpu->cpu = -1; in kvm_vcpu_init()
400 vcpu->kvm = kvm; in kvm_vcpu_init()
401 vcpu->vcpu_id = id; in kvm_vcpu_init()
402 vcpu->pid = NULL; in kvm_vcpu_init()
403 rcuwait_init(&vcpu->wait); in kvm_vcpu_init()
406 vcpu->pre_pcpu = -1; in kvm_vcpu_init()
407 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); in kvm_vcpu_init()
411 vcpu->preempted = false; in kvm_vcpu_init()
412 vcpu->ready = false; in kvm_vcpu_init()
413 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); in kvm_vcpu_init()
422 * the vcpu->pid pointer, and at destruction time all file descriptors in kvm_vcpu_destroy()
425 put_pid(rcu_dereference_protected(vcpu->pid, 1)); in kvm_vcpu_destroy()
427 free_page((unsigned long)vcpu->run); in kvm_vcpu_destroy()
445 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_invalidate_range()
447 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_invalidate_range()
458 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_change_pte()
459 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_change_pte()
460 kvm->mmu_notifier_seq++; in kvm_mmu_notifier_change_pte()
465 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_change_pte()
466 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_change_pte()
475 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_invalidate_range_start()
476 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_start()
479 * spte can be established without taking the mmu_lock and in kvm_mmu_notifier_invalidate_range_start()
482 kvm->mmu_notifier_count++; in kvm_mmu_notifier_invalidate_range_start()
483 need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end, in kvm_mmu_notifier_invalidate_range_start()
484 range->flags); in kvm_mmu_notifier_invalidate_range_start()
485 need_tlb_flush |= kvm->tlbs_dirty; in kvm_mmu_notifier_invalidate_range_start()
490 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_start()
491 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_invalidate_range_start()
501 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_end()
507 kvm->mmu_notifier_seq++; in kvm_mmu_notifier_invalidate_range_end()
514 kvm->mmu_notifier_count--; in kvm_mmu_notifier_invalidate_range_end()
515 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_end()
517 BUG_ON(kvm->mmu_notifier_count < 0); in kvm_mmu_notifier_invalidate_range_end()
528 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_clear_flush_young()
529 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_flush_young()
535 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_flush_young()
536 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_clear_flush_young()
549 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_clear_young()
550 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_young()
553 * affect performance on pre-Haswell Intel EPT, where there is in kvm_mmu_notifier_clear_young()
565 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_young()
566 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_clear_young()
578 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_test_young()
579 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_test_young()
581 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_test_young()
582 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_test_young()
593 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_release()
595 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_release()
611 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; in kvm_init_mmu_notifier()
612 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); in kvm_init_mmu_notifier()
634 slots->id_to_index[i] = -1; in kvm_alloc_memslots()
641 if (!memslot->dirty_bitmap) in kvm_destroy_dirty_bitmap()
644 kvfree(memslot->dirty_bitmap); in kvm_destroy_dirty_bitmap()
645 memslot->dirty_bitmap = NULL; in kvm_destroy_dirty_bitmap()
654 slot->flags = 0; in kvm_free_memslot()
655 slot->npages = 0; in kvm_free_memslot()
675 if (!kvm->debugfs_dentry) in kvm_destroy_vm_debugfs()
678 debugfs_remove_recursive(kvm->debugfs_dentry); in kvm_destroy_vm_debugfs()
680 if (kvm->debugfs_stat_data) { in kvm_destroy_vm_debugfs()
682 kfree(kvm->debugfs_stat_data[i]); in kvm_destroy_vm_debugfs()
683 kfree(kvm->debugfs_stat_data); in kvm_destroy_vm_debugfs()
696 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd); in kvm_create_vm_debugfs()
697 kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir); in kvm_create_vm_debugfs()
699 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, in kvm_create_vm_debugfs()
700 sizeof(*kvm->debugfs_stat_data), in kvm_create_vm_debugfs()
702 if (!kvm->debugfs_stat_data) in kvm_create_vm_debugfs()
703 return -ENOMEM; in kvm_create_vm_debugfs()
705 for (p = debugfs_entries; p->name; p++) { in kvm_create_vm_debugfs()
708 return -ENOMEM; in kvm_create_vm_debugfs()
710 stat_data->kvm = kvm; in kvm_create_vm_debugfs()
711 stat_data->dbgfs_item = p; in kvm_create_vm_debugfs()
712 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; in kvm_create_vm_debugfs()
713 debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p), in kvm_create_vm_debugfs()
714 kvm->debugfs_dentry, stat_data, in kvm_create_vm_debugfs()
740 int r = -ENOMEM; in kvm_create_vm()
744 return ERR_PTR(-ENOMEM); in kvm_create_vm()
746 spin_lock_init(&kvm->mmu_lock); in kvm_create_vm()
747 mmgrab(current->mm); in kvm_create_vm()
748 kvm->mm = current->mm; in kvm_create_vm()
750 mutex_init(&kvm->lock); in kvm_create_vm()
751 mutex_init(&kvm->irq_lock); in kvm_create_vm()
752 mutex_init(&kvm->slots_lock); in kvm_create_vm()
753 INIT_LIST_HEAD(&kvm->devices); in kvm_create_vm()
757 if (init_srcu_struct(&kvm->srcu)) in kvm_create_vm()
759 if (init_srcu_struct(&kvm->irq_srcu)) in kvm_create_vm()
762 refcount_set(&kvm->users_count, 1); in kvm_create_vm()
769 slots->generation = i; in kvm_create_vm()
770 rcu_assign_pointer(kvm->memslots[i], slots); in kvm_create_vm()
774 rcu_assign_pointer(kvm->buses[i], in kvm_create_vm()
776 if (!kvm->buses[i]) in kvm_create_vm()
780 kvm->max_halt_poll_ns = halt_poll_ns; in kvm_create_vm()
791 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); in kvm_create_vm()
803 list_add(&kvm->vm_list, &vm_list); in kvm_create_vm()
812 if (kvm->mmu_notifier.ops) in kvm_create_vm()
813 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm); in kvm_create_vm()
820 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count)); in kvm_create_vm()
825 cleanup_srcu_struct(&kvm->irq_srcu); in kvm_create_vm()
827 cleanup_srcu_struct(&kvm->srcu); in kvm_create_vm()
830 mmdrop(current->mm); in kvm_create_vm()
839 * We do not need to take the kvm->lock here, because nobody else in kvm_destroy_devices()
843 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { in kvm_destroy_devices()
844 list_del(&dev->vm_node); in kvm_destroy_devices()
845 dev->ops->destroy(dev); in kvm_destroy_devices()
852 struct mm_struct *mm = kvm->mm; in kvm_destroy_vm()
858 list_del(&kvm->vm_list); in kvm_destroy_vm()
868 kvm->buses[i] = NULL; in kvm_destroy_vm()
872 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); in kvm_destroy_vm()
880 cleanup_srcu_struct(&kvm->irq_srcu); in kvm_destroy_vm()
881 cleanup_srcu_struct(&kvm->srcu); in kvm_destroy_vm()
890 refcount_inc(&kvm->users_count); in kvm_get_kvm()
896 if (refcount_dec_and_test(&kvm->users_count)) in kvm_put_kvm()
903 * with a user-visible file descriptor, e.g. a vcpu or device, if installation
904 * of the new file descriptor fails and the reference cannot be transferred to
910 WARN_ON(refcount_dec_and_test(&kvm->users_count)); in kvm_put_kvm_no_destroy()
916 struct kvm *kvm = filp->private_data; in kvm_vm_release()
932 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT); in kvm_alloc_dirty_bitmap()
933 if (!memslot->dirty_bitmap) in kvm_alloc_dirty_bitmap()
934 return -ENOMEM; in kvm_alloc_dirty_bitmap()
946 struct kvm_memory_slot *mslots = slots->memslots; in kvm_memslot_delete()
949 if (WARN_ON(slots->id_to_index[memslot->id] == -1)) in kvm_memslot_delete()
952 slots->used_slots--; in kvm_memslot_delete()
954 if (atomic_read(&slots->lru_slot) >= slots->used_slots) in kvm_memslot_delete()
955 atomic_set(&slots->lru_slot, 0); in kvm_memslot_delete()
957 for (i = slots->id_to_index[memslot->id]; i < slots->used_slots; i++) { in kvm_memslot_delete()
959 slots->id_to_index[mslots[i].id] = i; in kvm_memslot_delete()
962 slots->id_to_index[memslot->id] = -1; in kvm_memslot_delete()
971 return slots->used_slots++; in kvm_memslot_insert_back()
984 struct kvm_memory_slot *mslots = slots->memslots; in kvm_memslot_move_backward()
987 if (WARN_ON_ONCE(slots->id_to_index[memslot->id] == -1) || in kvm_memslot_move_backward()
988 WARN_ON_ONCE(!slots->used_slots)) in kvm_memslot_move_backward()
989 return -1; in kvm_memslot_move_backward()
996 for (i = slots->id_to_index[memslot->id]; i < slots->used_slots - 1; i++) { in kvm_memslot_move_backward()
997 if (memslot->base_gfn > mslots[i + 1].base_gfn) in kvm_memslot_move_backward()
1000 WARN_ON_ONCE(memslot->base_gfn == mslots[i + 1].base_gfn); in kvm_memslot_move_backward()
1004 slots->id_to_index[mslots[i].id] = i; in kvm_memslot_move_backward()
1020 struct kvm_memory_slot *mslots = slots->memslots; in kvm_memslot_move_forward()
1023 for (i = start; i > 0; i--) { in kvm_memslot_move_forward()
1024 if (memslot->base_gfn < mslots[i - 1].base_gfn) in kvm_memslot_move_forward()
1027 WARN_ON_ONCE(memslot->base_gfn == mslots[i - 1].base_gfn); in kvm_memslot_move_forward()
1030 mslots[i] = mslots[i - 1]; in kvm_memslot_move_forward()
1031 slots->id_to_index[mslots[i].id] = i; in kvm_memslot_move_forward()
1037 * Re-sort memslots based on their GFN to account for an added, deleted, or
1050 * - When deleting a memslot, the deleted memslot simply needs to be moved to
1053 * - When creating a memslot, the algorithm "inserts" the new memslot at the
1056 * - When moving a memslot, the algorithm first moves the updated memslot
1062 * Note, slots are sorted from highest->lowest instead of lowest->highest for
1064 * GFN=0, thus sorting from highest->lowest naturally sorted invalid memslots
1068 * The other historical motiviation for highest->lowest was to improve the
1096 slots->memslots[i] = *memslot; in update_memslots()
1097 slots->id_to_index[memslot->id] = i; in update_memslots()
1109 if (mem->flags & ~valid_flags) in check_memory_region_flags()
1110 return -EINVAL; in check_memory_region_flags()
1119 u64 gen = old_memslots->generation; in install_new_memslots()
1122 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; in install_new_memslots()
1124 rcu_assign_pointer(kvm->memslots[as_id], slots); in install_new_memslots()
1125 synchronize_srcu_expedited(&kvm->srcu); in install_new_memslots()
1129 * update in-progress flag and incrementing the generation based on in install_new_memslots()
1133 gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; in install_new_memslots()
1146 slots->generation = gen; in install_new_memslots()
1163 (sizeof(struct kvm_memory_slot) * old->used_slots); in kvm_dup_memslots()
1189 return -ENOMEM; in kvm_set_memslot()
1196 slot = id_to_memslot(slots, old->id); in kvm_set_memslot()
1197 slot->flags |= KVM_MEMSLOT_INVALID; in kvm_set_memslot()
1200 * We can re-use the old memslots, the only difference from the in kvm_set_memslot()
1203 * old memslots if preparing the new memory region fails. in kvm_set_memslot()
1210 * validation of sp->gfn happens in: in kvm_set_memslot()
1211 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) in kvm_set_memslot()
1212 * - kvm_is_visible_gfn (mmu_check_root) in kvm_set_memslot()
1243 if (!old->npages) in kvm_delete_memslot()
1244 return -EINVAL; in kvm_delete_memslot()
1247 new.id = old->id; in kvm_delete_memslot()
1268 * Must be called holding kvm->slots_lock for write.
1283 as_id = mem->slot >> 16; in __kvm_set_memory_region()
1284 id = (u16)mem->slot; in __kvm_set_memory_region()
1287 if (mem->memory_size & (PAGE_SIZE - 1)) in __kvm_set_memory_region()
1288 return -EINVAL; in __kvm_set_memory_region()
1289 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) in __kvm_set_memory_region()
1290 return -EINVAL; in __kvm_set_memory_region()
1292 if ((mem->userspace_addr & (PAGE_SIZE - 1)) || in __kvm_set_memory_region()
1293 !access_ok((void __user *)(unsigned long)mem->userspace_addr, in __kvm_set_memory_region()
1294 mem->memory_size)) in __kvm_set_memory_region()
1295 return -EINVAL; in __kvm_set_memory_region()
1297 return -EINVAL; in __kvm_set_memory_region()
1298 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) in __kvm_set_memory_region()
1299 return -EINVAL; in __kvm_set_memory_region()
1303 * when the memslots are re-sorted by update_memslots(), and the old in __kvm_set_memory_region()
1316 if (!mem->memory_size) in __kvm_set_memory_region()
1321 new.base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; in __kvm_set_memory_region()
1322 new.npages = mem->memory_size >> PAGE_SHIFT; in __kvm_set_memory_region()
1323 new.flags = mem->flags; in __kvm_set_memory_region()
1324 new.userspace_addr = mem->userspace_addr; in __kvm_set_memory_region()
1327 return -EINVAL; in __kvm_set_memory_region()
1337 return -EINVAL; in __kvm_set_memory_region()
1354 if (tmp->id == id) in __kvm_set_memory_region()
1356 if (!((new.base_gfn + new.npages <= tmp->base_gfn) || in __kvm_set_memory_region()
1357 (new.base_gfn >= tmp->base_gfn + tmp->npages))) in __kvm_set_memory_region()
1358 return -EEXIST; in __kvm_set_memory_region()
1394 mutex_lock(&kvm->slots_lock); in kvm_set_memory_region()
1396 mutex_unlock(&kvm->slots_lock); in kvm_set_memory_region()
1404 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) in kvm_vm_ioctl_set_memory_region()
1405 return -EINVAL; in kvm_vm_ioctl_set_memory_region()
1412 * kvm_get_dirty_log - get a snapshot of dirty pages
1429 as_id = log->slot >> 16; in kvm_get_dirty_log()
1430 id = (u16)log->slot; in kvm_get_dirty_log()
1432 return -EINVAL; in kvm_get_dirty_log()
1436 if (!(*memslot) || !(*memslot)->dirty_bitmap) in kvm_get_dirty_log()
1437 return -ENOENT; in kvm_get_dirty_log()
1444 any = (*memslot)->dirty_bitmap[i]; in kvm_get_dirty_log()
1446 if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n)) in kvm_get_dirty_log()
1447 return -EFAULT; in kvm_get_dirty_log()
1457 * kvm_get_dirty_log_protect - get a snapshot of dirty pages
1487 as_id = log->slot >> 16; in kvm_get_dirty_log_protect()
1488 id = (u16)log->slot; in kvm_get_dirty_log_protect()
1490 return -EINVAL; in kvm_get_dirty_log_protect()
1494 if (!memslot || !memslot->dirty_bitmap) in kvm_get_dirty_log_protect()
1495 return -ENOENT; in kvm_get_dirty_log_protect()
1497 dirty_bitmap = memslot->dirty_bitmap; in kvm_get_dirty_log_protect()
1503 if (kvm->manual_dirty_log_protect) { in kvm_get_dirty_log_protect()
1517 spin_lock(&kvm->mmu_lock); in kvm_get_dirty_log_protect()
1533 spin_unlock(&kvm->mmu_lock); in kvm_get_dirty_log_protect()
1539 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) in kvm_get_dirty_log_protect()
1540 return -EFAULT; in kvm_get_dirty_log_protect()
1546 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
1550 * Steps 1-4 below provide general overview of dirty page logging. See
1553 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
1569 mutex_lock(&kvm->slots_lock); in kvm_vm_ioctl_get_dirty_log()
1573 mutex_unlock(&kvm->slots_lock); in kvm_vm_ioctl_get_dirty_log()
1578 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
1595 as_id = log->slot >> 16; in kvm_clear_dirty_log_protect()
1596 id = (u16)log->slot; in kvm_clear_dirty_log_protect()
1598 return -EINVAL; in kvm_clear_dirty_log_protect()
1600 if (log->first_page & 63) in kvm_clear_dirty_log_protect()
1601 return -EINVAL; in kvm_clear_dirty_log_protect()
1605 if (!memslot || !memslot->dirty_bitmap) in kvm_clear_dirty_log_protect()
1606 return -ENOENT; in kvm_clear_dirty_log_protect()
1608 dirty_bitmap = memslot->dirty_bitmap; in kvm_clear_dirty_log_protect()
1610 n = ALIGN(log->num_pages, BITS_PER_LONG) / 8; in kvm_clear_dirty_log_protect()
1612 if (log->first_page > memslot->npages || in kvm_clear_dirty_log_protect()
1613 log->num_pages > memslot->npages - log->first_page || in kvm_clear_dirty_log_protect()
1614 (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63))) in kvm_clear_dirty_log_protect()
1615 return -EINVAL; in kvm_clear_dirty_log_protect()
1621 if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n)) in kvm_clear_dirty_log_protect()
1622 return -EFAULT; in kvm_clear_dirty_log_protect()
1624 spin_lock(&kvm->mmu_lock); in kvm_clear_dirty_log_protect()
1625 for (offset = log->first_page, i = offset / BITS_PER_LONG, in kvm_clear_dirty_log_protect()
1626 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--; in kvm_clear_dirty_log_protect()
1639 * a problem if userspace sets them in log->dirty_bitmap. in kvm_clear_dirty_log_protect()
1647 spin_unlock(&kvm->mmu_lock); in kvm_clear_dirty_log_protect()
1660 mutex_lock(&kvm->slots_lock); in kvm_vm_ioctl_clear_dirty_log()
1664 mutex_unlock(&kvm->slots_lock); in kvm_vm_ioctl_clear_dirty_log()
1708 mmap_read_lock(current->mm); in kvm_host_page_size()
1709 vma = find_vma(current->mm, addr); in kvm_host_page_size()
1716 mmap_read_unlock(current->mm); in kvm_host_page_size()
1723 return slot->flags & KVM_MEM_READONLY; in memslot_is_readonly()
1729 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) in __gfn_to_hva_many()
1736 *nr_pages = slot->npages - (gfn - slot->base_gfn); in __gfn_to_hva_many()
1804 return rc == -EHWPOISON; in check_user_page_hwpoison()
1838 * 1 indicates success, -errno is returned if error is detected.
1877 if (unlikely(!(vma->vm_flags & VM_READ))) in vma_is_valid()
1880 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) in vma_is_valid()
1897 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does in hva_to_pfn_remapped()
1901 r = fixup_user_fault(current->mm, addr, in hva_to_pfn_remapped()
1905 return -EAGAIN; in hva_to_pfn_remapped()
1969 mmap_read_lock(current->mm); in hva_to_pfn()
1970 if (npages == -EHWPOISON || in hva_to_pfn()
1977 vma = find_vma_intersection(current->mm, addr, addr + 1); in hva_to_pfn()
1981 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { in hva_to_pfn()
1983 if (r == -EAGAIN) in hva_to_pfn()
1993 mmap_read_unlock(current->mm); in hva_to_pfn()
2072 return -1; in gfn_to_page_many_atomic()
2110 cache->pfn = cache->gfn = 0; in kvm_release_pfn()
2121 kvm_release_pfn(cache->pfn, cache->dirty, cache); in kvm_cache_gfn_to_pfn()
2123 cache->pfn = gfn_to_pfn_memslot(slot, gfn); in kvm_cache_gfn_to_pfn()
2124 cache->gfn = gfn; in kvm_cache_gfn_to_pfn()
2125 cache->dirty = false; in kvm_cache_gfn_to_pfn()
2126 cache->generation = gen; in kvm_cache_gfn_to_pfn()
2138 u64 gen = slots->generation; in __kvm_map_gfn()
2141 return -EINVAL; in __kvm_map_gfn()
2144 if (!cache->pfn || cache->gfn != gfn || in __kvm_map_gfn()
2145 cache->generation != gen) { in __kvm_map_gfn()
2147 return -EAGAIN; in __kvm_map_gfn()
2150 pfn = cache->pfn; in __kvm_map_gfn()
2153 return -EAGAIN; in __kvm_map_gfn()
2157 return -EINVAL; in __kvm_map_gfn()
2169 return -EINVAL; in __kvm_map_gfn()
2174 return -EFAULT; in __kvm_map_gfn()
2176 map->page = page; in __kvm_map_gfn()
2177 map->hva = hva; in __kvm_map_gfn()
2178 map->pfn = pfn; in __kvm_map_gfn()
2179 map->gfn = gfn; in __kvm_map_gfn()
2187 return __kvm_map_gfn(kvm_memslots(vcpu->kvm), gfn, map, in kvm_map_gfn()
2207 if (!map->hva) in __kvm_unmap_gfn()
2210 if (map->page != KVM_UNMAPPED_PAGE) { in __kvm_unmap_gfn()
2212 kunmap_atomic(map->hva); in __kvm_unmap_gfn()
2214 kunmap(map->page); in __kvm_unmap_gfn()
2218 memunmap(map->hva); in __kvm_unmap_gfn()
2224 mark_page_dirty_in_slot(memslot, map->gfn); in __kvm_unmap_gfn()
2227 cache->dirty |= dirty; in __kvm_unmap_gfn()
2229 kvm_release_pfn(map->pfn, dirty, NULL); in __kvm_unmap_gfn()
2231 map->hva = NULL; in __kvm_unmap_gfn()
2232 map->page = NULL; in __kvm_unmap_gfn()
2238 __kvm_unmap_gfn(gfn_to_memslot(vcpu->kvm, map->gfn), map, in kvm_unmap_gfn()
2246 __kvm_unmap_gfn(kvm_vcpu_gfn_to_memslot(vcpu, map->gfn), map, NULL, in kvm_vcpu_unmap()
2314 if (len > PAGE_SIZE - offset) in next_segment()
2315 return PAGE_SIZE - offset; in next_segment()
2328 return -EFAULT; in __kvm_read_guest_page()
2331 return -EFAULT; in __kvm_read_guest_page()
2365 len -= seg; in kvm_read_guest()
2385 len -= seg; in kvm_vcpu_read_guest()
2401 return -EFAULT; in __kvm_read_guest_atomic()
2406 return -EFAULT; in __kvm_read_guest_atomic()
2429 return -EFAULT; in __kvm_write_guest_page()
2432 return -EFAULT; in __kvm_write_guest_page()
2468 len -= seg; in kvm_write_guest()
2489 len -= seg; in kvm_vcpu_write_guest()
2503 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; in __kvm_gfn_to_hva_cache_init()
2504 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; in __kvm_gfn_to_hva_cache_init()
2507 /* Update ghc->generation before performing any error checks. */ in __kvm_gfn_to_hva_cache_init()
2508 ghc->generation = slots->generation; in __kvm_gfn_to_hva_cache_init()
2511 ghc->hva = KVM_HVA_ERR_BAD; in __kvm_gfn_to_hva_cache_init()
2512 return -EINVAL; in __kvm_gfn_to_hva_cache_init()
2520 ghc->memslot = __gfn_to_memslot(slots, start_gfn); in __kvm_gfn_to_hva_cache_init()
2521 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, in __kvm_gfn_to_hva_cache_init()
2523 if (kvm_is_error_hva(ghc->hva)) in __kvm_gfn_to_hva_cache_init()
2524 return -EFAULT; in __kvm_gfn_to_hva_cache_init()
2529 ghc->hva += offset; in __kvm_gfn_to_hva_cache_init()
2531 ghc->memslot = NULL; in __kvm_gfn_to_hva_cache_init()
2533 ghc->gpa = gpa; in __kvm_gfn_to_hva_cache_init()
2534 ghc->len = len; in __kvm_gfn_to_hva_cache_init()
2552 gpa_t gpa = ghc->gpa + offset; in kvm_write_guest_offset_cached()
2554 BUG_ON(len + offset > ghc->len); in kvm_write_guest_offset_cached()
2556 if (slots->generation != ghc->generation) { in kvm_write_guest_offset_cached()
2557 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len)) in kvm_write_guest_offset_cached()
2558 return -EFAULT; in kvm_write_guest_offset_cached()
2561 if (kvm_is_error_hva(ghc->hva)) in kvm_write_guest_offset_cached()
2562 return -EFAULT; in kvm_write_guest_offset_cached()
2564 if (unlikely(!ghc->memslot)) in kvm_write_guest_offset_cached()
2567 r = __copy_to_user((void __user *)ghc->hva + offset, data, len); in kvm_write_guest_offset_cached()
2569 return -EFAULT; in kvm_write_guest_offset_cached()
2570 mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT); in kvm_write_guest_offset_cached()
2589 gpa_t gpa = ghc->gpa + offset; in kvm_read_guest_offset_cached()
2591 BUG_ON(len + offset > ghc->len); in kvm_read_guest_offset_cached()
2593 if (slots->generation != ghc->generation) { in kvm_read_guest_offset_cached()
2594 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len)) in kvm_read_guest_offset_cached()
2595 return -EFAULT; in kvm_read_guest_offset_cached()
2598 if (kvm_is_error_hva(ghc->hva)) in kvm_read_guest_offset_cached()
2599 return -EFAULT; in kvm_read_guest_offset_cached()
2601 if (unlikely(!ghc->memslot)) in kvm_read_guest_offset_cached()
2604 r = __copy_from_user(data, (void __user *)ghc->hva + offset, len); in kvm_read_guest_offset_cached()
2606 return -EFAULT; in kvm_read_guest_offset_cached()
2639 len -= seg; in kvm_clear_guest()
2648 if (memslot && memslot->dirty_bitmap) { in mark_page_dirty_in_slot()
2649 unsigned long rel_gfn = gfn - memslot->base_gfn; in mark_page_dirty_in_slot()
2651 set_bit_le(rel_gfn, memslot->dirty_bitmap); in mark_page_dirty_in_slot()
2676 if (!vcpu->sigset_active) in kvm_sigset_activate()
2680 * This does a lockless modification of ->real_blocked, which is fine in kvm_sigset_activate()
2681 * because, only current can change ->real_blocked and all readers of in kvm_sigset_activate()
2682 * ->real_blocked don't care as long ->real_blocked is always a subset in kvm_sigset_activate()
2683 * of ->blocked. in kvm_sigset_activate()
2685 sigprocmask(SIG_SETMASK, &vcpu->sigset, ¤t->real_blocked); in kvm_sigset_activate()
2690 if (!vcpu->sigset_active) in kvm_sigset_deactivate()
2693 sigprocmask(SIG_SETMASK, ¤t->real_blocked, NULL); in kvm_sigset_deactivate()
2694 sigemptyset(¤t->real_blocked); in kvm_sigset_deactivate()
2701 old = val = vcpu->halt_poll_ns; in grow_halt_poll_ns()
2714 vcpu->halt_poll_ns = val; in grow_halt_poll_ns()
2716 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); in grow_halt_poll_ns()
2723 old = val = vcpu->halt_poll_ns; in shrink_halt_poll_ns()
2730 vcpu->halt_poll_ns = val; in shrink_halt_poll_ns()
2731 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); in shrink_halt_poll_ns()
2736 int ret = -EINTR; in kvm_vcpu_check_block()
2737 int idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_vcpu_check_block()
2750 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_vcpu_check_block()
2758 vcpu->stat.halt_poll_fail_ns += poll_ns; in update_halt_poll_stats()
2760 vcpu->stat.halt_poll_success_ns += poll_ns; in update_halt_poll_stats()
2764 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2775 if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) { in kvm_vcpu_block()
2776 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); in kvm_vcpu_block()
2778 ++vcpu->stat.halt_attempted_poll; in kvm_vcpu_block()
2785 ++vcpu->stat.halt_successful_poll; in kvm_vcpu_block()
2787 ++vcpu->stat.halt_poll_invalid; in kvm_vcpu_block()
2794 prepare_to_rcuwait(&vcpu->wait); in kvm_vcpu_block()
2804 finish_rcuwait(&vcpu->wait); in kvm_vcpu_block()
2808 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); in kvm_vcpu_block()
2816 } else if (vcpu->kvm->max_halt_poll_ns) { in kvm_vcpu_block()
2817 if (block_ns <= vcpu->halt_poll_ns) in kvm_vcpu_block()
2820 else if (vcpu->halt_poll_ns && in kvm_vcpu_block()
2821 block_ns > vcpu->kvm->max_halt_poll_ns) in kvm_vcpu_block()
2824 else if (vcpu->halt_poll_ns < vcpu->kvm->max_halt_poll_ns && in kvm_vcpu_block()
2825 block_ns < vcpu->kvm->max_halt_poll_ns) in kvm_vcpu_block()
2828 vcpu->halt_poll_ns = 0; in kvm_vcpu_block()
2843 WRITE_ONCE(vcpu->ready, true); in kvm_vcpu_wake_up()
2844 ++vcpu->stat.halt_wakeup; in kvm_vcpu_wake_up()
2859 int cpu = vcpu->cpu; in kvm_vcpu_kick()
2880 pid = rcu_dereference(target->pid); in kvm_vcpu_yield_to()
2897 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2901 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2906 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2907 * to preempted lock-holder could result in wrong VCPU selection and CPU
2908 * burning. Giving priority for a potential lock-holder increases lock
2911 * Since algorithm is based on heuristics, accessing another VCPU data without
2920 eligible = !vcpu->spin_loop.in_spin_loop || in kvm_vcpu_eligible_for_directed_yield()
2921 vcpu->spin_loop.dy_eligible; in kvm_vcpu_eligible_for_directed_yield()
2923 if (vcpu->spin_loop.in_spin_loop) in kvm_vcpu_eligible_for_directed_yield()
2924 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); in kvm_vcpu_eligible_for_directed_yield()
2948 if (!list_empty_careful(&vcpu->async_pf.done)) in vcpu_dy_runnable()
2957 struct kvm *kvm = me->kvm; in kvm_vcpu_on_spin()
2959 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; in kvm_vcpu_on_spin()
2971 * We approximate round-robin by starting at the last boosted VCPU. in kvm_vcpu_on_spin()
2980 if (!READ_ONCE(vcpu->ready)) in kvm_vcpu_on_spin()
2984 if (rcuwait_active(&vcpu->wait) && in kvm_vcpu_on_spin()
2987 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode && in kvm_vcpu_on_spin()
2995 kvm->last_boosted_vcpu = i; in kvm_vcpu_on_spin()
2998 try--; in kvm_vcpu_on_spin()
3013 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data; in kvm_vcpu_fault()
3016 if (vmf->pgoff == 0) in kvm_vcpu_fault()
3017 page = virt_to_page(vcpu->run); in kvm_vcpu_fault()
3019 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) in kvm_vcpu_fault()
3020 page = virt_to_page(vcpu->arch.pio_data); in kvm_vcpu_fault()
3023 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) in kvm_vcpu_fault()
3024 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); in kvm_vcpu_fault()
3029 vmf->page = page; in kvm_vcpu_fault()
3039 vma->vm_ops = &kvm_vcpu_vm_ops; in kvm_vcpu_mmap()
3045 struct kvm_vcpu *vcpu = filp->private_data; in kvm_vcpu_release()
3047 kvm_put_kvm(vcpu->kvm); in kvm_vcpu_release()
3066 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id); in create_vcpu_fd()
3079 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); in kvm_create_vcpu_debugfs()
3081 vcpu->kvm->debugfs_dentry); in kvm_create_vcpu_debugfs()
3097 return -EINVAL; in kvm_vm_ioctl_create_vcpu()
3099 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3100 if (kvm->created_vcpus == KVM_MAX_VCPUS) { in kvm_vm_ioctl_create_vcpu()
3101 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3102 return -EINVAL; in kvm_vm_ioctl_create_vcpu()
3105 kvm->created_vcpus++; in kvm_vm_ioctl_create_vcpu()
3106 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3114 r = -ENOMEM; in kvm_vm_ioctl_create_vcpu()
3121 r = -ENOMEM; in kvm_vm_ioctl_create_vcpu()
3124 vcpu->run = page_address(page); in kvm_vm_ioctl_create_vcpu()
3132 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3134 r = -EEXIST; in kvm_vm_ioctl_create_vcpu()
3138 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); in kvm_vm_ioctl_create_vcpu()
3139 BUG_ON(kvm->vcpus[vcpu->vcpu_idx]); in kvm_vm_ioctl_create_vcpu()
3149 kvm->vcpus[vcpu->vcpu_idx] = vcpu; in kvm_vm_ioctl_create_vcpu()
3152 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus in kvm_vm_ioctl_create_vcpu()
3153 * before kvm->online_vcpu's incremented value. in kvm_vm_ioctl_create_vcpu()
3156 atomic_inc(&kvm->online_vcpus); in kvm_vm_ioctl_create_vcpu()
3158 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3164 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3167 free_page((unsigned long)vcpu->run); in kvm_vm_ioctl_create_vcpu()
3171 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3172 kvm->created_vcpus--; in kvm_vm_ioctl_create_vcpu()
3173 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3181 vcpu->sigset_active = 1; in kvm_vcpu_ioctl_set_sigmask()
3182 vcpu->sigset = *sigset; in kvm_vcpu_ioctl_set_sigmask()
3184 vcpu->sigset_active = 0; in kvm_vcpu_ioctl_set_sigmask()
3191 struct kvm_vcpu *vcpu = filp->private_data; in kvm_vcpu_ioctl()
3197 if (vcpu->kvm->mm != current->mm) in kvm_vcpu_ioctl()
3198 return -EIO; in kvm_vcpu_ioctl()
3201 return -EINVAL; in kvm_vcpu_ioctl()
3208 if (r != -ENOIOCTLCMD) in kvm_vcpu_ioctl()
3211 if (mutex_lock_killable(&vcpu->mutex)) in kvm_vcpu_ioctl()
3212 return -EINTR; in kvm_vcpu_ioctl()
3216 r = -EINVAL; in kvm_vcpu_ioctl()
3219 oldpid = rcu_access_pointer(vcpu->pid); in kvm_vcpu_ioctl()
3229 rcu_assign_pointer(vcpu->pid, newpid); in kvm_vcpu_ioctl()
3235 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); in kvm_vcpu_ioctl()
3241 r = -ENOMEM; in kvm_vcpu_ioctl()
3248 r = -EFAULT; in kvm_vcpu_ioctl()
3271 r = -ENOMEM; in kvm_vcpu_ioctl()
3277 r = -EFAULT; in kvm_vcpu_ioctl()
3299 r = -EFAULT; in kvm_vcpu_ioctl()
3308 r = -EFAULT; in kvm_vcpu_ioctl()
3317 r = -EFAULT; in kvm_vcpu_ioctl()
3323 r = -EFAULT; in kvm_vcpu_ioctl()
3332 r = -EFAULT; in kvm_vcpu_ioctl()
3345 r = -EFAULT; in kvm_vcpu_ioctl()
3349 r = -EINVAL; in kvm_vcpu_ioctl()
3352 r = -EFAULT; in kvm_vcpu_ioctl()
3353 if (copy_from_user(&sigset, sigmask_arg->sigset, in kvm_vcpu_ioctl()
3363 r = -ENOMEM; in kvm_vcpu_ioctl()
3369 r = -EFAULT; in kvm_vcpu_ioctl()
3389 mutex_unlock(&vcpu->mutex); in kvm_vcpu_ioctl()
3399 struct kvm_vcpu *vcpu = filp->private_data; in kvm_vcpu_compat_ioctl()
3403 if (vcpu->kvm->mm != current->mm) in kvm_vcpu_compat_ioctl()
3404 return -EIO; in kvm_vcpu_compat_ioctl()
3413 r = -EFAULT; in kvm_vcpu_compat_ioctl()
3417 r = -EINVAL; in kvm_vcpu_compat_ioctl()
3420 r = -EFAULT; in kvm_vcpu_compat_ioctl()
3422 (compat_sigset_t __user *)sigmask_arg->sigset)) in kvm_vcpu_compat_ioctl()
3440 struct kvm_device *dev = filp->private_data; in kvm_device_mmap()
3442 if (dev->ops->mmap) in kvm_device_mmap()
3443 return dev->ops->mmap(dev, vma); in kvm_device_mmap()
3445 return -ENODEV; in kvm_device_mmap()
3456 return -EPERM; in kvm_device_ioctl_attr()
3459 return -EFAULT; in kvm_device_ioctl_attr()
3467 struct kvm_device *dev = filp->private_data; in kvm_device_ioctl()
3469 if (dev->kvm->mm != current->mm) in kvm_device_ioctl()
3470 return -EIO; in kvm_device_ioctl()
3474 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); in kvm_device_ioctl()
3476 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); in kvm_device_ioctl()
3478 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); in kvm_device_ioctl()
3480 if (dev->ops->ioctl) in kvm_device_ioctl()
3481 return dev->ops->ioctl(dev, ioctl, arg); in kvm_device_ioctl()
3483 return -ENOTTY; in kvm_device_ioctl()
3489 struct kvm_device *dev = filp->private_data; in kvm_device_release()
3490 struct kvm *kvm = dev->kvm; in kvm_device_release()
3492 if (dev->ops->release) { in kvm_device_release()
3493 mutex_lock(&kvm->lock); in kvm_device_release()
3494 list_del(&dev->vm_node); in kvm_device_release()
3495 dev->ops->release(dev); in kvm_device_release()
3496 mutex_unlock(&kvm->lock); in kvm_device_release()
3512 if (filp->f_op != &kvm_device_fops) in kvm_device_from_filp()
3515 return filp->private_data; in kvm_device_from_filp()
3528 return -ENOSPC; in kvm_register_device_ops()
3531 return -EEXIST; in kvm_register_device_ops()
3544 struct kvm_create_device *cd) in kvm_ioctl_create_device() argument
3548 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; in kvm_ioctl_create_device() local
3552 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) in kvm_ioctl_create_device()
3553 return -ENODEV; in kvm_ioctl_create_device()
3555 type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table)); in kvm_ioctl_create_device()
3558 return -ENODEV; in kvm_ioctl_create_device()
3560 if (test) in kvm_ioctl_create_device()
3565 return -ENOMEM; in kvm_ioctl_create_device()
3567 dev->ops = ops; in kvm_ioctl_create_device()
3568 dev->kvm = kvm; in kvm_ioctl_create_device()
3570 mutex_lock(&kvm->lock); in kvm_ioctl_create_device()
3571 ret = ops->create(dev, type); in kvm_ioctl_create_device()
3573 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
3577 list_add(&dev->vm_node, &kvm->devices); in kvm_ioctl_create_device()
3578 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
3580 if (ops->init) in kvm_ioctl_create_device()
3581 ops->init(dev); in kvm_ioctl_create_device()
3584 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); in kvm_ioctl_create_device()
3587 mutex_lock(&kvm->lock); in kvm_ioctl_create_device()
3588 list_del(&dev->vm_node); in kvm_ioctl_create_device()
3589 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
3590 ops->destroy(dev); in kvm_ioctl_create_device()
3594 cd->fd = ret; in kvm_ioctl_create_device()
3646 return -EINVAL; in kvm_vm_ioctl_enable_cap()
3652 switch (cap->cap) { in kvm_vm_ioctl_enable_cap_generic()
3657 if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE) in kvm_vm_ioctl_enable_cap_generic()
3660 if (cap->flags || (cap->args[0] & ~allowed_options)) in kvm_vm_ioctl_enable_cap_generic()
3661 return -EINVAL; in kvm_vm_ioctl_enable_cap_generic()
3662 kvm->manual_dirty_log_protect = cap->args[0]; in kvm_vm_ioctl_enable_cap_generic()
3667 if (cap->flags || cap->args[0] != (unsigned int)cap->args[0]) in kvm_vm_ioctl_enable_cap_generic()
3668 return -EINVAL; in kvm_vm_ioctl_enable_cap_generic()
3670 kvm->max_halt_poll_ns = cap->args[0]; in kvm_vm_ioctl_enable_cap_generic()
3681 struct kvm *kvm = filp->private_data; in kvm_vm_ioctl()
3685 if (kvm->mm != current->mm) in kvm_vm_ioctl()
3686 return -EIO; in kvm_vm_ioctl()
3694 r = -EFAULT; in kvm_vm_ioctl()
3703 r = -EFAULT; in kvm_vm_ioctl()
3714 r = -EFAULT; in kvm_vm_ioctl()
3724 r = -EFAULT; in kvm_vm_ioctl()
3735 r = -EFAULT; in kvm_vm_ioctl()
3744 r = -EFAULT; in kvm_vm_ioctl()
3754 r = -EFAULT; in kvm_vm_ioctl()
3763 r = -EFAULT; in kvm_vm_ioctl()
3773 r = -EFAULT; in kvm_vm_ioctl()
3785 r = -EFAULT; in kvm_vm_ioctl()
3794 r = -EFAULT; in kvm_vm_ioctl()
3810 r = -EFAULT; in kvm_vm_ioctl()
3813 r = -EINVAL; in kvm_vm_ioctl()
3822 entries = vmemdup_user(urouting->entries, in kvm_vm_ioctl()
3837 struct kvm_create_device cd; in kvm_vm_ioctl() local
3839 r = -EFAULT; in kvm_vm_ioctl()
3840 if (copy_from_user(&cd, argp, sizeof(cd))) in kvm_vm_ioctl()
3843 r = kvm_ioctl_create_device(kvm, &cd); in kvm_vm_ioctl()
3847 r = -EFAULT; in kvm_vm_ioctl()
3848 if (copy_to_user(argp, &cd, sizeof(cd))) in kvm_vm_ioctl()
3877 struct kvm *kvm = filp->private_data; in kvm_vm_compat_ioctl()
3880 if (kvm->mm != current->mm) in kvm_vm_compat_ioctl()
3881 return -EIO; in kvm_vm_compat_ioctl()
3889 return -EFAULT; in kvm_vm_compat_ioctl()
3930 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); in kvm_dev_ioctl_create_vm()
3938 * Don't call kvm_put_kvm anymore at this point; file->f_op is in kvm_dev_ioctl_create_vm()
3939 * already set, with ->release() being kvm_vm_release(). In error in kvm_dev_ioctl_create_vm()
3946 return -ENOMEM; in kvm_dev_ioctl_create_vm()
3961 long r = -EINVAL; in kvm_dev_ioctl()
3989 r = -EOPNOTSUPP; in kvm_dev_ioctl()
4061 kvm_usage_count--; in hardware_disable_all_nolock()
4086 r = -EBUSY; in hardware_enable_all()
4119 for (i = 0; i < bus->dev_count; i++) { in kvm_io_bus_destroy()
4120 struct kvm_io_device *pos = bus->range[i].dev; in kvm_io_bus_destroy()
4130 gpa_t addr1 = r1->addr; in kvm_io_bus_cmp()
4131 gpa_t addr2 = r2->addr; in kvm_io_bus_cmp()
4134 return -1; in kvm_io_bus_cmp()
4136 /* If r2->len == 0, match the exact address. If r2->len != 0, in kvm_io_bus_cmp()
4141 if (r2->len) { in kvm_io_bus_cmp()
4142 addr1 += r1->len; in kvm_io_bus_cmp()
4143 addr2 += r2->len; in kvm_io_bus_cmp()
4168 range = bsearch(&key, bus->range, bus->dev_count, in kvm_io_bus_get_first_dev()
4171 return -ENOENT; in kvm_io_bus_get_first_dev()
4173 off = range - bus->range; in kvm_io_bus_get_first_dev()
4175 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) in kvm_io_bus_get_first_dev()
4176 off--; in kvm_io_bus_get_first_dev()
4186 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); in __kvm_io_bus_write()
4188 return -EOPNOTSUPP; in __kvm_io_bus_write()
4190 while (idx < bus->dev_count && in __kvm_io_bus_write()
4191 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { in __kvm_io_bus_write()
4192 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, in __kvm_io_bus_write()
4193 range->len, val)) in __kvm_io_bus_write()
4198 return -EOPNOTSUPP; in __kvm_io_bus_write()
4201 /* kvm_io_bus_write - called under kvm->slots_lock */
4214 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_write()
4216 return -ENOMEM; in kvm_io_bus_write()
4222 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
4234 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_write_cookie()
4236 return -ENOMEM; in kvm_io_bus_write_cookie()
4239 if ((cookie >= 0) && (cookie < bus->dev_count) && in kvm_io_bus_write_cookie()
4240 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) in kvm_io_bus_write_cookie()
4241 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, in kvm_io_bus_write_cookie()
4257 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); in __kvm_io_bus_read()
4259 return -EOPNOTSUPP; in __kvm_io_bus_read()
4261 while (idx < bus->dev_count && in __kvm_io_bus_read()
4262 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { in __kvm_io_bus_read()
4263 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, in __kvm_io_bus_read()
4264 range->len, val)) in __kvm_io_bus_read()
4269 return -EOPNOTSUPP; in __kvm_io_bus_read()
4272 /* kvm_io_bus_read - called under kvm->slots_lock */
4285 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_read()
4287 return -ENOMEM; in kvm_io_bus_read()
4302 return -ENOMEM; in kvm_io_bus_register_dev()
4305 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) in kvm_io_bus_register_dev()
4306 return -ENOSPC; in kvm_io_bus_register_dev()
4308 new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1), in kvm_io_bus_register_dev()
4311 return -ENOMEM; in kvm_io_bus_register_dev()
4319 for (i = 0; i < bus->dev_count; i++) in kvm_io_bus_register_dev()
4320 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0) in kvm_io_bus_register_dev()
4324 new_bus->dev_count++; in kvm_io_bus_register_dev()
4325 new_bus->range[i] = range; in kvm_io_bus_register_dev()
4326 memcpy(new_bus->range + i + 1, bus->range + i, in kvm_io_bus_register_dev()
4327 (bus->dev_count - i) * sizeof(struct kvm_io_range)); in kvm_io_bus_register_dev()
4328 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); in kvm_io_bus_register_dev()
4329 synchronize_srcu_expedited(&kvm->srcu); in kvm_io_bus_register_dev()
4346 for (i = 0; i < bus->dev_count; i++) in kvm_io_bus_unregister_dev()
4347 if (bus->range[i].dev == dev) { in kvm_io_bus_unregister_dev()
4351 if (i == bus->dev_count) in kvm_io_bus_unregister_dev()
4354 new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1), in kvm_io_bus_unregister_dev()
4358 new_bus->dev_count--; in kvm_io_bus_unregister_dev()
4359 memcpy(new_bus->range + i, bus->range + i + 1, in kvm_io_bus_unregister_dev()
4360 flex_array_size(new_bus, range, new_bus->dev_count - i)); in kvm_io_bus_unregister_dev()
4363 for (j = 0; j < bus->dev_count; j++) { in kvm_io_bus_unregister_dev()
4366 kvm_iodevice_destructor(bus->range[j].dev); in kvm_io_bus_unregister_dev()
4370 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); in kvm_io_bus_unregister_dev()
4371 synchronize_srcu_expedited(&kvm->srcu); in kvm_io_bus_unregister_dev()
4383 srcu_idx = srcu_read_lock(&kvm->srcu); in kvm_io_bus_get_dev()
4385 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); in kvm_io_bus_get_dev()
4393 iodev = bus->range[dev_idx].dev; in kvm_io_bus_get_dev()
4396 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvm_io_bus_get_dev()
4407 inode->i_private; in kvm_debugfs_open()
4412 * directory we test against the users count. in kvm_debugfs_open()
4414 if (!refcount_inc_not_zero(&stat_data->kvm->users_count)) in kvm_debugfs_open()
4415 return -ENOENT; in kvm_debugfs_open()
4418 KVM_DBGFS_GET_MODE(stat_data->dbgfs_item) & 0222 in kvm_debugfs_open()
4421 kvm_put_kvm(stat_data->kvm); in kvm_debugfs_open()
4422 return -ENOMEM; in kvm_debugfs_open()
4431 inode->i_private; in kvm_debugfs_release()
4434 kvm_put_kvm(stat_data->kvm); in kvm_debugfs_release()
4479 int r = -EFAULT; in kvm_stat_data_get()
4482 switch (stat_data->dbgfs_item->kind) { in kvm_stat_data_get()
4484 r = kvm_get_stat_per_vm(stat_data->kvm, in kvm_stat_data_get()
4485 stat_data->dbgfs_item->offset, val); in kvm_stat_data_get()
4488 r = kvm_get_stat_per_vcpu(stat_data->kvm, in kvm_stat_data_get()
4489 stat_data->dbgfs_item->offset, val); in kvm_stat_data_get()
4498 int r = -EFAULT; in kvm_stat_data_clear()
4502 return -EINVAL; in kvm_stat_data_clear()
4504 switch (stat_data->dbgfs_item->kind) { in kvm_stat_data_clear()
4506 r = kvm_clear_stat_per_vm(stat_data->kvm, in kvm_stat_data_clear()
4507 stat_data->dbgfs_item->offset); in kvm_stat_data_clear()
4510 r = kvm_clear_stat_per_vcpu(stat_data->kvm, in kvm_stat_data_clear()
4511 stat_data->dbgfs_item->offset); in kvm_stat_data_clear()
4556 return -EINVAL; in vm_stat_clear()
4591 return -EINVAL; in vcpu_stat_clear()
4623 kvm_active_vms--; in kvm_uevent_notify_change()
4638 kvm->userspace_pid = task_pid_nr(current); in kvm_uevent_notify_change()
4642 add_uevent_var(env, "PID=%d", kvm->userspace_pid); in kvm_uevent_notify_change()
4644 if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) { in kvm_uevent_notify_change()
4648 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); in kvm_uevent_notify_change()
4655 env->envp[env->envp_idx++] = NULL; in kvm_uevent_notify_change()
4656 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp); in kvm_uevent_notify_change()
4667 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { in kvm_init_debug()
4668 debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p), in kvm_init_debug()
4669 kvm_debugfs_dir, (void *)(long)p->offset, in kvm_init_debug()
4670 stat_fops[p->kind]); in kvm_init_debug()
4706 WRITE_ONCE(vcpu->preempted, false); in kvm_sched_in()
4707 WRITE_ONCE(vcpu->ready, false); in kvm_sched_in()
4719 if (current->state == TASK_RUNNING) { in kvm_sched_out()
4720 WRITE_ONCE(vcpu->preempted, true); in kvm_sched_out()
4721 WRITE_ONCE(vcpu->ready, true); in kvm_sched_out()
4728 * kvm_get_running_vcpu - get the vcpu running on the current CPU.
4730 * We can disable preemption locally around accessing the per-CPU variable,
4733 * the per-CPU value later will give us the same value as we update the
4734 * per-CPU variable in the preempt notifier handlers.
4749 * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus.
4765 *c->ret = kvm_arch_check_processor_compat(c->opaque); in check_processor_compat()
4791 r = -ENOMEM; in kvm_init()
4823 r = -ENOMEM; in kvm_init()
4907 struct kvm *kvm = init_context->kvm; in kvm_vm_worker_thread()
4908 kvm_vm_thread_fn_t thread_fn = init_context->thread_fn; in kvm_vm_worker_thread()
4909 uintptr_t data = init_context->data; in kvm_vm_worker_thread()
4918 err = cgroup_attach_task_all(init_context->parent, current); in kvm_vm_worker_thread()
4925 set_user_nice(current, task_nice(init_context->parent)); in kvm_vm_worker_thread()
4928 init_context->err = err; in kvm_vm_worker_thread()
4929 complete(&init_context->init_done); in kvm_vm_worker_thread()
4959 "%s-%d", name, task_pid_nr(current)); in kvm_vm_create_worker_thread()