377 	ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);  in __vm_create()
449 	struct userspace_mem_region *region;  in kvm_vm_restart()  local
455 	hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {  in kvm_vm_restart()
456 		int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, &region->region);  in kvm_vm_restart()
461 			    ret, errno, region->region.slot,  in kvm_vm_restart()
462 			    region->region.flags,  in kvm_vm_restart()
463 			    region->region.guest_phys_addr,  in kvm_vm_restart()
464 			    region->region.memory_size);  in kvm_vm_restart()
576 		struct userspace_mem_region *region =  in userspace_mem_region_find()  local
578 		uint64_t existing_start = region->region.guest_phys_addr;  in userspace_mem_region_find()
579 		uint64_t existing_end = region->region.guest_phys_addr  in userspace_mem_region_find()
580 			+ region->region.memory_size - 1;  in userspace_mem_region_find()
582 			return region;  in userspace_mem_region_find()
613 	struct userspace_mem_region *region;  in kvm_userspace_memory_region_find()  local
615 	region = userspace_mem_region_find(vm, start, end);  in kvm_userspace_memory_region_find()
616 	if (!region)  in kvm_userspace_memory_region_find()
619 	return &region->region;  in kvm_userspace_memory_region_find()
677 				   struct userspace_mem_region *region,  in __vm_mem_region_delete()  argument
683 		rb_erase(&region->gpa_node, &vm->regions.gpa_tree);  in __vm_mem_region_delete()
684 		rb_erase(&region->hva_node, &vm->regions.hva_tree);  in __vm_mem_region_delete()
685 		hash_del(&region->slot_node);  in __vm_mem_region_delete()
688 	region->region.memory_size = 0;  in __vm_mem_region_delete()
689 	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in __vm_mem_region_delete()
691 	sparsebit_free(&region->unused_phy_pages);  in __vm_mem_region_delete()
692 	ret = munmap(region->mmap_start, region->mmap_size);  in __vm_mem_region_delete()
694 	if (region->fd >= 0) {  in __vm_mem_region_delete()
696 		ret = munmap(region->mmap_alias, region->mmap_size);  in __vm_mem_region_delete()
698 		close(region->fd);  in __vm_mem_region_delete()
701 	free(region);  in __vm_mem_region_delete()
711 	struct userspace_mem_region *region;  in kvm_vm_free()  local
723 	hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)  in kvm_vm_free()
724 		__vm_mem_region_delete(vmp, region, false);  in kvm_vm_free()
827 					       struct userspace_mem_region *region)  in vm_userspace_mem_region_gpa_insert()  argument
836 		if (region->region.guest_phys_addr <  in vm_userspace_mem_region_gpa_insert()
837 		    cregion->region.guest_phys_addr)  in vm_userspace_mem_region_gpa_insert()
840 			TEST_ASSERT(region->region.guest_phys_addr !=  in vm_userspace_mem_region_gpa_insert()
841 				    cregion->region.guest_phys_addr,  in vm_userspace_mem_region_gpa_insert()
848 	rb_link_node(&region->gpa_node, parent, cur);  in vm_userspace_mem_region_gpa_insert()
849 	rb_insert_color(&region->gpa_node, gpa_tree);  in vm_userspace_mem_region_gpa_insert()
853 					       struct userspace_mem_region *region)  in vm_userspace_mem_region_hva_insert()  argument
862 		if (region->host_mem < cregion->host_mem)  in vm_userspace_mem_region_hva_insert()
865 			TEST_ASSERT(region->host_mem !=  in vm_userspace_mem_region_hva_insert()
873 	rb_link_node(&region->hva_node, parent, cur);  in vm_userspace_mem_region_hva_insert()
874 	rb_insert_color(&region->hva_node, hva_tree);  in vm_userspace_mem_region_hva_insert()
881 	struct kvm_userspace_memory_region region = {  in __vm_set_user_memory_region()  local
889 	return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, &region);  in __vm_set_user_memory_region()
929 	struct userspace_mem_region *region;  in vm_userspace_mem_region_add()  local
952 	region = (struct userspace_mem_region *) userspace_mem_region_find(  in vm_userspace_mem_region_add()
954 	if (region != NULL)  in vm_userspace_mem_region_add()
961 			(uint64_t) region->region.guest_phys_addr,  in vm_userspace_mem_region_add()
962 			(uint64_t) region->region.memory_size);  in vm_userspace_mem_region_add()
965 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,  in vm_userspace_mem_region_add()
967 		if (region->region.slot != slot)  in vm_userspace_mem_region_add()
975 			region->region.slot,  in vm_userspace_mem_region_add()
976 			(uint64_t) region->region.guest_phys_addr,  in vm_userspace_mem_region_add()
977 			(uint64_t) region->region.memory_size);  in vm_userspace_mem_region_add()
981 	region = calloc(1, sizeof(*region));  in vm_userspace_mem_region_add()
982 	TEST_ASSERT(region != NULL, "Insufficient Memory");  in vm_userspace_mem_region_add()
983 	region->mmap_size = npages * vm->page_size;  in vm_userspace_mem_region_add()
1005 		region->mmap_size += alignment;  in vm_userspace_mem_region_add()
1007 	region->fd = -1;  in vm_userspace_mem_region_add()
1009 		region->fd = kvm_memfd_alloc(region->mmap_size,  in vm_userspace_mem_region_add()
1012 	region->mmap_start = mmap(NULL, region->mmap_size,  in vm_userspace_mem_region_add()
1015 				  region->fd, 0);  in vm_userspace_mem_region_add()
1016 	TEST_ASSERT(region->mmap_start != MAP_FAILED,  in vm_userspace_mem_region_add()
1020 		    region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz),  in vm_userspace_mem_region_add()
1022 		    region->mmap_start, backing_src_pagesz);  in vm_userspace_mem_region_add()
1025 	region->host_mem = align_ptr_up(region->mmap_start, alignment);  in vm_userspace_mem_region_add()
1030 		ret = madvise(region->host_mem, npages * vm->page_size,  in vm_userspace_mem_region_add()
1033 			    region->host_mem, npages * vm->page_size,  in vm_userspace_mem_region_add()
1037 	region->backing_src_type = src_type;  in vm_userspace_mem_region_add()
1038 	region->unused_phy_pages = sparsebit_alloc();  in vm_userspace_mem_region_add()
1039 	sparsebit_set_num(region->unused_phy_pages,  in vm_userspace_mem_region_add()
1041 	region->region.slot = slot;  in vm_userspace_mem_region_add()
1042 	region->region.flags = flags;  in vm_userspace_mem_region_add()
1043 	region->region.guest_phys_addr = guest_paddr;  in vm_userspace_mem_region_add()
1044 	region->region.memory_size = npages * vm->page_size;  in vm_userspace_mem_region_add()
1045 	region->region.userspace_addr = (uintptr_t) region->host_mem;  in vm_userspace_mem_region_add()
1046 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in vm_userspace_mem_region_add()
1052 		guest_paddr, (uint64_t) region->region.memory_size);  in vm_userspace_mem_region_add()
1055 	vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);  in vm_userspace_mem_region_add()
1056 	vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);  in vm_userspace_mem_region_add()
1057 	hash_add(vm->regions.slot_hash, &region->slot_node, slot);  in vm_userspace_mem_region_add()
1060 	if (region->fd >= 0) {  in vm_userspace_mem_region_add()
1061 		region->mmap_alias = mmap(NULL, region->mmap_size,  in vm_userspace_mem_region_add()
1064 					  region->fd, 0);  in vm_userspace_mem_region_add()
1065 		TEST_ASSERT(region->mmap_alias != MAP_FAILED,  in vm_userspace_mem_region_add()
1069 		region->host_alias = align_ptr_up(region->mmap_alias, alignment);  in vm_userspace_mem_region_add()
1091 	struct userspace_mem_region *region;  in memslot2region()  local
1093 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,  in memslot2region()
1095 		if (region->region.slot == memslot)  in memslot2region()
1096 			return region;  in memslot2region()
1123 	struct userspace_mem_region *region;  in vm_mem_region_set_flags()  local
1125 	region = memslot2region(vm, slot);  in vm_mem_region_set_flags()
1127 	region->region.flags = flags;  in vm_mem_region_set_flags()
1129 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in vm_mem_region_set_flags()
1152 	struct userspace_mem_region *region;  in vm_mem_region_move()  local
1155 	region = memslot2region(vm, slot);  in vm_mem_region_move()
1157 	region->region.guest_phys_addr = new_gpa;  in vm_mem_region_move()
1159 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in vm_mem_region_move()
1480 	struct userspace_mem_region *region;  in addr_gpa2hva()  local
1482 	region = userspace_mem_region_find(vm, gpa, gpa);  in addr_gpa2hva()
1483 	if (!region) {  in addr_gpa2hva()
1488 	return (void *)((uintptr_t)region->host_mem  in addr_gpa2hva()
1489 		+ (gpa - region->region.guest_phys_addr));  in addr_gpa2hva()
1514 		struct userspace_mem_region *region =  in addr_hva2gpa()  local
1517 		if (hva >= region->host_mem) {  in addr_hva2gpa()
1518 			if (hva <= (region->host_mem  in addr_hva2gpa()
1519 				+ region->region.memory_size - 1))  in addr_hva2gpa()
1521 					region->region.guest_phys_addr  in addr_hva2gpa()
1522 					+ (hva - (uintptr_t)region->host_mem));  in addr_hva2gpa()
1554 	struct userspace_mem_region *region;  in addr_gpa2alias()  local
1557 	region = userspace_mem_region_find(vm, gpa, gpa);  in addr_gpa2alias()
1558 	if (!region)  in addr_gpa2alias()
1561 	if (!region->host_alias)  in addr_gpa2alias()
1564 	offset = gpa - region->region.guest_phys_addr;  in addr_gpa2alias()
1565 	return (void *) ((uintptr_t) region->host_alias + offset);  in addr_gpa2alias()
1813 	struct userspace_mem_region *region;  in vm_dump()  local
1820 	hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {  in vm_dump()
1823 			(uint64_t) region->region.guest_phys_addr,  in vm_dump()
1824 			(uint64_t) region->region.memory_size,  in vm_dump()
1825 			region->host_mem);  in vm_dump()
1827 		sparsebit_dump(stream, region->unused_phy_pages, 0);  in vm_dump()
1943 	struct userspace_mem_region *region;  in vm_phy_pages_alloc()  local
1953 	region = memslot2region(vm, memslot);  in vm_phy_pages_alloc()
1958 			if (!sparsebit_is_set(region->unused_phy_pages, pg)) {  in vm_phy_pages_alloc()
1959 				base = pg = sparsebit_next_set(region->unused_phy_pages, pg);  in vm_phy_pages_alloc()
1975 		sparsebit_clear(region->unused_phy_pages, pg);  in vm_phy_pages_alloc()