321 	 * maximum page table size for a memory region will be when the  in vm_nr_pages_required()
414 	struct userspace_mem_region *region;  in kvm_vm_restart()  local
420 	hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {  in kvm_vm_restart()
421 		int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, &region->region);  in kvm_vm_restart()
426 			    ret, errno, region->region.slot,  in kvm_vm_restart()
427 			    region->region.flags,  in kvm_vm_restart()
428 			    region->region.guest_phys_addr,  in kvm_vm_restart()
429 			    region->region.memory_size);  in kvm_vm_restart()
447  * Userspace Memory Region Find
457  *   Pointer to overlapping region, NULL if no such region.
459  * Searches for a region with any physical memory that overlaps with
463  * region exists.
471 		struct userspace_mem_region *region =  in userspace_mem_region_find()  local
473 		uint64_t existing_start = region->region.guest_phys_addr;  in userspace_mem_region_find()
474 		uint64_t existing_end = region->region.guest_phys_addr  in userspace_mem_region_find()
475 			+ region->region.memory_size - 1;  in userspace_mem_region_find()
477 			return region;  in userspace_mem_region_find()
489  * KVM Userspace Memory Region Find
499  *   Pointer to overlapping region, NULL if no such region.
508 	struct userspace_mem_region *region;  in kvm_userspace_memory_region_find()  local
510 	region = userspace_mem_region_find(vm, start, end);  in kvm_userspace_memory_region_find()
511 	if (!region)  in kvm_userspace_memory_region_find()
514 	return &region->region;  in kvm_userspace_memory_region_find()
572 				   struct userspace_mem_region *region,  in __vm_mem_region_delete()  argument
578 		rb_erase(&region->gpa_node, &vm->regions.gpa_tree);  in __vm_mem_region_delete()
579 		rb_erase(&region->hva_node, &vm->regions.hva_tree);  in __vm_mem_region_delete()
580 		hash_del(&region->slot_node);  in __vm_mem_region_delete()
583 	region->region.memory_size = 0;  in __vm_mem_region_delete()
584 	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in __vm_mem_region_delete()
586 	sparsebit_free(&region->unused_phy_pages);  in __vm_mem_region_delete()
587 	ret = munmap(region->mmap_start, region->mmap_size);  in __vm_mem_region_delete()
590 	free(region);  in __vm_mem_region_delete()
600 	struct userspace_mem_region *region;  in kvm_vm_free()  local
612 	hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)  in kvm_vm_free()
613 		__vm_mem_region_delete(vmp, region, false);  in kvm_vm_free()
716 					       struct userspace_mem_region *region)  in vm_userspace_mem_region_gpa_insert()  argument
725 		if (region->region.guest_phys_addr <  in vm_userspace_mem_region_gpa_insert()
726 		    cregion->region.guest_phys_addr)  in vm_userspace_mem_region_gpa_insert()
729 			TEST_ASSERT(region->region.guest_phys_addr !=  in vm_userspace_mem_region_gpa_insert()
730 				    cregion->region.guest_phys_addr,  in vm_userspace_mem_region_gpa_insert()
731 				    "Duplicate GPA in region tree");  in vm_userspace_mem_region_gpa_insert()
737 	rb_link_node(&region->gpa_node, parent, cur);  in vm_userspace_mem_region_gpa_insert()
738 	rb_insert_color(&region->gpa_node, gpa_tree);  in vm_userspace_mem_region_gpa_insert()
742 					       struct userspace_mem_region *region)  in vm_userspace_mem_region_hva_insert()  argument
751 		if (region->host_mem < cregion->host_mem)  in vm_userspace_mem_region_hva_insert()
754 			TEST_ASSERT(region->host_mem !=  in vm_userspace_mem_region_hva_insert()
756 				    "Duplicate HVA in region tree");  in vm_userspace_mem_region_hva_insert()
762 	rb_link_node(&region->hva_node, parent, cur);  in vm_userspace_mem_region_hva_insert()
763 	rb_insert_color(&region->hva_node, hva_tree);  in vm_userspace_mem_region_hva_insert()
770 	struct kvm_userspace_memory_region region = {  in __vm_set_user_memory_region()  local
778 	return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, &region);  in __vm_set_user_memory_region()
791  * VM Userspace Memory Region Add
795  *   src_type - Storage source for this region.
798  *   slot - KVM region slot
800  *   flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
808  * given by guest_paddr.  The region is created with a KVM region slot
810  * region is created with the flags given by flags.
818 	struct userspace_mem_region *region;  in vm_userspace_mem_region_add()  local
838 	 * Confirm a mem region with an overlapping address doesn't  in vm_userspace_mem_region_add()
841 	region = (struct userspace_mem_region *) userspace_mem_region_find(  in vm_userspace_mem_region_add()
843 	if (region != NULL)  in vm_userspace_mem_region_add()
850 			(uint64_t) region->region.guest_phys_addr,  in vm_userspace_mem_region_add()
851 			(uint64_t) region->region.memory_size);  in vm_userspace_mem_region_add()
853 	/* Confirm no region with the requested slot already exists. */  in vm_userspace_mem_region_add()
854 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,  in vm_userspace_mem_region_add()
856 		if (region->region.slot != slot)  in vm_userspace_mem_region_add()
859 		TEST_FAIL("A mem region with the requested slot "  in vm_userspace_mem_region_add()
864 			region->region.slot,  in vm_userspace_mem_region_add()
865 			(uint64_t) region->region.guest_phys_addr,  in vm_userspace_mem_region_add()
866 			(uint64_t) region->region.memory_size);  in vm_userspace_mem_region_add()
869 	/* Allocate and initialize new mem region structure. */  in vm_userspace_mem_region_add()
870 	region = calloc(1, sizeof(*region));  in vm_userspace_mem_region_add()
871 	TEST_ASSERT(region != NULL, "Insufficient Memory");  in vm_userspace_mem_region_add()
872 	region->mmap_size = npages * vm->page_size;  in vm_userspace_mem_region_add()
894 		region->mmap_size += alignment;  in vm_userspace_mem_region_add()
896 	region->fd = -1;  in vm_userspace_mem_region_add()
898 		region->fd = kvm_memfd_alloc(region->mmap_size,  in vm_userspace_mem_region_add()
901 	region->mmap_start = mmap(NULL, region->mmap_size,  in vm_userspace_mem_region_add()
904 				  region->fd, 0);  in vm_userspace_mem_region_add()
905 	TEST_ASSERT(region->mmap_start != MAP_FAILED,  in vm_userspace_mem_region_add()
909 		    region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz),  in vm_userspace_mem_region_add()
911 		    region->mmap_start, backing_src_pagesz);  in vm_userspace_mem_region_add()
914 	region->host_mem = align_ptr_up(region->mmap_start, alignment);  in vm_userspace_mem_region_add()
919 		ret = madvise(region->host_mem, npages * vm->page_size,  in vm_userspace_mem_region_add()
922 			    region->host_mem, npages * vm->page_size,  in vm_userspace_mem_region_add()
926 	region->unused_phy_pages = sparsebit_alloc();  in vm_userspace_mem_region_add()
927 	sparsebit_set_num(region->unused_phy_pages,  in vm_userspace_mem_region_add()
929 	region->region.slot = slot;  in vm_userspace_mem_region_add()
930 	region->region.flags = flags;  in vm_userspace_mem_region_add()
931 	region->region.guest_phys_addr = guest_paddr;  in vm_userspace_mem_region_add()
932 	region->region.memory_size = npages * vm->page_size;  in vm_userspace_mem_region_add()
933 	region->region.userspace_addr = (uintptr_t) region->host_mem;  in vm_userspace_mem_region_add()
934 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in vm_userspace_mem_region_add()
940 		guest_paddr, (uint64_t) region->region.memory_size);  in vm_userspace_mem_region_add()
943 	vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);  in vm_userspace_mem_region_add()
944 	vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);  in vm_userspace_mem_region_add()
945 	hash_add(vm->regions.slot_hash, &region->slot_node, slot);  in vm_userspace_mem_region_add()
948 	if (region->fd >= 0) {  in vm_userspace_mem_region_add()
949 		region->mmap_alias = mmap(NULL, region->mmap_size,  in vm_userspace_mem_region_add()
952 					  region->fd, 0);  in vm_userspace_mem_region_add()
953 		TEST_ASSERT(region->mmap_alias != MAP_FAILED,  in vm_userspace_mem_region_add()
957 		region->host_alias = align_ptr_up(region->mmap_alias, alignment);  in vm_userspace_mem_region_add()
962  * Memslot to region
971  *   Pointer to memory region structure that describe memory region
973  *   on error (e.g. currently no memory region using memslot as a KVM
979 	struct userspace_mem_region *region;  in memslot2region()  local
981 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,  in memslot2region()
983 		if (region->region.slot == memslot)  in memslot2region()
984 			return region;  in memslot2region()
986 	fprintf(stderr, "No mem region with the requested slot found,\n"  in memslot2region()
990 	TEST_FAIL("Mem region not found");  in memslot2region()
995  * VM Memory Region Flags Set
1005  * Sets the flags of the memory region specified by the value of slot,
1011 	struct userspace_mem_region *region;  in vm_mem_region_set_flags()  local
1013 	region = memslot2region(vm, slot);  in vm_mem_region_set_flags()
1015 	region->region.flags = flags;  in vm_mem_region_set_flags()
1017 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in vm_mem_region_set_flags()
1025  * VM Memory Region Move
1029  *   slot - Slot of the memory region to move
1036  * Change the gpa of a memory region.
1040 	struct userspace_mem_region *region;  in vm_mem_region_move()  local
1043 	region = memslot2region(vm, slot);  in vm_mem_region_move()
1045 	region->region.guest_phys_addr = new_gpa;  in vm_mem_region_move()
1047 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);  in vm_mem_region_move()
1055  * VM Memory Region Delete
1059  *   slot - Slot of the memory region to delete
1065  * Delete a memory region.
1348  * Locates the memory region containing the VM physical address given
1351  * A TEST_ASSERT failure occurs if no region containing gpa exists.
1355 	struct userspace_mem_region *region;  in addr_gpa2hva()  local
1357 	region = userspace_mem_region_find(vm, gpa, gpa);  in addr_gpa2hva()
1358 	if (!region) {  in addr_gpa2hva()
1363 	return (void *)((uintptr_t)region->host_mem  in addr_gpa2hva()
1364 		+ (gpa - region->region.guest_phys_addr));  in addr_gpa2hva()
1379  * Locates the memory region containing the host virtual address given
1382  * region containing hva exists.
1389 		struct userspace_mem_region *region =  in addr_hva2gpa()  local
1392 		if (hva >= region->host_mem) {  in addr_hva2gpa()
1393 			if (hva <= (region->host_mem  in addr_hva2gpa()
1394 				+ region->region.memory_size - 1))  in addr_hva2gpa()
1396 					region->region.guest_phys_addr  in addr_hva2gpa()
1397 					+ (hva - (uintptr_t)region->host_mem));  in addr_hva2gpa()
1429 	struct userspace_mem_region *region;  in addr_gpa2alias()  local
1432 	region = userspace_mem_region_find(vm, gpa, gpa);  in addr_gpa2alias()
1433 	if (!region)  in addr_gpa2alias()
1436 	if (!region->host_alias)  in addr_gpa2alias()
1439 	offset = gpa - region->region.guest_phys_addr;  in addr_gpa2alias()
1440 	return (void *) ((uintptr_t) region->host_alias + offset);  in addr_gpa2alias()
1688 	struct userspace_mem_region *region;  in vm_dump()  local
1695 	hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {  in vm_dump()
1698 			(uint64_t) region->region.guest_phys_addr,  in vm_dump()
1699 			(uint64_t) region->region.memory_size,  in vm_dump()
1700 			region->host_mem);  in vm_dump()
1702 		sparsebit_dump(stream, region->unused_phy_pages, 0);  in vm_dump()
1787  *   memslot - Memory region to allocate page from
1802 	struct userspace_mem_region *region;  in vm_phy_pages_alloc()  local
1812 	region = memslot2region(vm, memslot);  in vm_phy_pages_alloc()
1817 			if (!sparsebit_is_set(region->unused_phy_pages, pg)) {  in vm_phy_pages_alloc()
1818 				base = pg = sparsebit_next_set(region->unused_phy_pages, pg);  in vm_phy_pages_alloc()
1834 		sparsebit_clear(region->unused_phy_pages, pg);  in vm_phy_pages_alloc()