19 static uint64_t page_align(struct kvm_vm *vm, uint64_t v)  in page_align()  argument
21 	return (v + vm->page_size) & ~(vm->page_size - 1);  in page_align()
24 static uint64_t pgd_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pgd_index()  argument
26 	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;  in pgd_index()
27 	uint64_t mask = (1UL << (vm->va_bits - shift)) - 1;  in pgd_index()
32 static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pud_index()  argument
34 	unsigned int shift = 2 * (vm->page_shift - 3) + vm->page_shift;  in pud_index()
35 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;  in pud_index()
37 	TEST_ASSERT(vm->pgtable_levels == 4,  in pud_index()
38 		"Mode %d does not have 4 page table levels", vm->mode);  in pud_index()
43 static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pmd_index()  argument
45 	unsigned int shift = (vm->page_shift - 3) + vm->page_shift;  in pmd_index()
46 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;  in pmd_index()
48 	TEST_ASSERT(vm->pgtable_levels >= 3,  in pmd_index()
49 		"Mode %d does not have >= 3 page table levels", vm->mode);  in pmd_index()
54 static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pte_index()  argument
56 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;  in pte_index()
57 	return (gva >> vm->page_shift) & mask;  in pte_index()
60 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry)  in pte_addr()  argument
62 	uint64_t mask = ((1UL << (vm->va_bits - vm->page_shift)) - 1) << vm->page_shift;  in pte_addr()
66 static uint64_t ptrs_per_pgd(struct kvm_vm *vm)  in ptrs_per_pgd()  argument
68 	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;  in ptrs_per_pgd()
69 	return 1 << (vm->va_bits - shift);  in ptrs_per_pgd()
72 static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm)  in ptrs_per_pte()  argument
74 	return 1 << (vm->page_shift - 3);  in ptrs_per_pte()
77 void virt_pgd_alloc(struct kvm_vm *vm)  in virt_pgd_alloc()  argument
79 	if (!vm->pgd_created) {  in virt_pgd_alloc()
80 		vm_paddr_t paddr = vm_phy_pages_alloc(vm,  in virt_pgd_alloc()
81 			page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size,  in virt_pgd_alloc()
83 		vm->pgd = paddr;  in virt_pgd_alloc()
84 		vm->pgd_created = true;  in virt_pgd_alloc()
88 static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,  in _virt_pg_map()  argument
94 	TEST_ASSERT((vaddr % vm->page_size) == 0,  in _virt_pg_map()
96 		"  vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);  in _virt_pg_map()
97 	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,  in _virt_pg_map()
98 		(vaddr >> vm->page_shift)),  in _virt_pg_map()
100 	TEST_ASSERT((paddr % vm->page_size) == 0,  in _virt_pg_map()
102 		"  paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);  in _virt_pg_map()
103 	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,  in _virt_pg_map()
105 		"  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",  in _virt_pg_map()
106 		paddr, vm->max_gfn, vm->page_size);  in _virt_pg_map()
108 	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8;  in _virt_pg_map()
110 		*ptep = vm_alloc_page_table(vm) | 3;  in _virt_pg_map()
112 	switch (vm->pgtable_levels) {  in _virt_pg_map()
114 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8;  in _virt_pg_map()
116 			*ptep = vm_alloc_page_table(vm) | 3;  in _virt_pg_map()
119 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8;  in _virt_pg_map()
121 			*ptep = vm_alloc_page_table(vm) | 3;  in _virt_pg_map()
124 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8;  in _virt_pg_map()
134 void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)  in virt_pg_map()  argument
138 	_virt_pg_map(vm, vaddr, paddr, attr_idx);  in virt_pg_map()
141 vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)  in addr_gva2gpa()  argument
145 	if (!vm->pgd_created)  in addr_gva2gpa()
148 	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, gva) * 8;  in addr_gva2gpa()
152 	switch (vm->pgtable_levels) {  in addr_gva2gpa()
154 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8;  in addr_gva2gpa()
159 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8;  in addr_gva2gpa()
164 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8;  in addr_gva2gpa()
172 	return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));  in addr_gva2gpa()
175 	TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);  in addr_gva2gpa()
179 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)  in pte_dump()  argument
188 	for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) {  in pte_dump()
189 		ptep = addr_gpa2hva(vm, pte);  in pte_dump()
193 		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level + 1);  in pte_dump()
198 void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)  in virt_dump()  argument
200 	int level = 4 - (vm->pgtable_levels - 1);  in virt_dump()
203 	if (!vm->pgd_created)  in virt_dump()
206 	for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pgd(vm) * 8; pgd += 8) {  in virt_dump()
207 		ptep = addr_gpa2hva(vm, pgd);  in virt_dump()
211 		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level);  in virt_dump()
215 void aarch64_vcpu_setup(struct kvm_vm *vm, int vcpuid, struct kvm_vcpu_init *init)  in aarch64_vcpu_setup()  argument
225 		vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred);  in aarch64_vcpu_setup()
229 	vcpu_ioctl(vm, vcpuid, KVM_ARM_VCPU_INIT, init);  in aarch64_vcpu_setup()
235 	set_reg(vm, vcpuid, ARM64_SYS_REG(CPACR_EL1), 3 << 20);  in aarch64_vcpu_setup()
237 	get_reg(vm, vcpuid, ARM64_SYS_REG(SCTLR_EL1), &sctlr_el1);  in aarch64_vcpu_setup()
238 	get_reg(vm, vcpuid, ARM64_SYS_REG(TCR_EL1), &tcr_el1);  in aarch64_vcpu_setup()
240 	switch (vm->mode) {  in aarch64_vcpu_setup()
268 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);  in aarch64_vcpu_setup()
274 	tcr_el1 |= (64 - vm->va_bits) /* T0SZ */;  in aarch64_vcpu_setup()
276 	set_reg(vm, vcpuid, ARM64_SYS_REG(SCTLR_EL1), sctlr_el1);  in aarch64_vcpu_setup()
277 	set_reg(vm, vcpuid, ARM64_SYS_REG(TCR_EL1), tcr_el1);  in aarch64_vcpu_setup()
278 	set_reg(vm, vcpuid, ARM64_SYS_REG(MAIR_EL1), DEFAULT_MAIR_EL1);  in aarch64_vcpu_setup()
279 	set_reg(vm, vcpuid, ARM64_SYS_REG(TTBR0_EL1), vm->pgd);  in aarch64_vcpu_setup()
282 void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)  in vcpu_dump()  argument
286 	get_reg(vm, vcpuid, ARM64_CORE_REG(regs.pstate), &pstate);  in vcpu_dump()
287 	get_reg(vm, vcpuid, ARM64_CORE_REG(regs.pc), &pc);  in vcpu_dump()
293 void aarch64_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid,  in aarch64_vcpu_add_default()  argument
296 	size_t stack_size = vm->page_size == 4096 ?  in aarch64_vcpu_add_default()
297 					DEFAULT_STACK_PGS * vm->page_size :  in aarch64_vcpu_add_default()
298 					vm->page_size;  in aarch64_vcpu_add_default()
299 	uint64_t stack_vaddr = vm_vaddr_alloc(vm, stack_size,  in aarch64_vcpu_add_default()
302 	vm_vcpu_add(vm, vcpuid);  in aarch64_vcpu_add_default()
303 	aarch64_vcpu_setup(vm, vcpuid, init);  in aarch64_vcpu_add_default()
305 	set_reg(vm, vcpuid, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);  in aarch64_vcpu_add_default()
306 	set_reg(vm, vcpuid, ARM64_CORE_REG(regs.pc), (uint64_t)guest_code);  in aarch64_vcpu_add_default()
309 void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)  in vm_vcpu_add_default()  argument
311 	aarch64_vcpu_add_default(vm, vcpuid, NULL, guest_code);  in vm_vcpu_add_default()
314 void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...)  in vcpu_args_set()  argument
325 		set_reg(vm, vcpuid, ARM64_CORE_REG(regs.regs[i]),  in vcpu_args_set()
339 void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid)  in assert_on_unhandled_exception()  argument
343 	if (get_ucall(vm, vcpuid, &uc) != UCALL_UNHANDLED)  in assert_on_unhandled_exception()
361 void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid)  in vcpu_init_descriptor_tables()  argument
365 	set_reg(vm, vcpuid, ARM64_SYS_REG(VBAR_EL1), (uint64_t)&vectors);  in vcpu_init_descriptor_tables()
401 void vm_init_descriptor_tables(struct kvm_vm *vm)  in vm_init_descriptor_tables()  argument
403 	vm->handlers = vm_vaddr_alloc(vm, sizeof(struct handlers),  in vm_init_descriptor_tables()
404 			vm->page_size);  in vm_init_descriptor_tables()
406 	*(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;  in vm_init_descriptor_tables()
409 void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec,  in vm_install_sync_handler()  argument
412 	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);  in vm_install_sync_handler()
420 void vm_install_exception_handler(struct kvm_vm *vm, int vector,  in vm_install_exception_handler()  argument
423 	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);  in vm_install_exception_handler()