16 static uint64_t page_align(struct kvm_vm *vm, uint64_t v)  in page_align()  argument
18 	return (v + vm->page_size) & ~(vm->page_size - 1);  in page_align()
21 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry)  in pte_addr()  argument
27 static uint64_t ptrs_per_pte(struct kvm_vm *vm)  in ptrs_per_pte()  argument
46 static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level)  in pte_index()  argument
50 	TEST_ASSERT(level < vm->pgtable_levels,  in pte_index()
56 void virt_arch_pgd_alloc(struct kvm_vm *vm)  in virt_arch_pgd_alloc()  argument
58 	if (!vm->pgd_created) {  in virt_arch_pgd_alloc()
59 		vm_paddr_t paddr = vm_phy_pages_alloc(vm,  in virt_arch_pgd_alloc()
60 			page_align(vm, ptrs_per_pte(vm) * 8) / vm->page_size,  in virt_arch_pgd_alloc()
62 		vm->pgd = paddr;  in virt_arch_pgd_alloc()
63 		vm->pgd_created = true;  in virt_arch_pgd_alloc()
67 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)  in virt_arch_pg_map()  argument
70 	int level = vm->pgtable_levels - 1;  in virt_arch_pg_map()
72 	TEST_ASSERT((vaddr % vm->page_size) == 0,  in virt_arch_pg_map()
74 		"  vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);  in virt_arch_pg_map()
75 	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,  in virt_arch_pg_map()
76 		(vaddr >> vm->page_shift)),  in virt_arch_pg_map()
78 	TEST_ASSERT((paddr % vm->page_size) == 0,  in virt_arch_pg_map()
80 		"  paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);  in virt_arch_pg_map()
81 	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,  in virt_arch_pg_map()
83 		"  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",  in virt_arch_pg_map()
84 		paddr, vm->max_gfn, vm->page_size);  in virt_arch_pg_map()
86 	ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, vaddr, level) * 8;  in virt_arch_pg_map()
88 		next_ppn = vm_alloc_page_table(vm) >> PGTBL_PAGE_SIZE_SHIFT;  in virt_arch_pg_map()
95 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) +  in virt_arch_pg_map()
96 		       pte_index(vm, vaddr, level) * 8;  in virt_arch_pg_map()
98 			next_ppn = vm_alloc_page_table(vm) >>  in virt_arch_pg_map()
111 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)  in addr_arch_gva2gpa()  argument
114 	int level = vm->pgtable_levels - 1;  in addr_arch_gva2gpa()
116 	if (!vm->pgd_created)  in addr_arch_gva2gpa()
119 	ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, gva, level) * 8;  in addr_arch_gva2gpa()
125 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) +  in addr_arch_gva2gpa()
126 		       pte_index(vm, gva, level) * 8;  in addr_arch_gva2gpa()
132 	return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));  in addr_arch_gva2gpa()
135 	TEST_FAIL("No mapping for vm virtual address gva: 0x%lx level: %d",  in addr_arch_gva2gpa()
140 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent,  in pte_dump()  argument
150 	for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) {  in pte_dump()
151 		ptep = addr_gpa2hva(vm, pte);  in pte_dump()
156 		pte_dump(stream, vm, indent + 1,  in pte_dump()
157 			 pte_addr(vm, *ptep), level - 1);  in pte_dump()
162 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)  in virt_arch_dump()  argument
164 	int level = vm->pgtable_levels - 1;  in virt_arch_dump()
167 	if (!vm->pgd_created)  in virt_arch_dump()
170 	for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pte(vm) * 8; pgd += 8) {  in virt_arch_dump()
171 		ptep = addr_gpa2hva(vm, pgd);  in virt_arch_dump()
176 		pte_dump(stream, vm, indent + 1,  in virt_arch_dump()
177 			 pte_addr(vm, *ptep), level - 1);  in virt_arch_dump()
183 	struct kvm_vm *vm = vcpu->vm;  in riscv_vcpu_mmu_setup()  local
190 	switch (vm->mode) {  in riscv_vcpu_mmu_setup()
196 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);  in riscv_vcpu_mmu_setup()
199 	satp = (vm->pgd >> PGTBL_PAGE_SIZE_SHIFT) & SATP_PPN;  in riscv_vcpu_mmu_setup()
278 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,  in vm_arch_vcpu_add()  argument
282 	size_t stack_size = vm->page_size == 4096 ?  in vm_arch_vcpu_add()
283 					DEFAULT_STACK_PGS * vm->page_size :  in vm_arch_vcpu_add()
284 					vm->page_size;  in vm_arch_vcpu_add()
285 	unsigned long stack_vaddr = vm_vaddr_alloc(vm, stack_size,  in vm_arch_vcpu_add()
291 	vcpu = __vm_vcpu_add(vm, vcpu_id);  in vm_arch_vcpu_add()