53 	struct kvm_pgtable		*pgt;  member
91 static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)  in __kvm_pgd_page_idx()  argument
93 	u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */  in __kvm_pgd_page_idx()
94 	u64 mask = BIT(pgt->ia_bits) - 1;  in __kvm_pgd_page_idx()
101 	return __kvm_pgd_page_idx(data->pgt, data->addr);  in kvm_pgd_page_idx()
106 	struct kvm_pgtable pgt = {  in kvm_pgd_pages()  local
111 	return __kvm_pgd_page_idx(&pgt, -1ULL) + 1;  in kvm_pgd_pages()
216 	childp = kvm_pte_follow(pte, data->pgt->mm_ops);  in __kvm_pgtable_visit()
257 	struct kvm_pgtable *pgt = data->pgt;  in _kvm_pgtable_walk()  local
258 	u64 limit = BIT(pgt->ia_bits);  in _kvm_pgtable_walk()
263 	if (!pgt->pgd)  in _kvm_pgtable_walk()
267 		kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE];  in _kvm_pgtable_walk()
269 		ret = __kvm_pgtable_walk(data, ptep, pgt->start_level);  in _kvm_pgtable_walk()
277 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_walk()  argument
281 		.pgt	= pgt,  in kvm_pgtable_walk()
306 int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,  in kvm_pgtable_get_leaf()  argument
317 	ret = kvm_pgtable_walk(pgt, ALIGN_DOWN(addr, PAGE_SIZE),  in kvm_pgtable_get_leaf()
430 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,  in kvm_pgtable_hyp_map()  argument
436 		.mm_ops	= pgt->mm_ops,  in kvm_pgtable_hyp_map()
448 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_hyp_map()
499 u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_hyp_unmap()  argument
502 		.mm_ops	= pgt->mm_ops,  in kvm_pgtable_hyp_unmap()
510 	if (!pgt->mm_ops->page_count)  in kvm_pgtable_hyp_unmap()
513 	kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_hyp_unmap()
517 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,  in kvm_pgtable_hyp_init()  argument
522 	pgt->pgd = (kvm_pte_t *)mm_ops->zalloc_page(NULL);  in kvm_pgtable_hyp_init()
523 	if (!pgt->pgd)  in kvm_pgtable_hyp_init()
526 	pgt->ia_bits		= va_bits;  in kvm_pgtable_hyp_init()
527 	pgt->start_level	= KVM_PGTABLE_MAX_LEVELS - levels;  in kvm_pgtable_hyp_init()
528 	pgt->mm_ops		= mm_ops;  in kvm_pgtable_hyp_init()
529 	pgt->mmu		= NULL;  in kvm_pgtable_hyp_init()
530 	pgt->force_pte_cb	= NULL;  in kvm_pgtable_hyp_init()
552 void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)  in kvm_pgtable_hyp_destroy()  argument
557 		.arg	= pgt->mm_ops,  in kvm_pgtable_hyp_destroy()
560 	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));  in kvm_pgtable_hyp_destroy()
561 	pgt->mm_ops->put_page(pgt->pgd);  in kvm_pgtable_hyp_destroy()
562 	pgt->pgd = NULL;  in kvm_pgtable_hyp_destroy()
613 static bool stage2_has_fwb(struct kvm_pgtable *pgt)  in stage2_has_fwb()  argument
618 	return !(pgt->flags & KVM_PGTABLE_S2_NOFWB);  in stage2_has_fwb()
621 #define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt))  argument
623 static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,  in stage2_set_prot_attr()  argument
627 	kvm_pte_t attr = device ? KVM_S2_MEMATTR(pgt, DEVICE_nGnRE) :  in stage2_set_prot_attr()
628 			    KVM_S2_MEMATTR(pgt, NORMAL);  in stage2_set_prot_attr()
700 static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)  in stage2_pte_cacheable()  argument
703 	return memattr == KVM_S2_MEMATTR(pgt, NORMAL);  in stage2_pte_cacheable()
726 	struct kvm_pgtable *pgt = data->mmu->pgt;  in stage2_map_walker_try_leaf()  local
751 	if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new))  in stage2_map_walker_try_leaf()
893 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_stage2_map()  argument
900 		.mmu		= pgt->mmu,  in kvm_pgtable_stage2_map()
902 		.mm_ops		= pgt->mm_ops,  in kvm_pgtable_stage2_map()
903 		.force_pte	= pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),  in kvm_pgtable_stage2_map()
913 	if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys)))  in kvm_pgtable_stage2_map()
916 	ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);  in kvm_pgtable_stage2_map()
920 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_map()
925 int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_stage2_set_owner()  argument
931 		.mmu		= pgt->mmu,  in kvm_pgtable_stage2_set_owner()
933 		.mm_ops		= pgt->mm_ops,  in kvm_pgtable_stage2_set_owner()
948 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_set_owner()
956 	struct kvm_pgtable *pgt = arg;  in stage2_unmap_walker()  local
957 	struct kvm_s2_mmu *mmu = pgt->mmu;  in stage2_unmap_walker()
958 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in stage2_unmap_walker()
975 	} else if (stage2_pte_cacheable(pgt, pte)) {  in stage2_unmap_walker()
976 		need_flush = !stage2_has_fwb(pgt);  in stage2_unmap_walker()
996 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_unmap()  argument
1000 		.arg	= pgt,  in kvm_pgtable_stage2_unmap()
1004 	return kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_unmap()
1051 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,  in stage2_update_leaf_attrs()  argument
1061 		.mm_ops		= pgt->mm_ops,  in stage2_update_leaf_attrs()
1069 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in stage2_update_leaf_attrs()
1081 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_wrprotect()  argument
1083 	return stage2_update_leaf_attrs(pgt, addr, size, 0,  in kvm_pgtable_stage2_wrprotect()
1088 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_mkyoung()  argument
1091 	stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,  in kvm_pgtable_stage2_mkyoung()
1097 kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_mkold()  argument
1100 	stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF,  in kvm_pgtable_stage2_mkold()
1111 bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_is_young()  argument
1114 	stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL);  in kvm_pgtable_stage2_is_young()
1118 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,  in kvm_pgtable_stage2_relax_perms()  argument
1137 	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level);  in kvm_pgtable_stage2_relax_perms()
1139 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level);  in kvm_pgtable_stage2_relax_perms()
1147 	struct kvm_pgtable *pgt = arg;  in stage2_flush_walker()  local
1148 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in stage2_flush_walker()
1151 	if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte))  in stage2_flush_walker()
1160 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_flush()  argument
1165 		.arg	= pgt,  in kvm_pgtable_stage2_flush()
1168 	if (stage2_has_fwb(pgt))  in kvm_pgtable_stage2_flush()
1171 	return kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_flush()
1175 int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,  in __kvm_pgtable_stage2_init()  argument
1187 	pgt->pgd = mm_ops->zalloc_pages_exact(pgd_sz);  in __kvm_pgtable_stage2_init()
1188 	if (!pgt->pgd)  in __kvm_pgtable_stage2_init()
1191 	pgt->ia_bits		= ia_bits;  in __kvm_pgtable_stage2_init()
1192 	pgt->start_level	= start_level;  in __kvm_pgtable_stage2_init()
1193 	pgt->mm_ops		= mm_ops;  in __kvm_pgtable_stage2_init()
1194 	pgt->mmu		= mmu;  in __kvm_pgtable_stage2_init()
1195 	pgt->flags		= flags;  in __kvm_pgtable_stage2_init()
1196 	pgt->force_pte_cb	= force_pte_cb;  in __kvm_pgtable_stage2_init()
1221 void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)  in kvm_pgtable_stage2_destroy()  argument
1228 		.arg	= pgt->mm_ops,  in kvm_pgtable_stage2_destroy()
1231 	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));  in kvm_pgtable_stage2_destroy()
1232 	pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;  in kvm_pgtable_stage2_destroy()
1233 	pgt->mm_ops->free_pages_exact(pgt->pgd, pgd_sz);  in kvm_pgtable_stage2_destroy()
1234 	pgt->pgd = NULL;  in kvm_pgtable_stage2_destroy()