127 static bool kvm_pte_table(kvm_pte_t pte, u32 level)  in kvm_pte_table()
138 static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)  in kvm_pte_follow()
143 static void kvm_clear_pte(kvm_pte_t *ptep)  in kvm_clear_pte()
148 static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)  in kvm_init_table_pte()
150 	kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));  in kvm_init_table_pte()
157 static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)  in kvm_init_valid_leaf_pte()
159 	kvm_pte_t pte = kvm_phys_to_pte(pa);  in kvm_init_valid_leaf_pte()
170 static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id)  in kvm_init_invalid_leaf_owner()
213 	kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref);  in __kvm_pgtable_visit()
342 	kvm_pte_t	pte;
358 			 kvm_pte_t *ptep, u32 *level)  in kvm_pgtable_get_leaf()
382 	kvm_pte_t			attr;
385 static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)  in hyp_set_prot_attr()
389 	kvm_pte_t attr = FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_ATTRIDX, mtype);  in hyp_set_prot_attr()
419 enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte)  in kvm_pgtable_hyp_pte_prot()
443 	kvm_pte_t new;  in hyp_map_walker_try_leaf()
463 	kvm_pte_t *childp, new;  in hyp_map_walker()
473 	childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);  in hyp_map_walker()
510 	kvm_pte_t *childp = NULL;  in hyp_unmap_walker()
611 	kvm_pte_t			attr;
614 	kvm_pte_t			*anchor;
615 	kvm_pte_t			*childp;
696 				kvm_pte_t *ptep)  in stage2_set_prot_attr()
699 	kvm_pte_t attr = device ? KVM_S2_MEMATTR(pgt, DEVICE_nGnRE) :  in stage2_set_prot_attr()
722 enum kvm_pgtable_prot kvm_pgtable_stage2_pte_prot(kvm_pte_t pte)  in kvm_pgtable_stage2_pte_prot()
739 static bool stage2_pte_needs_update(kvm_pte_t old, kvm_pte_t new)  in stage2_pte_needs_update()
747 static bool stage2_pte_is_counted(kvm_pte_t pte)  in stage2_pte_is_counted()
757 static bool stage2_pte_is_locked(kvm_pte_t pte)  in stage2_pte_is_locked()
762 static bool stage2_try_set_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)  in stage2_try_set_pte()
822 static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)  in stage2_make_pte()
869 static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)  in stage2_pte_cacheable()
875 static bool stage2_pte_executable(kvm_pte_t pte)  in stage2_pte_executable()
914 	kvm_pte_t new;  in stage2_map_walker_try_leaf()
959 	kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);  in stage2_map_walk_table_pre()
977 	kvm_pte_t *childp, new;  in stage2_map_walk_leaf()
1096 	kvm_pte_t *childp = NULL;  in stage2_unmap_walker()
1151 	kvm_pte_t			attr_set;
1152 	kvm_pte_t			attr_clr;
1153 	kvm_pte_t			pte;
1160 	kvm_pte_t pte = ctx->old;  in stage2_attr_walker()
1195 				    u64 size, kvm_pte_t attr_set,  in stage2_update_leaf_attrs()
1196 				    kvm_pte_t attr_clr, kvm_pte_t *orig_pte,  in stage2_update_leaf_attrs()
1200 	kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI;  in stage2_update_leaf_attrs()
1230 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_mkyoung()
1232 	kvm_pte_t pte = 0;  in kvm_pgtable_stage2_mkyoung()
1253 	kvm_pte_t new = ctx->old & ~KVM_PTE_LEAF_ATTR_LO_S2_AF;  in stage2_age_walker()
1300 	kvm_pte_t set = 0, clr = 0;  in kvm_pgtable_stage2_relax_perms()
1351 kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,  in kvm_pgtable_stage2_create_unlinked()
1380 	kvm_pte_t *pgtable;  in kvm_pgtable_stage2_create_unlinked()
1432 	kvm_pte_t pte = ctx->old, new, *childp;  in stage2_split_walker()