140 static void kvm_clear_pte(kvm_pte_t *ptep)  in kvm_clear_pte()  argument
142 	WRITE_ONCE(*ptep, 0);  in kvm_clear_pte()
145 static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp,  in kvm_set_table_pte()  argument
148 	kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));  in kvm_set_table_pte()
154 	smp_store_release(ptep, pte);  in kvm_set_table_pte()
176 				  u32 level, kvm_pte_t *ptep,  in kvm_pgtable_visitor_cb()  argument
180 	return walker->cb(addr, data->end, level, ptep, flag, walker->arg);  in kvm_pgtable_visitor_cb()
187 				      kvm_pte_t *ptep, u32 level)  in __kvm_pgtable_visit()  argument
191 	kvm_pte_t *childp, pte = *ptep;  in __kvm_pgtable_visit()
196 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
201 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
203 		pte = *ptep;  in __kvm_pgtable_visit()
222 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
240 		kvm_pte_t *ptep = &pgtable[idx];  in __kvm_pgtable_walk()  local
245 		ret = __kvm_pgtable_visit(data, ptep, level);  in __kvm_pgtable_walk()
267 		kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE];  in _kvm_pgtable_walk()  local
269 		ret = __kvm_pgtable_walk(data, ptep, pgt->start_level);  in _kvm_pgtable_walk()
295 static int leaf_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in leaf_walker()  argument
300 	data->pte   = *ptep;  in leaf_walker()
307 			 kvm_pte_t *ptep, u32 *level)  in kvm_pgtable_get_leaf()  argument
320 		if (ptep)  in kvm_pgtable_get_leaf()
321 			*ptep  = data.pte;  in kvm_pgtable_get_leaf()
335 static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)  in hyp_set_prot_attr()  argument
361 	*ptep = attr;  in hyp_set_prot_attr()
402 				    kvm_pte_t *ptep, struct hyp_map_data *data)  in hyp_map_walker_try_leaf()  argument
404 	kvm_pte_t new, old = *ptep;  in hyp_map_walker_try_leaf()
412 		smp_store_release(ptep, new);  in hyp_map_walker_try_leaf()
418 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_map_walker()  argument
425 	if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))  in hyp_map_walker()
435 	kvm_set_table_pte(ptep, childp, mm_ops);  in hyp_map_walker()
481 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_free_walker()  argument
486 	mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops));  in hyp_free_walker()
562 				kvm_pte_t *ptep)  in stage2_set_prot_attr()  argument
583 	*ptep = attr;  in stage2_set_prot_attr()
623 static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr,  in stage2_put_pte()  argument
630 	if (kvm_pte_valid(*ptep)) {  in stage2_put_pte()
631 		kvm_clear_pte(ptep);  in stage2_put_pte()
635 	mm_ops->put_page(ptep);  in stage2_put_pte()
659 				      kvm_pte_t *ptep,  in stage2_map_walker_try_leaf()  argument
662 	kvm_pte_t new, old = *ptep;  in stage2_map_walker_try_leaf()
685 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walker_try_leaf()
696 	smp_store_release(ptep, new);  in stage2_map_walker_try_leaf()
698 		mm_ops->get_page(ptep);  in stage2_map_walker_try_leaf()
705 				     kvm_pte_t *ptep,  in stage2_map_walk_table_pre()  argument
714 	data->childp = kvm_pte_follow(*ptep, data->mm_ops);  in stage2_map_walk_table_pre()
715 	kvm_clear_pte(ptep);  in stage2_map_walk_table_pre()
723 	data->anchor = ptep;  in stage2_map_walk_table_pre()
727 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walk_leaf()  argument
731 	kvm_pte_t *childp, pte = *ptep;  in stage2_map_walk_leaf()
736 			mm_ops->put_page(ptep);  in stage2_map_walk_leaf()
741 	ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);  in stage2_map_walk_leaf()
761 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walk_leaf()
763 	kvm_set_table_pte(ptep, childp, mm_ops);  in stage2_map_walk_leaf()
764 	mm_ops->get_page(ptep);  in stage2_map_walk_leaf()
770 				      kvm_pte_t *ptep,  in stage2_map_walk_table_post()  argument
780 	if (data->anchor == ptep) {  in stage2_map_walk_table_post()
784 		ret = stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walk_table_post()
786 		childp = kvm_pte_follow(*ptep, mm_ops);  in stage2_map_walk_table_post()
790 	mm_ops->put_page(ptep);  in stage2_map_walk_table_post()
814 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walker()  argument
821 		return stage2_map_walk_table_pre(addr, end, level, ptep, data);  in stage2_map_walker()
823 		return stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walker()
825 		return stage2_map_walk_table_post(addr, end, level, ptep, data);  in stage2_map_walker()
890 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_unmap_walker()  argument
897 	kvm_pte_t pte = *ptep, *childp = NULL;  in stage2_unmap_walker()
902 			kvm_clear_pte(ptep);  in stage2_unmap_walker()
903 			mm_ops->put_page(ptep);  in stage2_unmap_walker()
922 	stage2_put_pte(ptep, mmu, addr, level, mm_ops);  in stage2_unmap_walker()
957 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_attr_walker()  argument
961 	kvm_pte_t pte = *ptep;  in stage2_attr_walker()
984 		    stage2_pte_executable(pte) && !stage2_pte_executable(*ptep))  in stage2_attr_walker()
987 		WRITE_ONCE(*ptep, pte);  in stage2_attr_walker()
1085 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_flush_walker()  argument
1091 	kvm_pte_t pte = *ptep;  in stage2_flush_walker()
1147 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_free_walker()  argument
1152 	kvm_pte_t pte = *ptep;  in stage2_free_walker()
1157 	mm_ops->put_page(ptep);  in stage2_free_walker()