140 static void kvm_clear_pte(kvm_pte_t *ptep)  in kvm_clear_pte()  argument
142 	WRITE_ONCE(*ptep, 0);  in kvm_clear_pte()
145 static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp,  in kvm_set_table_pte()  argument
148 	kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));  in kvm_set_table_pte()
154 	smp_store_release(ptep, pte);  in kvm_set_table_pte()
176 				  u32 level, kvm_pte_t *ptep,  in kvm_pgtable_visitor_cb()  argument
180 	return walker->cb(addr, data->end, level, ptep, flag, walker->arg);  in kvm_pgtable_visitor_cb()
187 				      kvm_pte_t *ptep, u32 level)  in __kvm_pgtable_visit()  argument
191 	kvm_pte_t *childp, pte = *ptep;  in __kvm_pgtable_visit()
196 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
201 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
203 		pte = *ptep;  in __kvm_pgtable_visit()
222 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
240 		kvm_pte_t *ptep = &pgtable[idx];  in __kvm_pgtable_walk()  local
245 		ret = __kvm_pgtable_visit(data, ptep, level);  in __kvm_pgtable_walk()
267 		kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE];  in _kvm_pgtable_walk()  local
269 		ret = __kvm_pgtable_walk(data, ptep, pgt->start_level);  in _kvm_pgtable_walk()
295 static int leaf_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in leaf_walker()  argument
300 	data->pte   = *ptep;  in leaf_walker()
307 			 kvm_pte_t *ptep, u32 *level)  in kvm_pgtable_get_leaf()  argument
320 		if (ptep)  in kvm_pgtable_get_leaf()
321 			*ptep  = data.pte;  in kvm_pgtable_get_leaf()
335 static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)  in hyp_set_prot_attr()  argument
361 	*ptep = attr;  in hyp_set_prot_attr()
387 				    kvm_pte_t *ptep, struct hyp_map_data *data)  in hyp_map_walker_try_leaf()  argument
389 	kvm_pte_t new, old = *ptep;  in hyp_map_walker_try_leaf()
400 		data->mm_ops->get_page(ptep);  in hyp_map_walker_try_leaf()
404 	smp_store_release(ptep, new);  in hyp_map_walker_try_leaf()
408 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_map_walker()  argument
415 	if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))  in hyp_map_walker()
425 	kvm_set_table_pte(ptep, childp, mm_ops);  in hyp_map_walker()
426 	mm_ops->get_page(ptep);  in hyp_map_walker()
459 static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_unmap_walker()  argument
462 	kvm_pte_t pte = *ptep, *childp = NULL;  in hyp_unmap_walker()
476 		kvm_clear_pte(ptep);  in hyp_unmap_walker()
483 		kvm_clear_pte(ptep);  in hyp_unmap_walker()
491 	mm_ops->put_page(ptep);  in hyp_unmap_walker()
535 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_free_walker()  argument
539 	kvm_pte_t pte = *ptep;  in hyp_free_walker()
544 	mm_ops->put_page(ptep);  in hyp_free_walker()
624 				kvm_pte_t *ptep)  in stage2_set_prot_attr()  argument
645 	*ptep = attr;  in stage2_set_prot_attr()
685 static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr,  in stage2_put_pte()  argument
692 	if (kvm_pte_valid(*ptep)) {  in stage2_put_pte()
693 		kvm_clear_pte(ptep);  in stage2_put_pte()
697 	mm_ops->put_page(ptep);  in stage2_put_pte()
721 				      kvm_pte_t *ptep,  in stage2_map_walker_try_leaf()  argument
724 	kvm_pte_t new, old = *ptep;  in stage2_map_walker_try_leaf()
747 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walker_try_leaf()
758 	smp_store_release(ptep, new);  in stage2_map_walker_try_leaf()
760 		mm_ops->get_page(ptep);  in stage2_map_walker_try_leaf()
767 				     kvm_pte_t *ptep,  in stage2_map_walk_table_pre()  argument
776 	data->childp = kvm_pte_follow(*ptep, data->mm_ops);  in stage2_map_walk_table_pre()
777 	kvm_clear_pte(ptep);  in stage2_map_walk_table_pre()
785 	data->anchor = ptep;  in stage2_map_walk_table_pre()
789 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walk_leaf()  argument
793 	kvm_pte_t *childp, pte = *ptep;  in stage2_map_walk_leaf()
798 			mm_ops->put_page(ptep);  in stage2_map_walk_leaf()
803 	ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);  in stage2_map_walk_leaf()
823 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walk_leaf()
825 	kvm_set_table_pte(ptep, childp, mm_ops);  in stage2_map_walk_leaf()
826 	mm_ops->get_page(ptep);  in stage2_map_walk_leaf()
832 				      kvm_pte_t *ptep,  in stage2_map_walk_table_post()  argument
842 	if (data->anchor == ptep) {  in stage2_map_walk_table_post()
846 		ret = stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walk_table_post()
848 		childp = kvm_pte_follow(*ptep, mm_ops);  in stage2_map_walk_table_post()
852 	mm_ops->put_page(ptep);  in stage2_map_walk_table_post()
876 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walker()  argument
883 		return stage2_map_walk_table_pre(addr, end, level, ptep, data);  in stage2_map_walker()
885 		return stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walker()
887 		return stage2_map_walk_table_post(addr, end, level, ptep, data);  in stage2_map_walker()
952 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_unmap_walker()  argument
959 	kvm_pte_t pte = *ptep, *childp = NULL;  in stage2_unmap_walker()
964 			kvm_clear_pte(ptep);  in stage2_unmap_walker()
965 			mm_ops->put_page(ptep);  in stage2_unmap_walker()
984 	stage2_put_pte(ptep, mmu, addr, level, mm_ops);  in stage2_unmap_walker()
1015 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_attr_walker()  argument
1019 	kvm_pte_t pte = *ptep;  in stage2_attr_walker()
1042 		    stage2_pte_executable(pte) && !stage2_pte_executable(*ptep))  in stage2_attr_walker()
1045 		WRITE_ONCE(*ptep, pte);  in stage2_attr_walker()
1143 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_flush_walker()  argument
1149 	kvm_pte_t pte = *ptep;  in stage2_flush_walker()
1203 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_free_walker()  argument
1208 	kvm_pte_t pte = *ptep;  in stage2_free_walker()
1213 	mm_ops->put_page(ptep);  in stage2_free_walker()