355 static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr)  in __gmap_unlink_by_vmaddr()  argument
362 	entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);  in __gmap_unlink_by_vmaddr()
380 	unsigned long vmaddr;  in __gmap_unmap_by_gaddr()  local
382 	vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host,  in __gmap_unmap_by_gaddr()
384 	return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0;  in __gmap_unmap_by_gaddr()
475 	unsigned long vmaddr;  in __gmap_translate()  local
477 	vmaddr = (unsigned long)  in __gmap_translate()
480 	return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT;  in __gmap_translate()
511 		 unsigned long vmaddr)  in gmap_unlink()  argument
518 		flush = __gmap_unlink_by_vmaddr(gmap, vmaddr);  in gmap_unlink()
539 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)  in __gmap_link()  argument
581 	pgd = pgd_offset(mm, vmaddr);  in __gmap_link()
583 	p4d = p4d_offset(pgd, vmaddr);  in __gmap_link()
585 	pud = pud_offset(p4d, vmaddr);  in __gmap_link()
590 	pmd = pmd_offset(pud, vmaddr);  in __gmap_link()
603 				       vmaddr >> PMD_SHIFT, table);  in __gmap_link()
638 	unsigned long vmaddr;  in gmap_fault()  local
646 	vmaddr = __gmap_translate(gmap, gaddr);  in gmap_fault()
647 	if (IS_ERR_VALUE(vmaddr)) {  in gmap_fault()
648 		rc = vmaddr;  in gmap_fault()
651 	if (fixup_user_fault(gmap->mm, vmaddr, fault_flags,  in gmap_fault()
663 	rc = __gmap_link(gmap, gaddr, vmaddr);  in gmap_fault()
675 	unsigned long vmaddr;  in __gmap_zap()  local
680 	vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host,  in __gmap_zap()
682 	if (vmaddr) {  in __gmap_zap()
683 		vmaddr |= gaddr & ~PMD_MASK;  in __gmap_zap()
685 		ptep = get_locked_pte(gmap->mm, vmaddr, &ptl);  in __gmap_zap()
687 			ptep_zap_unused(gmap->mm, vmaddr, ptep, 0);  in __gmap_zap()
695 	unsigned long gaddr, vmaddr, size;  in gmap_discard()  local
702 		vmaddr = (unsigned long)  in gmap_discard()
705 		if (!vmaddr)  in gmap_discard()
707 		vmaddr |= gaddr & ~PMD_MASK;  in gmap_discard()
709 		vma = find_vma(gmap->mm, vmaddr);  in gmap_discard()
719 		zap_page_range(vma, vmaddr, size);  in gmap_discard()
873 			     unsigned long vmaddr, int prot)  in gmap_pte_op_fixup()  argument
881 	if (fixup_user_fault(mm, vmaddr, fault_flags, &unlocked))  in gmap_pte_op_fixup()
887 	return __gmap_link(gmap, gaddr, vmaddr);  in gmap_pte_op_fixup()
1042 	unsigned long vmaddr, dist;  in gmap_protect_range()  local
1074 			vmaddr = __gmap_translate(gmap, gaddr);  in gmap_protect_range()
1075 			if (IS_ERR_VALUE(vmaddr))  in gmap_protect_range()
1076 				return vmaddr;  in gmap_protect_range()
1077 			rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);  in gmap_protect_range()
1130 	unsigned long address, vmaddr;  in gmap_read_table()  local
1155 		vmaddr = __gmap_translate(gmap, gaddr);  in gmap_read_table()
1156 		if (IS_ERR_VALUE(vmaddr)) {  in gmap_read_table()
1157 			rc = vmaddr;  in gmap_read_table()
1160 		rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ);  in gmap_read_table()
1176 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,  in gmap_insert_rmap()  argument
1182 	slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_insert_rmap()
1189 		radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,  in gmap_insert_rmap()
1209 	unsigned long vmaddr;  in gmap_protect_rmap()  local
1217 		vmaddr = __gmap_translate(parent, paddr);  in gmap_protect_rmap()
1218 		if (IS_ERR_VALUE(vmaddr))  in gmap_protect_rmap()
1219 			return vmaddr;  in gmap_protect_rmap()
1236 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_protect_rmap()
1243 			rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ);  in gmap_protect_rmap()
2108 	unsigned long vmaddr, paddr;  in gmap_shadow_page()  local
2125 		vmaddr = __gmap_translate(parent, paddr);  in gmap_shadow_page()
2126 		if (IS_ERR_VALUE(vmaddr)) {  in gmap_shadow_page()
2127 			rc = vmaddr;  in gmap_shadow_page()
2148 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_shadow_page()
2158 		rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot);  in gmap_shadow_page()
2172 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,  in gmap_shadow_notify()  argument
2198 	head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_shadow_notify()
2234 void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,  in ptep_notify()  argument
2247 					  vmaddr >> PMD_SHIFT);  in ptep_notify()
2258 				gmap_shadow_notify(sg, vmaddr, gaddr);  in ptep_notify()
2301 static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr,  in gmap_pmdp_clear()  argument
2312 						  vmaddr >> PMD_SHIFT);  in gmap_pmdp_clear()
2333 void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_invalidate()  argument
2335 	gmap_pmdp_clear(mm, vmaddr, 0);  in gmap_pmdp_invalidate()
2344 void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_csp()  argument
2346 	gmap_pmdp_clear(mm, vmaddr, 1);  in gmap_pmdp_csp()
2355 void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_idte_local()  argument
2365 					  vmaddr >> PMD_SHIFT);  in gmap_pmdp_idte_local()
2390 void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_idte_global()  argument
2400 					  vmaddr >> PMD_SHIFT);  in gmap_pmdp_idte_global()
2459 			     unsigned long gaddr, unsigned long vmaddr)  in gmap_sync_dirty_log_pmd()  argument
2474 		for (i = 0; i < _PAGE_ENTRIES; i++, vmaddr += PAGE_SIZE) {  in gmap_sync_dirty_log_pmd()
2475 			ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl);  in gmap_sync_dirty_log_pmd()
2478 			if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep))  in gmap_sync_dirty_log_pmd()