565 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,  in __do_huge_pmd_anonymous_page()  argument
568 	struct vm_area_struct *vma = vmf->vma;  in __do_huge_pmd_anonymous_page()
570 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in __do_huge_pmd_anonymous_page()
589 	clear_huge_page(page, vmf->address, HPAGE_PMD_NR);  in __do_huge_pmd_anonymous_page()
597 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
598 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
611 			spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
614 			ret2 = handle_userfault(vmf, VM_UFFD_MISSING);  in __do_huge_pmd_anonymous_page()
623 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
624 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
627 		spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
634 	spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
695 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)  in do_huge_pmd_anonymous_page()  argument
697 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_anonymous_page()
700 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_anonymous_page()
708 	if (!(vmf->flags & FAULT_FLAG_WRITE) &&  in do_huge_pmd_anonymous_page()
723 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
725 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
728 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
731 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
733 				ret = handle_userfault(vmf, VM_UFFD_MISSING);  in do_huge_pmd_anonymous_page()
737 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
738 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
741 			spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
753 	return __do_huge_pmd_anonymous_page(vmf, page, gfp);  in do_huge_pmd_anonymous_page()
816 vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,  in vmf_insert_pfn_pmd_prot()  argument
819 	unsigned long addr = vmf->address & PMD_MASK;  in vmf_insert_pfn_pmd_prot()
820 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pmd_prot()
845 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd_prot()
907 vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,  in vmf_insert_pfn_pud_prot()  argument
910 	unsigned long addr = vmf->address & PUD_MASK;  in vmf_insert_pfn_pud_prot()
911 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pud_prot()
929 	insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);  in vmf_insert_pfn_pud_prot()
1218 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)  in huge_pud_set_accessed()  argument
1222 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pud_set_accessed()
1224 	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);  in huge_pud_set_accessed()
1225 	if (unlikely(!pud_same(*vmf->pud, orig_pud)))  in huge_pud_set_accessed()
1231 	haddr = vmf->address & HPAGE_PUD_MASK;  in huge_pud_set_accessed()
1232 	if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write))  in huge_pud_set_accessed()
1233 		update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud);  in huge_pud_set_accessed()
1236 	spin_unlock(vmf->ptl);  in huge_pud_set_accessed()
1240 void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd)  in huge_pmd_set_accessed()  argument
1244 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pmd_set_accessed()
1246 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1247 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1253 	haddr = vmf->address & HPAGE_PMD_MASK;  in huge_pmd_set_accessed()
1254 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1255 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1258 	spin_unlock(vmf->ptl);  in huge_pmd_set_accessed()
1261 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)  in do_huge_pmd_wp_page()  argument
1263 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_wp_page()
1265 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_wp_page()
1267 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1273 	spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1275 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1276 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1286 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1288 		spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1289 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1290 			spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1306 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))  in do_huge_pmd_wp_page()
1307 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1309 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1314 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1316 	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);  in do_huge_pmd_wp_page()
1401 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)  in do_huge_pmd_numa_page()  argument
1403 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_numa_page()
1406 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_numa_page()
1414 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1415 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))  in do_huge_pmd_numa_page()
1423 	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {  in do_huge_pmd_numa_page()
1424 		page = pmd_page(*vmf->pmd);  in do_huge_pmd_numa_page()
1427 		spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1463 		spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1473 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1477 	spin_lock(vmf->ptl);  in do_huge_pmd_numa_page()
1478 	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1522 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1525 				vmf->pmd, pmd, vmf->address, page, target_nid);  in do_huge_pmd_numa_page()
1540 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1541 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1544 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()