596 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,  in __do_huge_pmd_anonymous_page()  argument
599 	struct vm_area_struct *vma = vmf->vma;  in __do_huge_pmd_anonymous_page()
601 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in __do_huge_pmd_anonymous_page()
620 	clear_huge_page(page, vmf->address, HPAGE_PMD_NR);  in __do_huge_pmd_anonymous_page()
628 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
629 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
640 			spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
643 			ret = handle_userfault(vmf, VM_UFFD_MISSING);  in __do_huge_pmd_anonymous_page()
652 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
653 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
654 		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in __do_huge_pmd_anonymous_page()
657 		spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
664 	spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
724 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)  in do_huge_pmd_anonymous_page()  argument
726 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_anonymous_page()
729 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_anonymous_page()
737 	if (!(vmf->flags & FAULT_FLAG_WRITE) &&  in do_huge_pmd_anonymous_page()
752 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
754 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
757 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
760 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
762 				ret = handle_userfault(vmf, VM_UFFD_MISSING);  in do_huge_pmd_anonymous_page()
766 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
767 				update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_anonymous_page()
768 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
771 			spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
783 	return __do_huge_pmd_anonymous_page(vmf, page, gfp);  in do_huge_pmd_anonymous_page()
846 vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,  in vmf_insert_pfn_pmd_prot()  argument
849 	unsigned long addr = vmf->address & PMD_MASK;  in vmf_insert_pfn_pmd_prot()
850 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pmd_prot()
875 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd_prot()
937 vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,  in vmf_insert_pfn_pud_prot()  argument
940 	unsigned long addr = vmf->address & PUD_MASK;  in vmf_insert_pfn_pud_prot()
941 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pud_prot()
959 	insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);  in vmf_insert_pfn_pud_prot()
1239 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)  in huge_pud_set_accessed()  argument
1243 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pud_set_accessed()
1245 	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);  in huge_pud_set_accessed()
1246 	if (unlikely(!pud_same(*vmf->pud, orig_pud)))  in huge_pud_set_accessed()
1252 	haddr = vmf->address & HPAGE_PUD_MASK;  in huge_pud_set_accessed()
1253 	if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write))  in huge_pud_set_accessed()
1254 		update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud);  in huge_pud_set_accessed()
1257 	spin_unlock(vmf->ptl);  in huge_pud_set_accessed()
1261 void huge_pmd_set_accessed(struct vm_fault *vmf)  in huge_pmd_set_accessed()  argument
1265 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pmd_set_accessed()
1266 	pmd_t orig_pmd = vmf->orig_pmd;  in huge_pmd_set_accessed()
1268 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1269 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1275 	haddr = vmf->address & HPAGE_PMD_MASK;  in huge_pmd_set_accessed()
1276 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1277 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1280 	spin_unlock(vmf->ptl);  in huge_pmd_set_accessed()
1283 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)  in do_huge_pmd_wp_page()  argument
1285 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_wp_page()
1287 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_wp_page()
1288 	pmd_t orig_pmd = vmf->orig_pmd;  in do_huge_pmd_wp_page()
1290 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1296 	spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1298 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1299 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1309 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1311 		spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1312 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1313 			spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1329 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))  in do_huge_pmd_wp_page()
1330 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1332 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1337 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1339 	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);  in do_huge_pmd_wp_page()
1424 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)  in do_huge_pmd_numa_page()  argument
1426 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_numa_page()
1427 	pmd_t oldpmd = vmf->orig_pmd;  in do_huge_pmd_numa_page()
1430 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_numa_page()
1437 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1438 	if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1439 		spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1462 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1470 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1471 		if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1472 			spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1491 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1492 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1493 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()