646 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,  in __do_huge_pmd_anonymous_page()  argument
649 	struct vm_area_struct *vma = vmf->vma;  in __do_huge_pmd_anonymous_page()
652 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in __do_huge_pmd_anonymous_page()
671 	clear_huge_page(page, vmf->address, HPAGE_PMD_NR);  in __do_huge_pmd_anonymous_page()
679 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
680 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
691 			spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
694 			ret = handle_userfault(vmf, VM_UFFD_MISSING);  in __do_huge_pmd_anonymous_page()
703 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
704 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
705 		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in __do_huge_pmd_anonymous_page()
708 		spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
715 	spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
774 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)  in do_huge_pmd_anonymous_page()  argument
776 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_anonymous_page()
779 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_anonymous_page()
787 	if (!(vmf->flags & FAULT_FLAG_WRITE) &&  in do_huge_pmd_anonymous_page()
802 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
804 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
807 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
810 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
812 				ret = handle_userfault(vmf, VM_UFFD_MISSING);  in do_huge_pmd_anonymous_page()
816 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
817 				update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_anonymous_page()
818 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
821 			spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
832 	return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp);  in do_huge_pmd_anonymous_page()
892 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write)  in vmf_insert_pfn_pmd()  argument
894 	unsigned long addr = vmf->address & PMD_MASK;  in vmf_insert_pfn_pmd()
895 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pmd()
921 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd()
981 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write)  in vmf_insert_pfn_pud()  argument
983 	unsigned long addr = vmf->address & PUD_MASK;  in vmf_insert_pfn_pud()
984 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pud()
1003 	insert_pfn_pud(vma, addr, vmf->pud, pfn, write);  in vmf_insert_pfn_pud()
1261 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)  in huge_pud_set_accessed()  argument
1263 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pud_set_accessed()
1265 	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);  in huge_pud_set_accessed()
1266 	if (unlikely(!pud_same(*vmf->pud, orig_pud)))  in huge_pud_set_accessed()
1269 	touch_pud(vmf->vma, vmf->address, vmf->pud, write);  in huge_pud_set_accessed()
1271 	spin_unlock(vmf->ptl);  in huge_pud_set_accessed()
1275 void huge_pmd_set_accessed(struct vm_fault *vmf)  in huge_pmd_set_accessed()  argument
1277 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pmd_set_accessed()
1279 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1280 	if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd)))  in huge_pmd_set_accessed()
1283 	touch_pmd(vmf->vma, vmf->address, vmf->pmd, write);  in huge_pmd_set_accessed()
1286 	spin_unlock(vmf->ptl);  in huge_pmd_set_accessed()
1289 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)  in do_huge_pmd_wp_page()  argument
1291 	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;  in do_huge_pmd_wp_page()
1292 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_wp_page()
1295 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_wp_page()
1296 	pmd_t orig_pmd = vmf->orig_pmd;  in do_huge_pmd_wp_page()
1298 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1304 	spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1306 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1307 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1321 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1323 		spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1324 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1325 			spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1356 			spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1361 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))  in do_huge_pmd_wp_page()
1362 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1363 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1369 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1371 	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);  in do_huge_pmd_wp_page()
1488 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)  in do_huge_pmd_numa_page()  argument
1490 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_numa_page()
1491 	pmd_t oldpmd = vmf->orig_pmd;  in do_huge_pmd_numa_page()
1494 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_numa_page()
1500 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1501 	if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1502 		spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1514 	    can_change_pmd_writable(vma, vmf->address, pmd))  in do_huge_pmd_numa_page()
1540 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1549 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1550 		if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1551 			spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1570 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1571 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1572 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()