830 static int copy_cow_page_dax(struct vm_fault *vmf, const struct iomap_iter *iter)  in copy_cow_page_dax()  argument
844 	vto = kmap_atomic(vmf->cow_page);  in copy_cow_page_dax()
845 	copy_user_page(vto, kaddr, vmf->address, vmf->cow_page);  in copy_cow_page_dax()
869 static void *dax_insert_entry(struct xa_state *xas, struct vm_fault *vmf,  in dax_insert_entry()  argument
873 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_insert_entry()
876 	bool dirty = write && !dax_fault_is_synchronous(iter, vmf->vma);  in dax_insert_entry()
898 		dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address,  in dax_insert_entry()
1186 static vm_fault_t dax_load_hole(struct xa_state *xas, struct vm_fault *vmf,  in dax_load_hole()  argument
1190 	unsigned long vaddr = vmf->address;  in dax_load_hole()
1194 	*entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, DAX_ZERO_PAGE);  in dax_load_hole()
1196 	ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);  in dax_load_hole()
1197 	trace_dax_load_hole(inode, vmf, ret);  in dax_load_hole()
1202 static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,  in dax_pmd_load_hole()  argument
1205 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_pmd_load_hole()
1206 	unsigned long pmd_addr = vmf->address & PMD_MASK;  in dax_pmd_load_hole()
1207 	struct vm_area_struct *vma = vmf->vma;  in dax_pmd_load_hole()
1215 	zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);  in dax_pmd_load_hole()
1221 	*entry = dax_insert_entry(xas, vmf, iter, *entry, pfn,  in dax_pmd_load_hole()
1230 	ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in dax_pmd_load_hole()
1231 	if (!pmd_none(*(vmf->pmd))) {  in dax_pmd_load_hole()
1237 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in dax_pmd_load_hole()
1240 	pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);  in dax_pmd_load_hole()
1242 	set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);  in dax_pmd_load_hole()
1244 	trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);  in dax_pmd_load_hole()
1250 	trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);  in dax_pmd_load_hole()
1254 static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,  in dax_pmd_load_hole()  argument
1603 static vm_fault_t dax_fault_cow_page(struct vm_fault *vmf,  in dax_fault_cow_page()  argument
1612 		clear_user_highpage(vmf->cow_page, vmf->address);  in dax_fault_cow_page()
1615 		error = copy_cow_page_dax(vmf, iter);  in dax_fault_cow_page()
1626 	__SetPageUptodate(vmf->cow_page);  in dax_fault_cow_page()
1627 	ret = finish_fault(vmf);  in dax_fault_cow_page()
1642 static vm_fault_t dax_fault_iter(struct vm_fault *vmf,  in dax_fault_iter()  argument
1656 	if (!pmd && vmf->cow_page)  in dax_fault_iter()
1657 		return dax_fault_cow_page(vmf, iter);  in dax_fault_iter()
1663 			return dax_load_hole(xas, vmf, iter, entry);  in dax_fault_iter()
1664 		return dax_pmd_load_hole(xas, vmf, iter, entry);  in dax_fault_iter()
1676 	*entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, entry_flags);  in dax_fault_iter()
1684 	if (dax_fault_is_synchronous(iter, vmf->vma))  in dax_fault_iter()
1689 		return vmf_insert_pfn_pmd(vmf, pfn, write);  in dax_fault_iter()
1693 		return vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);  in dax_fault_iter()
1694 	return vmf_insert_mixed(vmf->vma, vmf->address, pfn);  in dax_fault_iter()
1697 static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,  in dax_iomap_pte_fault()  argument
1700 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_iomap_pte_fault()
1701 	XA_STATE(xas, &mapping->i_pages, vmf->pgoff);  in dax_iomap_pte_fault()
1704 		.pos		= (loff_t)vmf->pgoff << PAGE_SHIFT,  in dax_iomap_pte_fault()
1712 	trace_dax_pte_fault(iter.inode, vmf, ret);  in dax_iomap_pte_fault()
1723 	if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page)  in dax_iomap_pte_fault()
1738 	if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {  in dax_iomap_pte_fault()
1749 		ret = dax_fault_iter(vmf, &iter, pfnp, &xas, &entry, false);  in dax_iomap_pte_fault()
1753 			count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);  in dax_iomap_pte_fault()
1769 	trace_dax_pte_fault_done(iter.inode, vmf, ret);  in dax_iomap_pte_fault()
1774 static bool dax_fault_check_fallback(struct vm_fault *vmf, struct xa_state *xas,  in dax_fault_check_fallback()  argument
1777 	unsigned long pmd_addr = vmf->address & PMD_MASK;  in dax_fault_check_fallback()
1778 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in dax_fault_check_fallback()
1786 	if ((vmf->pgoff & PG_PMD_COLOUR) !=  in dax_fault_check_fallback()
1787 	    ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))  in dax_fault_check_fallback()
1791 	if (write && !(vmf->vma->vm_flags & VM_SHARED))  in dax_fault_check_fallback()
1795 	if (pmd_addr < vmf->vma->vm_start)  in dax_fault_check_fallback()
1797 	if ((pmd_addr + PMD_SIZE) > vmf->vma->vm_end)  in dax_fault_check_fallback()
1807 static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,  in dax_iomap_pmd_fault()  argument
1810 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_iomap_pmd_fault()
1811 	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);  in dax_iomap_pmd_fault()
1821 	if (vmf->flags & FAULT_FLAG_WRITE)  in dax_iomap_pmd_fault()
1831 	trace_dax_pmd_fault(iter.inode, vmf, max_pgoff, 0);  in dax_iomap_pmd_fault()
1838 	if (dax_fault_check_fallback(vmf, &xas, max_pgoff))  in dax_iomap_pmd_fault()
1859 	if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&  in dax_iomap_pmd_fault()
1860 			!pmd_devmap(*vmf->pmd)) {  in dax_iomap_pmd_fault()
1870 		ret = dax_fault_iter(vmf, &iter, pfnp, &xas, &entry, true);  in dax_iomap_pmd_fault()
1879 		split_huge_pmd(vmf->vma, vmf->pmd, vmf->address);  in dax_iomap_pmd_fault()
1883 	trace_dax_pmd_fault_done(iter.inode, vmf, max_pgoff, ret);  in dax_iomap_pmd_fault()
1887 static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,  in dax_iomap_pmd_fault()  argument
1907 vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,  in dax_iomap_fault()  argument
1911 		return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);  in dax_iomap_fault()
1913 		return dax_iomap_pmd_fault(vmf, pfnp, ops);  in dax_iomap_fault()
1929 dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)  in dax_insert_pfn_mkwrite()  argument
1931 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_insert_pfn_mkwrite()
1932 	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);  in dax_insert_pfn_mkwrite()
1943 		trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,  in dax_insert_pfn_mkwrite()
1951 		ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);  in dax_insert_pfn_mkwrite()
1954 		ret = vmf_insert_pfn_pmd(vmf, pfn, FAULT_FLAG_WRITE);  in dax_insert_pfn_mkwrite()
1959 	trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);  in dax_insert_pfn_mkwrite()
1973 vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, unsigned int order,  in dax_finish_sync_fault()  argument
1977 	loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;  in dax_finish_sync_fault()
1980 	err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);  in dax_finish_sync_fault()
1983 	return dax_insert_pfn_mkwrite(vmf, pfn, order);  in dax_finish_sync_fault()