31 	struct vm_area_struct *dst_vma;  in find_dst_vma()  local
33 	dst_vma = find_vma(dst_mm, dst_start);  in find_dst_vma()
34 	if (!range_in_vma(dst_vma, dst_start, dst_start + len))  in find_dst_vma()
42 	if (!dst_vma->vm_userfaultfd_ctx.ctx)  in find_dst_vma()
45 	return dst_vma;  in find_dst_vma()
49 static bool mfill_file_over_size(struct vm_area_struct *dst_vma,  in mfill_file_over_size()  argument
55 	if (!dst_vma->vm_file)  in mfill_file_over_size()
58 	inode = dst_vma->vm_file->f_inode;  in mfill_file_over_size()
59 	offset = linear_page_index(dst_vma, dst_addr);  in mfill_file_over_size()
71 			     struct vm_area_struct *dst_vma,  in mfill_atomic_install_pte()  argument
76 	struct mm_struct *dst_mm = dst_vma->vm_mm;  in mfill_atomic_install_pte()
78 	bool writable = dst_vma->vm_flags & VM_WRITE;  in mfill_atomic_install_pte()
79 	bool vm_shared = dst_vma->vm_flags & VM_SHARED;  in mfill_atomic_install_pte()
84 	_dst_pte = mk_pte(page, dst_vma->vm_page_prot);  in mfill_atomic_install_pte()
89 		_dst_pte = pte_mkwrite(_dst_pte, dst_vma);  in mfill_atomic_install_pte()
98 	if (mfill_file_over_size(dst_vma, dst_addr)) {  in mfill_atomic_install_pte()
117 		page_add_file_rmap(page, dst_vma, false);  in mfill_atomic_install_pte()
119 		page_add_new_anon_rmap(page, dst_vma, dst_addr);  in mfill_atomic_install_pte()
120 		folio_add_lru_vma(folio, dst_vma);  in mfill_atomic_install_pte()
132 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mfill_atomic_install_pte()
141 				 struct vm_area_struct *dst_vma,  in mfill_atomic_pte_copy()  argument
153 		folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, dst_vma,  in mfill_atomic_pte_copy()
202 	if (mem_cgroup_charge(folio, dst_vma->vm_mm, GFP_KERNEL))  in mfill_atomic_pte_copy()
205 	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,  in mfill_atomic_pte_copy()
217 				     struct vm_area_struct *dst_vma,  in mfill_atomic_pte_zeropage()  argument
225 					 dst_vma->vm_page_prot));  in mfill_atomic_pte_zeropage()
227 	dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl);  in mfill_atomic_pte_zeropage()
230 	if (mfill_file_over_size(dst_vma, dst_addr)) {  in mfill_atomic_pte_zeropage()
237 	set_pte_at(dst_vma->vm_mm, dst_addr, dst_pte, _dst_pte);  in mfill_atomic_pte_zeropage()
239 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mfill_atomic_pte_zeropage()
249 				     struct vm_area_struct *dst_vma,  in mfill_atomic_pte_continue()  argument
253 	struct inode *inode = file_inode(dst_vma->vm_file);  in mfill_atomic_pte_continue()
254 	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);  in mfill_atomic_pte_continue()
276 	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,  in mfill_atomic_pte_continue()
293 				   struct vm_area_struct *dst_vma,  in mfill_atomic_pte_poison()  argument
298 	struct mm_struct *dst_mm = dst_vma->vm_mm;  in mfill_atomic_pte_poison()
308 	if (mfill_file_over_size(dst_vma, dst_addr)) {  in mfill_atomic_pte_poison()
321 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mfill_atomic_pte_poison()
356 					      struct vm_area_struct *dst_vma,  in mfill_atomic_hugetlb()  argument
362 	struct mm_struct *dst_mm = dst_vma->vm_mm;  in mfill_atomic_hugetlb()
363 	int vm_shared = dst_vma->vm_flags & VM_SHARED;  in mfill_atomic_hugetlb()
389 	vma_hpagesize = vma_kernel_pagesize(dst_vma);  in mfill_atomic_hugetlb()
403 	if (!dst_vma) {  in mfill_atomic_hugetlb()
405 		dst_vma = find_dst_vma(dst_mm, dst_start, len);  in mfill_atomic_hugetlb()
406 		if (!dst_vma || !is_vm_hugetlb_page(dst_vma))  in mfill_atomic_hugetlb()
410 		if (vma_hpagesize != vma_kernel_pagesize(dst_vma))  in mfill_atomic_hugetlb()
413 		vm_shared = dst_vma->vm_flags & VM_SHARED;  in mfill_atomic_hugetlb()
421 		if (unlikely(anon_vma_prepare(dst_vma)))  in mfill_atomic_hugetlb()
434 		idx = linear_page_index(dst_vma, dst_addr);  in mfill_atomic_hugetlb()
435 		mapping = dst_vma->vm_file->f_mapping;  in mfill_atomic_hugetlb()
438 		hugetlb_vma_lock_read(dst_vma);  in mfill_atomic_hugetlb()
441 		dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize);  in mfill_atomic_hugetlb()
443 			hugetlb_vma_unlock_read(dst_vma);  in mfill_atomic_hugetlb()
451 			hugetlb_vma_unlock_read(dst_vma);  in mfill_atomic_hugetlb()
456 		err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr,  in mfill_atomic_hugetlb()
459 		hugetlb_vma_unlock_read(dst_vma);  in mfill_atomic_hugetlb()
476 			dst_vma = NULL;  in mfill_atomic_hugetlb()
505 extern ssize_t mfill_atomic_hugetlb(struct vm_area_struct *dst_vma,
513 						struct vm_area_struct *dst_vma,  in mfill_atomic_pte()  argument
522 		return mfill_atomic_pte_continue(dst_pmd, dst_vma,  in mfill_atomic_pte()
525 		return mfill_atomic_pte_poison(dst_pmd, dst_vma,  in mfill_atomic_pte()
539 	if (!(dst_vma->vm_flags & VM_SHARED)) {  in mfill_atomic_pte()
541 			err = mfill_atomic_pte_copy(dst_pmd, dst_vma,  in mfill_atomic_pte()
546 						 dst_vma, dst_addr);  in mfill_atomic_pte()
548 		err = shmem_mfill_atomic_pte(dst_pmd, dst_vma,  in mfill_atomic_pte()
563 	struct vm_area_struct *dst_vma;  in mfill_atomic()  local
601 	dst_vma = find_dst_vma(dst_mm, dst_start, len);  in mfill_atomic()
602 	if (!dst_vma)  in mfill_atomic()
610 	if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) &&  in mfill_atomic()
611 	    dst_vma->vm_flags & VM_SHARED))  in mfill_atomic()
618 	if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP))  in mfill_atomic()
624 	if (is_vm_hugetlb_page(dst_vma))  in mfill_atomic()
625 		return  mfill_atomic_hugetlb(dst_vma, dst_start,  in mfill_atomic()
628 	if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))  in mfill_atomic()
630 	if (!vma_is_shmem(dst_vma) &&  in mfill_atomic()
640 	if (!(dst_vma->vm_flags & VM_SHARED) &&  in mfill_atomic()
641 	    unlikely(anon_vma_prepare(dst_vma)))  in mfill_atomic()
678 		err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr,  in mfill_atomic()
756 long uffd_wp_range(struct vm_area_struct *dst_vma,  in uffd_wp_range()  argument
763 	VM_WARN_ONCE(start < dst_vma->vm_start || start + len > dst_vma->vm_end,  in uffd_wp_range()
776 	if (!enable_wp && vma_wants_manual_pte_write_upgrade(dst_vma))  in uffd_wp_range()
778 	tlb_gather_mmu(&tlb, dst_vma->vm_mm);  in uffd_wp_range()
779 	ret = change_protection(&tlb, dst_vma, start, start + len, mm_cp_flags);  in uffd_wp_range()
791 	struct vm_area_struct *dst_vma;  in mwriteprotect_range()  local
817 	for_each_vma_range(vmi, dst_vma, end) {  in mwriteprotect_range()
819 		if (!userfaultfd_wp(dst_vma)) {  in mwriteprotect_range()
824 		if (is_vm_hugetlb_page(dst_vma)) {  in mwriteprotect_range()
826 			page_mask = vma_kernel_pagesize(dst_vma) - 1;  in mwriteprotect_range()
831 		_start = max(dst_vma->vm_start, start);  in mwriteprotect_range()
832 		_end = min(dst_vma->vm_end, end);  in mwriteprotect_range()
834 		err = uffd_wp_range(dst_vma, _start, _end - _start, enable_wp);  in mwriteprotect_range()