25 			    struct vm_area_struct *dst_vma,  in mcopy_atomic_pte()  argument
39 		page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, dst_vma, dst_addr);  in mcopy_atomic_pte()
72 	_dst_pte = mk_pte(page, dst_vma->vm_page_prot);  in mcopy_atomic_pte()
73 	if (dst_vma->vm_flags & VM_WRITE)  in mcopy_atomic_pte()
82 	page_add_new_anon_rmap(page, dst_vma, dst_addr, false);  in mcopy_atomic_pte()
84 	lru_cache_add_active_or_unevictable(page, dst_vma);  in mcopy_atomic_pte()
89 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mcopy_atomic_pte()
105 			      struct vm_area_struct *dst_vma,  in mfill_zeropage_pte()  argument
113 					 dst_vma->vm_page_prot));  in mfill_zeropage_pte()
120 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mfill_zeropage_pte()
154 					      struct vm_area_struct *dst_vma,  in __mcopy_atomic_hugetlb()  argument
160 	int vm_alloc_shared = dst_vma->vm_flags & VM_SHARED;  in __mcopy_atomic_hugetlb()
161 	int vm_shared = dst_vma->vm_flags & VM_SHARED;  in __mcopy_atomic_hugetlb()
188 	vma_hpagesize = vma_kernel_pagesize(dst_vma);  in __mcopy_atomic_hugetlb()
202 	if (!dst_vma) {  in __mcopy_atomic_hugetlb()
204 		dst_vma = find_vma(dst_mm, dst_start);  in __mcopy_atomic_hugetlb()
205 		if (!dst_vma || !is_vm_hugetlb_page(dst_vma))  in __mcopy_atomic_hugetlb()
211 		if (!dst_vma->vm_userfaultfd_ctx.ctx)  in __mcopy_atomic_hugetlb()
214 		if (dst_start < dst_vma->vm_start ||  in __mcopy_atomic_hugetlb()
215 		    dst_start + len > dst_vma->vm_end)  in __mcopy_atomic_hugetlb()
219 		if (vma_hpagesize != vma_kernel_pagesize(dst_vma))  in __mcopy_atomic_hugetlb()
222 		vm_shared = dst_vma->vm_flags & VM_SHARED;  in __mcopy_atomic_hugetlb()
234 		if (unlikely(anon_vma_prepare(dst_vma)))  in __mcopy_atomic_hugetlb()
238 	h = hstate_vma(dst_vma);  in __mcopy_atomic_hugetlb()
249 		idx = linear_page_index(dst_vma, dst_addr);  in __mcopy_atomic_hugetlb()
250 		mapping = dst_vma->vm_file->f_mapping;  in __mcopy_atomic_hugetlb()
251 		hash = hugetlb_fault_mutex_hash(h, dst_mm, dst_vma, mapping,  in __mcopy_atomic_hugetlb()
269 		err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,  in __mcopy_atomic_hugetlb()
290 			dst_vma = NULL;  in __mcopy_atomic_hugetlb()
366 				      struct vm_area_struct *dst_vma,
375 						struct vm_area_struct *dst_vma,  in mfill_atomic_pte()  argument
383 	if (vma_is_anonymous(dst_vma)) {  in mfill_atomic_pte()
385 			err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,  in mfill_atomic_pte()
389 						 dst_vma, dst_addr);  in mfill_atomic_pte()
393 						     dst_vma, dst_addr,  in mfill_atomic_pte()
397 						       dst_vma, dst_addr);  in mfill_atomic_pte()
410 	struct vm_area_struct *dst_vma;  in __mcopy_atomic()  local
448 	dst_vma = find_vma(dst_mm, dst_start);  in __mcopy_atomic()
449 	if (!dst_vma)  in __mcopy_atomic()
460 	if (!dst_vma->vm_userfaultfd_ctx.ctx)  in __mcopy_atomic()
463 	if (dst_start < dst_vma->vm_start ||  in __mcopy_atomic()
464 	    dst_start + len > dst_vma->vm_end)  in __mcopy_atomic()
472 	if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) &&  in __mcopy_atomic()
473 	    dst_vma->vm_flags & VM_SHARED))  in __mcopy_atomic()
479 	if (is_vm_hugetlb_page(dst_vma))  in __mcopy_atomic()
480 		return  __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,  in __mcopy_atomic()
483 	if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))  in __mcopy_atomic()
492 	if (vma_is_anonymous(dst_vma) && unlikely(anon_vma_prepare(dst_vma)))  in __mcopy_atomic()
529 		err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,  in __mcopy_atomic()