59 static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,  in alloc_new_pmd()  argument
84 static void take_rmap_locks(struct vm_area_struct *vma)  in take_rmap_locks()  argument
86 	if (vma->vm_file)  in take_rmap_locks()
87 		i_mmap_lock_write(vma->vm_file->f_mapping);  in take_rmap_locks()
88 	if (vma->anon_vma)  in take_rmap_locks()
89 		anon_vma_lock_write(vma->anon_vma);  in take_rmap_locks()
92 static void drop_rmap_locks(struct vm_area_struct *vma)  in drop_rmap_locks()  argument
94 	if (vma->anon_vma)  in drop_rmap_locks()
95 		anon_vma_unlock_write(vma->anon_vma);  in drop_rmap_locks()
96 	if (vma->vm_file)  in drop_rmap_locks()
97 		i_mmap_unlock_write(vma->vm_file->f_mapping);  in drop_rmap_locks()
115 static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,  in move_ptes()  argument
120 	struct mm_struct *mm = vma->vm_mm;  in move_ptes()
145 		take_rmap_locks(vma);  in move_ptes()
156 	flush_tlb_batched_pending(vma->vm_mm);  in move_ptes()
185 		flush_tlb_range(vma, old_end - len, old_end);  in move_ptes()
191 		drop_rmap_locks(vma);  in move_ptes()
195 static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,  in move_normal_pmd()  argument
199 	struct mm_struct *mm = vma->vm_mm;  in move_normal_pmd()
232 	old_ptl = pmd_lock(vma->vm_mm, old_pmd);  in move_normal_pmd()
245 	flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);  in move_normal_pmd()
254 unsigned long move_page_tables(struct vm_area_struct *vma,  in move_page_tables()  argument
264 	flush_cache_range(vma, old_addr, old_end);  in move_page_tables()
266 	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,  in move_page_tables()
280 		old_pmd = get_old_pmd(vma->vm_mm, old_addr);  in move_page_tables()
283 		new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);  in move_page_tables()
291 					take_rmap_locks(vma);  in move_page_tables()
292 				moved = move_huge_pmd(vma, old_addr, new_addr,  in move_page_tables()
295 					drop_rmap_locks(vma);  in move_page_tables()
299 			split_huge_pmd(vma, old_pmd, old_addr);  in move_page_tables()
311 				take_rmap_locks(vma);  in move_page_tables()
312 			moved = move_normal_pmd(vma, old_addr, new_addr,  in move_page_tables()
315 				drop_rmap_locks(vma);  in move_page_tables()
323 		move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma,  in move_page_tables()
332 static unsigned long move_vma(struct vm_area_struct *vma,  in move_vma()  argument
338 	struct mm_struct *mm = vma->vm_mm;  in move_vma()
340 	unsigned long vm_flags = vma->vm_flags;  in move_vma()
363 	err = ksm_madvise(vma, old_addr, old_addr + old_len,  in move_vma()
368 	new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);  in move_vma()
369 	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,  in move_vma()
374 	moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,  in move_vma()
378 	} else if (vma->vm_ops && vma->vm_ops->mremap) {  in move_vma()
379 		err = vma->vm_ops->mremap(new_vma);  in move_vma()
388 		move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,  in move_vma()
390 		vma = new_vma;  in move_vma()
402 		vma->vm_flags &= ~VM_ACCOUNT;  in move_vma()
403 		excess = vma->vm_end - vma->vm_start - old_len;  in move_vma()
404 		if (old_addr > vma->vm_start &&  in move_vma()
405 		    old_addr + old_len < vma->vm_end)  in move_vma()
419 	vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT);  in move_vma()
422 	if (unlikely(vma->vm_flags & VM_PFNMAP))  in move_vma()
423 		untrack_pfn_moved(vma);  in move_vma()
428 			vma->vm_flags |= VM_ACCOUNT;  in move_vma()
441 		if (split && new_vma == vma)  in move_vma()
445 		vma->vm_flags &= VM_LOCKED_CLEAR_MASK;  in move_vma()
466 		vma->vm_flags |= VM_ACCOUNT;  in move_vma()
468 			vma->vm_next->vm_flags |= VM_ACCOUNT;  in move_vma()
479 	struct vm_area_struct *vma = find_vma(mm, addr);  in vma_to_resize()  local
482 	if (!vma || vma->vm_start > addr)  in vma_to_resize()
493 	if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) {  in vma_to_resize()
498 	if (flags & MREMAP_DONTUNMAP && (!vma_is_anonymous(vma) ||  in vma_to_resize()
499 			vma->vm_flags & VM_SHARED))  in vma_to_resize()
502 	if (is_vm_hugetlb_page(vma))  in vma_to_resize()
506 	if (old_len > vma->vm_end - addr)  in vma_to_resize()
510 		return vma;  in vma_to_resize()
513 	pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;  in vma_to_resize()
514 	pgoff += vma->vm_pgoff;  in vma_to_resize()
518 	if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))  in vma_to_resize()
521 	if (vma->vm_flags & VM_LOCKED) {  in vma_to_resize()
530 	if (!may_expand_vm(mm, vma->vm_flags,  in vma_to_resize()
534 	if (vma->vm_flags & VM_ACCOUNT) {  in vma_to_resize()
541 	return vma;  in vma_to_resize()
551 	struct vm_area_struct *vma;  in mremap_to()  local
596 	vma = vma_to_resize(addr, old_len, new_len, flags, &charged);  in mremap_to()
597 	if (IS_ERR(vma)) {  in mremap_to()
598 		ret = PTR_ERR(vma);  in mremap_to()
604 		!may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) {  in mremap_to()
612 	if (vma->vm_flags & VM_MAYSHARE)  in mremap_to()
615 	ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +  in mremap_to()
616 				((addr - vma->vm_start) >> PAGE_SHIFT),  in mremap_to()
625 	ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,  in mremap_to()
638 static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)  in vma_expandable()  argument
640 	unsigned long end = vma->vm_end + delta;  in vma_expandable()
641 	if (end < vma->vm_end) /* overflow */  in vma_expandable()
643 	if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */  in vma_expandable()
645 	if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,  in vma_expandable()
663 	struct vm_area_struct *vma;  in SYSCALL_DEFINE5()  local
747 	vma = vma_to_resize(addr, old_len, new_len, flags, &charged);  in SYSCALL_DEFINE5()
748 	if (IS_ERR(vma)) {  in SYSCALL_DEFINE5()
749 		ret = PTR_ERR(vma);  in SYSCALL_DEFINE5()
755 	if (old_len == vma->vm_end - addr) {  in SYSCALL_DEFINE5()
757 		if (vma_expandable(vma, new_len - old_len)) {  in SYSCALL_DEFINE5()
760 			if (vma_adjust(vma, vma->vm_start, addr + new_len,  in SYSCALL_DEFINE5()
761 				       vma->vm_pgoff, NULL)) {  in SYSCALL_DEFINE5()
766 			vm_stat_account(mm, vma->vm_flags, pages);  in SYSCALL_DEFINE5()
767 			if (vma->vm_flags & VM_LOCKED) {  in SYSCALL_DEFINE5()
784 		if (vma->vm_flags & VM_MAYSHARE)  in SYSCALL_DEFINE5()
787 		new_addr = get_unmapped_area(vma->vm_file, 0, new_len,  in SYSCALL_DEFINE5()
788 					vma->vm_pgoff +  in SYSCALL_DEFINE5()
789 					((addr - vma->vm_start) >> PAGE_SHIFT),  in SYSCALL_DEFINE5()
796 		ret = move_vma(vma, addr, old_len, new_len, new_addr,  in SYSCALL_DEFINE5()