2107 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_hole()  local
2111 		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;  in migrate_vma_collect_hole()
2112 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_hole()
2113 		migrate->npages++;  in migrate_vma_collect_hole()
2114 		migrate->cpages++;  in migrate_vma_collect_hole()
2124 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_skip()  local
2128 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_skip()
2129 		migrate->src[migrate->npages++] = 0;  in migrate_vma_collect_skip()
2140 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_pmd()  local
2204 			migrate->cpages++;  in migrate_vma_collect_pmd()
2229 				migrate->cpages++;  in migrate_vma_collect_pmd()
2233 			page = _vm_normal_page(migrate->vma, addr, pte, true);  in migrate_vma_collect_pmd()
2255 		migrate->cpages++;  in migrate_vma_collect_pmd()
2289 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_pmd()
2290 		migrate->src[migrate->npages++] = mpfn;  in migrate_vma_collect_pmd()
2310 static void migrate_vma_collect(struct migrate_vma *migrate)  in migrate_vma_collect()  argument
2319 	mm_walk.vma = migrate->vma;  in migrate_vma_collect()
2320 	mm_walk.mm = migrate->vma->vm_mm;  in migrate_vma_collect()
2321 	mm_walk.private = migrate;  in migrate_vma_collect()
2324 					    migrate->start,  in migrate_vma_collect()
2325 					    migrate->end);  in migrate_vma_collect()
2326 	walk_page_range(migrate->start, migrate->end, &mm_walk);  in migrate_vma_collect()
2328 					  migrate->start,  in migrate_vma_collect()
2329 					  migrate->end);  in migrate_vma_collect()
2331 	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);  in migrate_vma_collect()
2405 static void migrate_vma_prepare(struct migrate_vma *migrate)  in migrate_vma_prepare()  argument
2407 	const unsigned long npages = migrate->npages;  in migrate_vma_prepare()
2408 	const unsigned long start = migrate->start;  in migrate_vma_prepare()
2414 	for (i = 0; (i < npages) && migrate->cpages; i++) {  in migrate_vma_prepare()
2415 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2421 		if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {  in migrate_vma_prepare()
2431 				migrate->src[i] = 0;  in migrate_vma_prepare()
2432 				migrate->cpages--;  in migrate_vma_prepare()
2437 			migrate->src[i] |= MIGRATE_PFN_LOCKED;  in migrate_vma_prepare()
2450 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2451 					migrate->cpages--;  in migrate_vma_prepare()
2454 					migrate->src[i] = 0;  in migrate_vma_prepare()
2456 					migrate->cpages--;  in migrate_vma_prepare()
2468 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2469 				migrate->cpages--;  in migrate_vma_prepare()
2477 				migrate->src[i] = 0;  in migrate_vma_prepare()
2479 				migrate->cpages--;  in migrate_vma_prepare()
2490 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2492 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_prepare()
2495 		remove_migration_pte(page, migrate->vma, addr, page);  in migrate_vma_prepare()
2497 		migrate->src[i] = 0;  in migrate_vma_prepare()
2515 static void migrate_vma_unmap(struct migrate_vma *migrate)  in migrate_vma_unmap()  argument
2518 	const unsigned long npages = migrate->npages;  in migrate_vma_unmap()
2519 	const unsigned long start = migrate->start;  in migrate_vma_unmap()
2523 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2525 		if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2538 		migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_unmap()
2539 		migrate->cpages--;  in migrate_vma_unmap()
2544 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2546 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2551 		migrate->src[i] = 0;  in migrate_vma_unmap()
2562 static void migrate_vma_insert_page(struct migrate_vma *migrate,  in migrate_vma_insert_page()  argument
2568 	struct vm_area_struct *vma = migrate->vma;  in migrate_vma_insert_page()
2706 static void migrate_vma_pages(struct migrate_vma *migrate)  in migrate_vma_pages()  argument
2708 	const unsigned long npages = migrate->npages;  in migrate_vma_pages()
2709 	const unsigned long start = migrate->start;  in migrate_vma_pages()
2710 	struct vm_area_struct *vma = migrate->vma;  in migrate_vma_pages()
2716 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_pages()
2717 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_pages()
2722 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2727 			if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) {  in migrate_vma_pages()
2735 								migrate->end);  in migrate_vma_pages()
2737 			migrate_vma_insert_page(migrate, addr, newpage,  in migrate_vma_pages()
2738 						&migrate->src[i],  in migrate_vma_pages()
2739 						&migrate->dst[i]);  in migrate_vma_pages()
2752 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2760 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2767 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2777 						       migrate->end);  in migrate_vma_pages()
2791 static void migrate_vma_finalize(struct migrate_vma *migrate)  in migrate_vma_finalize()  argument
2793 	const unsigned long npages = migrate->npages;  in migrate_vma_finalize()
2797 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_finalize()
2798 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_finalize()
2808 		if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {  in migrate_vma_finalize()
2818 		migrate->cpages--;  in migrate_vma_finalize()
2894 	struct migrate_vma migrate;  in migrate_vma()  local
2910 	migrate.src = src;  in migrate_vma()
2911 	migrate.dst = dst;  in migrate_vma()
2912 	migrate.start = start;  in migrate_vma()
2913 	migrate.npages = 0;  in migrate_vma()
2914 	migrate.cpages = 0;  in migrate_vma()
2915 	migrate.end = end;  in migrate_vma()
2916 	migrate.vma = vma;  in migrate_vma()
2919 	migrate_vma_collect(&migrate);  in migrate_vma()
2920 	if (!migrate.cpages)  in migrate_vma()
2924 	migrate_vma_prepare(&migrate);  in migrate_vma()
2925 	if (!migrate.cpages)  in migrate_vma()
2929 	migrate_vma_unmap(&migrate);  in migrate_vma()
2930 	if (!migrate.cpages)  in migrate_vma()
2944 	migrate_vma_pages(&migrate);  in migrate_vma()
2949 	migrate_vma_finalize(&migrate);  in migrate_vma()