2128 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_hole()  local
2132 		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;  in migrate_vma_collect_hole()
2133 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_hole()
2134 		migrate->npages++;  in migrate_vma_collect_hole()
2135 		migrate->cpages++;  in migrate_vma_collect_hole()
2145 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_skip()  local
2149 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_skip()
2150 		migrate->src[migrate->npages++] = 0;  in migrate_vma_collect_skip()
2161 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_pmd()  local
2224 			migrate->cpages++;  in migrate_vma_collect_pmd()
2249 				migrate->cpages++;  in migrate_vma_collect_pmd()
2252 			page = vm_normal_page(migrate->vma, addr, pte);  in migrate_vma_collect_pmd()
2273 		migrate->cpages++;  in migrate_vma_collect_pmd()
2307 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_pmd()
2308 		migrate->src[migrate->npages++] = mpfn;  in migrate_vma_collect_pmd()
2333 static void migrate_vma_collect(struct migrate_vma *migrate)  in migrate_vma_collect()  argument
2338 			migrate->vma->vm_mm, migrate->start, migrate->end);  in migrate_vma_collect()
2341 	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
2342 			&migrate_vma_walk_ops, migrate);  in migrate_vma_collect()
2345 	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);  in migrate_vma_collect()
2410 static void migrate_vma_prepare(struct migrate_vma *migrate)  in migrate_vma_prepare()  argument
2412 	const unsigned long npages = migrate->npages;  in migrate_vma_prepare()
2413 	const unsigned long start = migrate->start;  in migrate_vma_prepare()
2419 	for (i = 0; (i < npages) && migrate->cpages; i++) {  in migrate_vma_prepare()
2420 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2426 		if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {  in migrate_vma_prepare()
2436 				migrate->src[i] = 0;  in migrate_vma_prepare()
2437 				migrate->cpages--;  in migrate_vma_prepare()
2442 			migrate->src[i] |= MIGRATE_PFN_LOCKED;  in migrate_vma_prepare()
2455 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2456 					migrate->cpages--;  in migrate_vma_prepare()
2459 					migrate->src[i] = 0;  in migrate_vma_prepare()
2461 					migrate->cpages--;  in migrate_vma_prepare()
2473 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2474 				migrate->cpages--;  in migrate_vma_prepare()
2482 				migrate->src[i] = 0;  in migrate_vma_prepare()
2484 				migrate->cpages--;  in migrate_vma_prepare()
2495 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2497 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_prepare()
2500 		remove_migration_pte(page, migrate->vma, addr, page);  in migrate_vma_prepare()
2502 		migrate->src[i] = 0;  in migrate_vma_prepare()
2520 static void migrate_vma_unmap(struct migrate_vma *migrate)  in migrate_vma_unmap()  argument
2523 	const unsigned long npages = migrate->npages;  in migrate_vma_unmap()
2524 	const unsigned long start = migrate->start;  in migrate_vma_unmap()
2528 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2530 		if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2543 		migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_unmap()
2544 		migrate->cpages--;  in migrate_vma_unmap()
2549 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2551 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2556 		migrate->src[i] = 0;  in migrate_vma_unmap()
2671 static void migrate_vma_insert_page(struct migrate_vma *migrate,  in migrate_vma_insert_page()  argument
2677 	struct vm_area_struct *vma = migrate->vma;  in migrate_vma_insert_page()
2810 void migrate_vma_pages(struct migrate_vma *migrate)  in migrate_vma_pages()  argument
2812 	const unsigned long npages = migrate->npages;  in migrate_vma_pages()
2813 	const unsigned long start = migrate->start;  in migrate_vma_pages()
2819 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_pages()
2820 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_pages()
2825 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2830 			if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) {  in migrate_vma_pages()
2839 							migrate->vma->vm_mm,  in migrate_vma_pages()
2840 							addr, migrate->end);  in migrate_vma_pages()
2843 			migrate_vma_insert_page(migrate, addr, newpage,  in migrate_vma_pages()
2844 						&migrate->src[i],  in migrate_vma_pages()
2845 						&migrate->dst[i]);  in migrate_vma_pages()
2858 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2866 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2873 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2897 void migrate_vma_finalize(struct migrate_vma *migrate)  in migrate_vma_finalize()  argument
2899 	const unsigned long npages = migrate->npages;  in migrate_vma_finalize()
2903 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_finalize()
2904 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_finalize()
2914 		if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {  in migrate_vma_finalize()
2924 		migrate->cpages--;  in migrate_vma_finalize()