3  * Memory Migration functionality - linux/mm/migrate.c
16 #include <linux/migrate.h>
57 #include <trace/events/migrate.h>
632  * Common logic to directly migrate a single LRU page suitable for
938 			 * cannot try to migrate this page.  in move_to_new_page()
1131  *	  -1, // Node 2 does not migrate
1134  *	  -1} // Node 5 does not migrate
1178  * Obtain the lock on page, remove all ptes and migrate the page
1232 		 * Compaction can migrate also non-LRU pages which are  in unmap_and_move()
1268  * This means that when we try to migrate hugepage whose subpages are
1420  * migrate_pages - migrate the pages specified in a list, to the free pages
1530 				 * When memory is low, don't bother to try to migrate  in migrate_pages()
1764 		 * pages to migrate.  Since we are going to  in move_pages_and_store_status()
1778  * Migrate an array of page address onto an array of nodes and fill
2116 	/* Do not migrate THP mapped by multiple processes */  in numamigrate_isolate_page()
2141  * Attempt to migrate a misplaced page to the specified destination
2169 	 * Don't migrate file pages that are mapped in multiple processes  in migrate_misplaced_page()
2177 	 * Also do not migrate dirty pages as not all filesystems can move  in migrate_misplaced_page()
2215 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_skip()  local
2219 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_skip()
2220 		migrate->src[migrate->npages++] = 0;  in migrate_vma_collect_skip()
2231 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_hole()  local
2239 		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;  in migrate_vma_collect_hole()
2240 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_hole()
2241 		migrate->npages++;  in migrate_vma_collect_hole()
2242 		migrate->cpages++;  in migrate_vma_collect_hole()
2253 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_pmd()  local
2317 				migrate->cpages++;  in migrate_vma_collect_pmd()
2333 			if (!(migrate->flags &  in migrate_vma_collect_pmd()
2335 			    page->pgmap->owner != migrate->pgmap_owner)  in migrate_vma_collect_pmd()
2343 			if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))  in migrate_vma_collect_pmd()
2348 				migrate->cpages++;  in migrate_vma_collect_pmd()
2351 			page = vm_normal_page(migrate->vma, addr, pte);  in migrate_vma_collect_pmd()
2372 		migrate->cpages++;  in migrate_vma_collect_pmd()
2419 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_pmd()
2420 		migrate->src[migrate->npages++] = mpfn;  in migrate_vma_collect_pmd()
2439  * @migrate: migrate struct containing all migration information
2445 static void migrate_vma_collect(struct migrate_vma *migrate)  in migrate_vma_collect()  argument
2455 		migrate->vma, migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
2456 		migrate->pgmap_owner);  in migrate_vma_collect()
2459 	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
2460 			&migrate_vma_walk_ops, migrate);  in migrate_vma_collect()
2463 	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);  in migrate_vma_collect()
2499 		 * avoid 2 racing thread trying to migrate back to CPU to enter  in migrate_vma_check_page()
2521  * @migrate: migrate struct containing all migration information
2528 static void migrate_vma_prepare(struct migrate_vma *migrate)  in migrate_vma_prepare()  argument
2530 	const unsigned long npages = migrate->npages;  in migrate_vma_prepare()
2531 	const unsigned long start = migrate->start;  in migrate_vma_prepare()
2537 	for (i = 0; (i < npages) && migrate->cpages; i++) {  in migrate_vma_prepare()
2538 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2544 		if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {  in migrate_vma_prepare()
2554 				migrate->src[i] = 0;  in migrate_vma_prepare()
2555 				migrate->cpages--;  in migrate_vma_prepare()
2560 			migrate->src[i] |= MIGRATE_PFN_LOCKED;  in migrate_vma_prepare()
2573 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2574 					migrate->cpages--;  in migrate_vma_prepare()
2577 					migrate->src[i] = 0;  in migrate_vma_prepare()
2579 					migrate->cpages--;  in migrate_vma_prepare()
2591 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2592 				migrate->cpages--;  in migrate_vma_prepare()
2600 				migrate->src[i] = 0;  in migrate_vma_prepare()
2602 				migrate->cpages--;  in migrate_vma_prepare()
2613 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2615 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_prepare()
2618 		remove_migration_pte(page, migrate->vma, addr, page);  in migrate_vma_prepare()
2620 		migrate->src[i] = 0;  in migrate_vma_prepare()
2629  * @migrate: migrate struct containing all migration information
2633  * cannot migrate them.
2638 static void migrate_vma_unmap(struct migrate_vma *migrate)  in migrate_vma_unmap()  argument
2640 	const unsigned long npages = migrate->npages;  in migrate_vma_unmap()
2641 	const unsigned long start = migrate->start;  in migrate_vma_unmap()
2645 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2647 		if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2660 		migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_unmap()
2661 		migrate->cpages--;  in migrate_vma_unmap()
2666 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2668 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2673 		migrate->src[i] = 0;  in migrate_vma_unmap()
2685  * migrate_vma_setup() - prepare to migrate a range of memory
2691  * Prepare to migrate a range of memory virtual address range by collecting all
2706  * Note that the caller does not have to migrate all the pages that are marked
2708  * device memory to system memory.  If the caller cannot migrate a device page
2725  * then migrate_vma_pages() to migrate struct page information from the source
2726  * struct page to the destination struct page.  If it fails to migrate the
2796 static void migrate_vma_insert_page(struct migrate_vma *migrate,  in migrate_vma_insert_page()  argument
2801 	struct vm_area_struct *vma = migrate->vma;  in migrate_vma_insert_page()
2933  * migrate_vma_pages() - migrate meta-data from src page to dst page
2934  * @migrate: migrate struct containing all migration information
2940 void migrate_vma_pages(struct migrate_vma *migrate)  in migrate_vma_pages()  argument
2942 	const unsigned long npages = migrate->npages;  in migrate_vma_pages()
2943 	const unsigned long start = migrate->start;  in migrate_vma_pages()
2949 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_pages()
2950 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_pages()
2955 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2960 			if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_pages()
2966 					MMU_NOTIFY_MIGRATE, 0, migrate->vma,  in migrate_vma_pages()
2967 					migrate->vma->vm_mm, addr, migrate->end,  in migrate_vma_pages()
2968 					migrate->pgmap_owner);  in migrate_vma_pages()
2971 			migrate_vma_insert_page(migrate, addr, newpage,  in migrate_vma_pages()
2972 						&migrate->src[i]);  in migrate_vma_pages()
2985 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
2993 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
3000 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
3015  * @migrate: migrate struct containing all migration information
3024 void migrate_vma_finalize(struct migrate_vma *migrate)  in migrate_vma_finalize()  argument
3026 	const unsigned long npages = migrate->npages;  in migrate_vma_finalize()
3030 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_finalize()
3031 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_finalize()
3041 		if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {  in migrate_vma_finalize()
3138  * node1 migrates to can migrate to node0.
3212  * This leaves migrate-on-reclaim transiently disabled between