Lines Matching refs:page
86 int isolate_movable_page(struct page *page, isolate_mode_t mode) in isolate_movable_page() argument
99 if (unlikely(!get_page_unless_zero(page))) in isolate_movable_page()
107 if (unlikely(!__PageMovable(page))) in isolate_movable_page()
120 if (unlikely(!trylock_page(page))) in isolate_movable_page()
123 if (!PageMovable(page) || PageIsolated(page)) in isolate_movable_page()
126 mapping = page_mapping(page); in isolate_movable_page()
127 VM_BUG_ON_PAGE(!mapping, page); in isolate_movable_page()
129 if (!mapping->a_ops->isolate_page(page, mode)) in isolate_movable_page()
133 WARN_ON_ONCE(PageIsolated(page)); in isolate_movable_page()
134 __SetPageIsolated(page); in isolate_movable_page()
135 unlock_page(page); in isolate_movable_page()
140 unlock_page(page); in isolate_movable_page()
142 put_page(page); in isolate_movable_page()
148 void putback_movable_page(struct page *page) in putback_movable_page() argument
152 VM_BUG_ON_PAGE(!PageLocked(page), page); in putback_movable_page()
153 VM_BUG_ON_PAGE(!PageMovable(page), page); in putback_movable_page()
154 VM_BUG_ON_PAGE(!PageIsolated(page), page); in putback_movable_page()
156 mapping = page_mapping(page); in putback_movable_page()
157 mapping->a_ops->putback_page(page); in putback_movable_page()
158 __ClearPageIsolated(page); in putback_movable_page()
171 struct page *page; in putback_movable_pages() local
172 struct page *page2; in putback_movable_pages()
174 list_for_each_entry_safe(page, page2, l, lru) { in putback_movable_pages()
175 if (unlikely(PageHuge(page))) { in putback_movable_pages()
176 putback_active_hugepage(page); in putback_movable_pages()
179 list_del(&page->lru); in putback_movable_pages()
185 if (unlikely(__PageMovable(page))) { in putback_movable_pages()
186 VM_BUG_ON_PAGE(!PageIsolated(page), page); in putback_movable_pages()
187 lock_page(page); in putback_movable_pages()
188 if (PageMovable(page)) in putback_movable_pages()
189 putback_movable_page(page); in putback_movable_pages()
191 __ClearPageIsolated(page); in putback_movable_pages()
192 unlock_page(page); in putback_movable_pages()
193 put_page(page); in putback_movable_pages()
195 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + in putback_movable_pages()
196 page_is_file_lru(page), -thp_nr_pages(page)); in putback_movable_pages()
197 putback_lru_page(page); in putback_movable_pages()
205 static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, in remove_migration_pte() argument
209 .page = old, in remove_migration_pte()
214 struct page *new; in remove_migration_pte()
218 VM_BUG_ON_PAGE(PageTail(page), page); in remove_migration_pte()
220 if (PageKsm(page)) in remove_migration_pte()
221 new = page; in remove_migration_pte()
223 new = page - pvmw.page->index + in remove_migration_pte()
229 VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); in remove_migration_pte()
280 if (PageTransHuge(page) && PageMlocked(page)) in remove_migration_pte()
281 clear_page_mlock(page); in remove_migration_pte()
294 void remove_migration_ptes(struct page *old, struct page *new, bool locked) in remove_migration_ptes()
317 struct page *page; in __migration_entry_wait() local
328 page = migration_entry_to_page(entry); in __migration_entry_wait()
335 if (!get_page_unless_zero(page)) in __migration_entry_wait()
338 put_and_wait_on_page_locked(page); in __migration_entry_wait()
363 struct page *page; in pmd_migration_entry_wait() local
368 page = migration_entry_to_page(pmd_to_swp_entry(*pmd)); in pmd_migration_entry_wait()
369 if (!get_page_unless_zero(page)) in pmd_migration_entry_wait()
372 put_and_wait_on_page_locked(page); in pmd_migration_entry_wait()
379 static int expected_page_refs(struct address_space *mapping, struct page *page) in expected_page_refs() argument
387 expected_count += is_device_private_page(page); in expected_page_refs()
389 expected_count += thp_nr_pages(page) + page_has_private(page); in expected_page_refs()
403 struct page *newpage, struct page *page, int extra_count) in migrate_page_move_mapping() argument
405 XA_STATE(xas, &mapping->i_pages, page_index(page)); in migrate_page_move_mapping()
408 int expected_count = expected_page_refs(mapping, page) + extra_count; in migrate_page_move_mapping()
412 if (page_count(page) != expected_count) in migrate_page_move_mapping()
416 newpage->index = page->index; in migrate_page_move_mapping()
417 newpage->mapping = page->mapping; in migrate_page_move_mapping()
418 if (PageSwapBacked(page)) in migrate_page_move_mapping()
424 oldzone = page_zone(page); in migrate_page_move_mapping()
428 if (page_count(page) != expected_count || xas_load(&xas) != page) { in migrate_page_move_mapping()
433 if (!page_ref_freeze(page, expected_count)) { in migrate_page_move_mapping()
442 newpage->index = page->index; in migrate_page_move_mapping()
443 newpage->mapping = page->mapping; in migrate_page_move_mapping()
444 page_ref_add(newpage, thp_nr_pages(page)); /* add cache reference */ in migrate_page_move_mapping()
445 if (PageSwapBacked(page)) { in migrate_page_move_mapping()
447 if (PageSwapCache(page)) { in migrate_page_move_mapping()
449 set_page_private(newpage, page_private(page)); in migrate_page_move_mapping()
452 VM_BUG_ON_PAGE(PageSwapCache(page), page); in migrate_page_move_mapping()
456 dirty = PageDirty(page); in migrate_page_move_mapping()
458 ClearPageDirty(page); in migrate_page_move_mapping()
463 if (PageTransHuge(page)) { in migrate_page_move_mapping()
477 page_ref_unfreeze(page, expected_count - thp_nr_pages(page)); in migrate_page_move_mapping()
496 memcg = page_memcg(page); in migrate_page_move_mapping()
502 if (PageSwapBacked(page) && !PageSwapCache(page)) { in migrate_page_move_mapping()
524 struct page *newpage, struct page *page) in migrate_huge_page_move_mapping() argument
526 XA_STATE(xas, &mapping->i_pages, page_index(page)); in migrate_huge_page_move_mapping()
530 expected_count = 2 + page_has_private(page); in migrate_huge_page_move_mapping()
531 if (page_count(page) != expected_count || xas_load(&xas) != page) { in migrate_huge_page_move_mapping()
536 if (!page_ref_freeze(page, expected_count)) { in migrate_huge_page_move_mapping()
541 newpage->index = page->index; in migrate_huge_page_move_mapping()
542 newpage->mapping = page->mapping; in migrate_huge_page_move_mapping()
548 page_ref_unfreeze(page, expected_count - 1); in migrate_huge_page_move_mapping()
560 static void __copy_gigantic_page(struct page *dst, struct page *src, in __copy_gigantic_page()
564 struct page *dst_base = dst; in __copy_gigantic_page()
565 struct page *src_base = src; in __copy_gigantic_page()
577 static void copy_huge_page(struct page *dst, struct page *src) in copy_huge_page()
606 void migrate_page_states(struct page *newpage, struct page *page) in migrate_page_states() argument
610 if (PageError(page)) in migrate_page_states()
612 if (PageReferenced(page)) in migrate_page_states()
614 if (PageUptodate(page)) in migrate_page_states()
616 if (TestClearPageActive(page)) { in migrate_page_states()
617 VM_BUG_ON_PAGE(PageUnevictable(page), page); in migrate_page_states()
619 } else if (TestClearPageUnevictable(page)) in migrate_page_states()
621 if (PageWorkingset(page)) in migrate_page_states()
623 if (PageChecked(page)) in migrate_page_states()
625 if (PageMappedToDisk(page)) in migrate_page_states()
629 if (PageDirty(page)) in migrate_page_states()
632 if (page_is_young(page)) in migrate_page_states()
634 if (page_is_idle(page)) in migrate_page_states()
641 cpupid = page_cpupid_xchg_last(page, -1); in migrate_page_states()
644 ksm_migrate_page(newpage, page); in migrate_page_states()
649 if (PageSwapCache(page)) in migrate_page_states()
650 ClearPageSwapCache(page); in migrate_page_states()
651 ClearPagePrivate(page); in migrate_page_states()
652 set_page_private(page, 0); in migrate_page_states()
666 if (PageReadahead(page)) in migrate_page_states()
669 copy_page_owner(page, newpage); in migrate_page_states()
671 if (!PageHuge(page)) in migrate_page_states()
672 mem_cgroup_migrate(page, newpage); in migrate_page_states()
676 void migrate_page_copy(struct page *newpage, struct page *page) in migrate_page_copy() argument
678 if (PageHuge(page) || PageTransHuge(page)) in migrate_page_copy()
679 copy_huge_page(newpage, page); in migrate_page_copy()
681 copy_highpage(newpage, page); in migrate_page_copy()
683 migrate_page_states(newpage, page); in migrate_page_copy()
698 struct page *newpage, struct page *page, in migrate_page() argument
703 BUG_ON(PageWriteback(page)); /* Writeback must be complete */ in migrate_page()
705 rc = migrate_page_move_mapping(mapping, newpage, page, 0); in migrate_page()
711 migrate_page_copy(newpage, page); in migrate_page()
713 migrate_page_states(newpage, page); in migrate_page()
758 struct page *newpage, struct page *page, enum migrate_mode mode, in __buffer_migrate_page() argument
765 if (!page_has_buffers(page)) in __buffer_migrate_page()
766 return migrate_page(mapping, newpage, page, mode); in __buffer_migrate_page()
769 expected_count = expected_page_refs(mapping, page); in __buffer_migrate_page()
770 if (page_count(page) != expected_count) in __buffer_migrate_page()
773 head = page_buffers(page); in __buffer_migrate_page()
804 rc = migrate_page_move_mapping(mapping, newpage, page, 0); in __buffer_migrate_page()
808 attach_page_private(newpage, detach_page_private(page)); in __buffer_migrate_page()
818 migrate_page_copy(newpage, page); in __buffer_migrate_page()
820 migrate_page_states(newpage, page); in __buffer_migrate_page()
842 struct page *newpage, struct page *page, enum migrate_mode mode) in buffer_migrate_page() argument
844 return __buffer_migrate_page(mapping, newpage, page, mode, false); in buffer_migrate_page()
855 struct page *newpage, struct page *page, enum migrate_mode mode) in buffer_migrate_page_norefs() argument
857 return __buffer_migrate_page(mapping, newpage, page, mode, true); in buffer_migrate_page_norefs()
864 static int writeout(struct address_space *mapping, struct page *page) in writeout() argument
879 if (!clear_page_dirty_for_io(page)) in writeout()
891 remove_migration_ptes(page, page, false); in writeout()
893 rc = mapping->a_ops->writepage(page, &wbc); in writeout()
897 lock_page(page); in writeout()
906 struct page *newpage, struct page *page, enum migrate_mode mode) in fallback_migrate_page() argument
908 if (PageDirty(page)) { in fallback_migrate_page()
917 return writeout(mapping, page); in fallback_migrate_page()
924 if (page_has_private(page) && in fallback_migrate_page()
925 !try_to_release_page(page, GFP_KERNEL)) in fallback_migrate_page()
928 return migrate_page(mapping, newpage, page, mode); in fallback_migrate_page()
942 static int move_to_new_page(struct page *newpage, struct page *page, in move_to_new_page() argument
947 bool is_lru = !__PageMovable(page); in move_to_new_page()
949 VM_BUG_ON_PAGE(!PageLocked(page), page); in move_to_new_page()
952 mapping = page_mapping(page); in move_to_new_page()
956 rc = migrate_page(mapping, newpage, page, mode); in move_to_new_page()
966 page, mode); in move_to_new_page()
969 page, mode); in move_to_new_page()
975 VM_BUG_ON_PAGE(!PageIsolated(page), page); in move_to_new_page()
976 if (!PageMovable(page)) { in move_to_new_page()
978 __ClearPageIsolated(page); in move_to_new_page()
983 page, mode); in move_to_new_page()
985 !PageIsolated(page)); in move_to_new_page()
993 if (__PageMovable(page)) { in move_to_new_page()
994 VM_BUG_ON_PAGE(!PageIsolated(page), page); in move_to_new_page()
1000 __ClearPageIsolated(page); in move_to_new_page()
1008 if (!PageMappingFlags(page)) in move_to_new_page()
1009 page->mapping = NULL; in move_to_new_page()
1019 static int __unmap_and_move(struct page *page, struct page *newpage, in __unmap_and_move() argument
1025 bool is_lru = !__PageMovable(page); in __unmap_and_move()
1027 if (!trylock_page(page)) { in __unmap_and_move()
1047 lock_page(page); in __unmap_and_move()
1050 if (PageWriteback(page)) { in __unmap_and_move()
1067 wait_on_page_writeback(page); in __unmap_and_move()
1084 if (PageAnon(page) && !PageKsm(page)) in __unmap_and_move()
1085 anon_vma = page_get_anon_vma(page); in __unmap_and_move()
1099 rc = move_to_new_page(newpage, page, mode); in __unmap_and_move()
1115 if (!page->mapping) { in __unmap_and_move()
1116 VM_BUG_ON_PAGE(PageAnon(page), page); in __unmap_and_move()
1117 if (page_has_private(page)) { in __unmap_and_move()
1118 try_to_free_buffers(page); in __unmap_and_move()
1121 } else if (page_mapped(page)) { in __unmap_and_move()
1123 VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma, in __unmap_and_move()
1124 page); in __unmap_and_move()
1125 try_to_unmap(page, in __unmap_and_move()
1130 if (!page_mapped(page)) in __unmap_and_move()
1131 rc = move_to_new_page(newpage, page, mode); in __unmap_and_move()
1134 remove_migration_ptes(page, in __unmap_and_move()
1135 rc == MIGRATEPAGE_SUCCESS ? newpage : page, false); in __unmap_and_move()
1143 unlock_page(page); in __unmap_and_move()
1170 unsigned long private, struct page *page, in unmap_and_move() argument
1175 struct page *newpage = NULL; in unmap_and_move()
1177 if (!thp_migration_supported() && PageTransHuge(page)) in unmap_and_move()
1180 if (page_count(page) == 1) { in unmap_and_move()
1182 ClearPageActive(page); in unmap_and_move()
1183 ClearPageUnevictable(page); in unmap_and_move()
1184 if (unlikely(__PageMovable(page))) { in unmap_and_move()
1185 lock_page(page); in unmap_and_move()
1186 if (!PageMovable(page)) in unmap_and_move()
1187 __ClearPageIsolated(page); in unmap_and_move()
1188 unlock_page(page); in unmap_and_move()
1193 newpage = get_new_page(page, private); in unmap_and_move()
1197 rc = __unmap_and_move(page, newpage, force, mode); in unmap_and_move()
1208 list_del(&page->lru); in unmap_and_move()
1215 if (likely(!__PageMovable(page))) in unmap_and_move()
1216 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + in unmap_and_move()
1217 page_is_file_lru(page), -thp_nr_pages(page)); in unmap_and_move()
1230 put_page(page); in unmap_and_move()
1233 if (likely(!__PageMovable(page))) { in unmap_and_move()
1234 putback_lru_page(page); in unmap_and_move()
1238 lock_page(page); in unmap_and_move()
1239 if (PageMovable(page)) in unmap_and_move()
1240 putback_movable_page(page); in unmap_and_move()
1242 __ClearPageIsolated(page); in unmap_and_move()
1243 unlock_page(page); in unmap_and_move()
1244 put_page(page); in unmap_and_move()
1276 struct page *hpage, int force, in unmap_and_move_huge_page()
1281 struct page *new_hpage; in unmap_and_move_huge_page()
1429 struct page *page; in migrate_pages() local
1430 struct page *page2; in migrate_pages()
1441 list_for_each_entry_safe(page, page2, from, lru) { in migrate_pages()
1448 is_thp = PageTransHuge(page) && !PageHuge(page); in migrate_pages()
1449 nr_subpages = thp_nr_pages(page); in migrate_pages()
1452 if (PageHuge(page)) in migrate_pages()
1454 put_new_page, private, page, in migrate_pages()
1458 private, page, pass > 2, mode, in migrate_pages()
1475 lock_page(page); in migrate_pages()
1476 rc = split_huge_page_to_list(page, from); in migrate_pages()
1477 unlock_page(page); in migrate_pages()
1479 list_safe_reset_next(page, page2, lru); in migrate_pages()
1540 struct page *alloc_migration_target(struct page *page, unsigned long private) in alloc_migration_target() argument
1545 struct page *new_page = NULL; in alloc_migration_target()
1553 nid = page_to_nid(page); in alloc_migration_target()
1555 if (PageHuge(page)) { in alloc_migration_target()
1556 struct hstate *h = page_hstate(compound_head(page)); in alloc_migration_target()
1562 if (PageTransHuge(page)) { in alloc_migration_target()
1571 zidx = zone_idx(page_zone(page)); in alloc_migration_target()
1625 struct page *page; in add_page_for_migration() local
1637 page = follow_page(vma, addr, follflags); in add_page_for_migration()
1639 err = PTR_ERR(page); in add_page_for_migration()
1640 if (IS_ERR(page)) in add_page_for_migration()
1644 if (!page) in add_page_for_migration()
1648 if (page_to_nid(page) == node) in add_page_for_migration()
1652 if (page_mapcount(page) > 1 && !migrate_all) in add_page_for_migration()
1655 if (PageHuge(page)) { in add_page_for_migration()
1656 if (PageHead(page)) { in add_page_for_migration()
1657 isolate_huge_page(page, pagelist); in add_page_for_migration()
1661 struct page *head; in add_page_for_migration()
1663 head = compound_head(page); in add_page_for_migration()
1680 put_page(page); in add_page_for_migration()
1812 struct page *page; in do_pages_stat_array() local
1820 page = follow_page(vma, addr, FOLL_DUMP); in do_pages_stat_array()
1822 err = PTR_ERR(page); in do_pages_stat_array()
1823 if (IS_ERR(page)) in do_pages_stat_array()
1826 err = page ? page_to_nid(page) : -ENOENT; in do_pages_stat_array()
2009 static struct page *alloc_misplaced_dst_page(struct page *page, in alloc_misplaced_dst_page() argument
2013 struct page *newpage; in alloc_misplaced_dst_page()
2024 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) in numamigrate_isolate_page() argument
2028 VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page); in numamigrate_isolate_page()
2031 if (!migrate_balanced_pgdat(pgdat, compound_nr(page))) in numamigrate_isolate_page()
2034 if (isolate_lru_page(page)) in numamigrate_isolate_page()
2044 if (PageTransHuge(page) && page_count(page) != 3) { in numamigrate_isolate_page()
2045 putback_lru_page(page); in numamigrate_isolate_page()
2049 page_lru = page_is_file_lru(page); in numamigrate_isolate_page()
2050 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru, in numamigrate_isolate_page()
2051 thp_nr_pages(page)); in numamigrate_isolate_page()
2058 put_page(page); in numamigrate_isolate_page()
2064 struct page *page = pmd_page(pmd); in pmd_trans_migrating() local
2065 return PageLocked(page); in pmd_trans_migrating()
2073 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, in migrate_misplaced_page() argument
2085 if (page_mapcount(page) != 1 && page_is_file_lru(page) && in migrate_misplaced_page()
2093 if (page_is_file_lru(page) && PageDirty(page)) in migrate_misplaced_page()
2096 isolated = numamigrate_isolate_page(pgdat, page); in migrate_misplaced_page()
2100 list_add(&page->lru, &migratepages); in migrate_misplaced_page()
2106 list_del(&page->lru); in migrate_misplaced_page()
2107 dec_node_page_state(page, NR_ISOLATED_ANON + in migrate_misplaced_page()
2108 page_is_file_lru(page)); in migrate_misplaced_page()
2109 putback_lru_page(page); in migrate_misplaced_page()
2118 put_page(page); in migrate_misplaced_page()
2132 struct page *page, int node) in migrate_misplaced_transhuge_page() argument
2137 struct page *new_page = NULL; in migrate_misplaced_transhuge_page()
2138 int page_lru = page_is_file_lru(page); in migrate_misplaced_transhuge_page()
2148 isolated = numamigrate_isolate_page(pgdat, page); in migrate_misplaced_transhuge_page()
2156 if (PageSwapBacked(page)) in migrate_misplaced_transhuge_page()
2160 new_page->mapping = page->mapping; in migrate_misplaced_transhuge_page()
2161 new_page->index = page->index; in migrate_misplaced_transhuge_page()
2164 migrate_page_copy(new_page, page); in migrate_misplaced_transhuge_page()
2169 if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) { in migrate_misplaced_transhuge_page()
2174 SetPageActive(page); in migrate_misplaced_transhuge_page()
2176 SetPageUnevictable(page); in migrate_misplaced_transhuge_page()
2182 get_page(page); in migrate_misplaced_transhuge_page()
2183 putback_lru_page(page); in migrate_misplaced_transhuge_page()
2184 mod_node_page_state(page_pgdat(page), in migrate_misplaced_transhuge_page()
2216 page_ref_unfreeze(page, 2); in migrate_misplaced_transhuge_page()
2217 mlock_migrate_page(new_page, page); in migrate_misplaced_transhuge_page()
2218 page_remove_rmap(page, true); in migrate_misplaced_transhuge_page()
2228 unlock_page(page); in migrate_misplaced_transhuge_page()
2229 put_page(page); /* Drop the rmap reference */ in migrate_misplaced_transhuge_page()
2230 put_page(page); /* Drop the LRU isolation reference */ in migrate_misplaced_transhuge_page()
2235 mod_node_page_state(page_pgdat(page), in migrate_misplaced_transhuge_page()
2251 unlock_page(page); in migrate_misplaced_transhuge_page()
2252 put_page(page); in migrate_misplaced_transhuge_page()
2320 struct page *page; in migrate_vma_collect_pmd() local
2328 page = pmd_page(*pmdp); in migrate_vma_collect_pmd()
2329 if (is_huge_zero_page(page)) { in migrate_vma_collect_pmd()
2338 get_page(page); in migrate_vma_collect_pmd()
2340 if (unlikely(!trylock_page(page))) in migrate_vma_collect_pmd()
2343 ret = split_huge_page(page); in migrate_vma_collect_pmd()
2344 unlock_page(page); in migrate_vma_collect_pmd()
2345 put_page(page); in migrate_vma_collect_pmd()
2363 struct page *page; in migrate_vma_collect_pmd() local
2387 page = device_private_entry_to_page(entry); in migrate_vma_collect_pmd()
2390 page->pgmap->owner != migrate->pgmap_owner) in migrate_vma_collect_pmd()
2393 mpfn = migrate_pfn(page_to_pfn(page)) | in migrate_vma_collect_pmd()
2406 page = vm_normal_page(migrate->vma, addr, pte); in migrate_vma_collect_pmd()
2412 if (!page || !page->mapping || PageTransCompound(page)) { in migrate_vma_collect_pmd()
2426 get_page(page); in migrate_vma_collect_pmd()
2434 if (trylock_page(page)) { in migrate_vma_collect_pmd()
2441 entry = make_migration_entry(page, mpfn & in migrate_vma_collect_pmd()
2462 page_remove_rmap(page, false); in migrate_vma_collect_pmd()
2463 put_page(page); in migrate_vma_collect_pmd()
2525 static bool migrate_vma_check_page(struct page *page) in migrate_vma_check_page() argument
2539 if (PageCompound(page)) in migrate_vma_check_page()
2543 if (is_zone_device_page(page)) { in migrate_vma_check_page()
2557 return is_device_private_page(page); in migrate_vma_check_page()
2561 if (page_mapping(page)) in migrate_vma_check_page()
2562 extra += 1 + page_has_private(page); in migrate_vma_check_page()
2564 if ((page_count(page) - extra) > page_mapcount(page)) in migrate_vma_check_page()
2589 struct page *page = migrate_pfn_to_page(migrate->src[i]); in migrate_vma_prepare() local
2592 if (!page) in migrate_vma_prepare()
2604 if (!trylock_page(page)) { in migrate_vma_prepare()
2607 put_page(page); in migrate_vma_prepare()
2615 if (!is_zone_device_page(page)) { in migrate_vma_prepare()
2616 if (!PageLRU(page) && allow_drain) { in migrate_vma_prepare()
2622 if (isolate_lru_page(page)) { in migrate_vma_prepare()
2629 unlock_page(page); in migrate_vma_prepare()
2631 put_page(page); in migrate_vma_prepare()
2637 put_page(page); in migrate_vma_prepare()
2640 if (!migrate_vma_check_page(page)) { in migrate_vma_prepare()
2646 if (!is_zone_device_page(page)) { in migrate_vma_prepare()
2647 get_page(page); in migrate_vma_prepare()
2648 putback_lru_page(page); in migrate_vma_prepare()
2652 unlock_page(page); in migrate_vma_prepare()
2655 if (!is_zone_device_page(page)) in migrate_vma_prepare()
2656 putback_lru_page(page); in migrate_vma_prepare()
2658 put_page(page); in migrate_vma_prepare()
2664 struct page *page = migrate_pfn_to_page(migrate->src[i]); in migrate_vma_prepare() local
2666 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE)) in migrate_vma_prepare()
2669 remove_migration_pte(page, migrate->vma, addr, page); in migrate_vma_prepare()
2672 unlock_page(page); in migrate_vma_prepare()
2673 put_page(page); in migrate_vma_prepare()
2697 struct page *page = migrate_pfn_to_page(migrate->src[i]); in migrate_vma_unmap() local
2699 if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE)) in migrate_vma_unmap()
2702 if (page_mapped(page)) { in migrate_vma_unmap()
2703 try_to_unmap(page, flags); in migrate_vma_unmap()
2704 if (page_mapped(page)) in migrate_vma_unmap()
2708 if (migrate_vma_check_page(page)) in migrate_vma_unmap()
2718 struct page *page = migrate_pfn_to_page(migrate->src[i]); in migrate_vma_unmap() local
2720 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE)) in migrate_vma_unmap()
2723 remove_migration_ptes(page, page, false); in migrate_vma_unmap()
2726 unlock_page(page); in migrate_vma_unmap()
2729 if (is_zone_device_page(page)) in migrate_vma_unmap()
2730 put_page(page); in migrate_vma_unmap()
2732 putback_lru_page(page); in migrate_vma_unmap()
2850 struct page *page, in migrate_vma_insert_page() argument
2902 if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL)) in migrate_vma_insert_page()
2910 __SetPageUptodate(page); in migrate_vma_insert_page()
2912 if (is_zone_device_page(page)) { in migrate_vma_insert_page()
2913 if (is_device_private_page(page)) { in migrate_vma_insert_page()
2916 swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE); in migrate_vma_insert_page()
2920 entry = mk_pte(page, vma->vm_page_prot); in migrate_vma_insert_page()
2947 page_add_new_anon_rmap(page, vma, addr, false); in migrate_vma_insert_page()
2948 if (!is_zone_device_page(page)) in migrate_vma_insert_page()
2949 lru_cache_add_inactive_or_unevictable(page, vma); in migrate_vma_insert_page()
2950 get_page(page); in migrate_vma_insert_page()
2990 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]); in migrate_vma_pages()
2991 struct page *page = migrate_pfn_to_page(migrate->src[i]); in migrate_vma_pages() local
3000 if (!page) { in migrate_vma_pages()
3019 mapping = page_mapping(page); in migrate_vma_pages()
3041 r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY); in migrate_vma_pages()
3073 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]); in migrate_vma_finalize()
3074 struct page *page = migrate_pfn_to_page(migrate->src[i]); in migrate_vma_finalize() local
3076 if (!page) { in migrate_vma_finalize()
3089 newpage = page; in migrate_vma_finalize()
3092 remove_migration_ptes(page, newpage, false); in migrate_vma_finalize()
3093 unlock_page(page); in migrate_vma_finalize()
3095 if (is_zone_device_page(page)) in migrate_vma_finalize()
3096 put_page(page); in migrate_vma_finalize()
3098 putback_lru_page(page); in migrate_vma_finalize()
3100 if (newpage != page) { in migrate_vma_finalize()