Lines Matching refs:page
478 struct anon_vma *page_get_anon_vma(struct page *page) in page_get_anon_vma() argument
484 anon_mapping = (unsigned long)READ_ONCE(page->mapping); in page_get_anon_vma()
487 if (!page_mapped(page)) in page_get_anon_vma()
503 if (!page_mapped(page)) { in page_get_anon_vma()
521 struct anon_vma *page_lock_anon_vma_read(struct page *page) in page_lock_anon_vma_read() argument
528 anon_mapping = (unsigned long)READ_ONCE(page->mapping); in page_lock_anon_vma_read()
531 if (!page_mapped(page)) in page_lock_anon_vma_read()
542 if (!page_mapped(page)) { in page_lock_anon_vma_read()
555 if (!page_mapped(page)) { in page_lock_anon_vma_read()
701 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) in page_address_in_vma() argument
704 if (PageAnon(page)) { in page_address_in_vma()
705 struct anon_vma *page__anon_vma = page_anon_vma(page); in page_address_in_vma()
713 } else if (page->mapping) { in page_address_in_vma()
714 if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping) in page_address_in_vma()
718 address = __vma_address(page, vma); in page_address_in_vma()
767 static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, in page_referenced_one() argument
772 .page = page, in page_referenced_one()
814 clear_page_idle(page); in page_referenced_one()
815 if (test_and_clear_page_young(page)) in page_referenced_one()
850 int page_referenced(struct page *page, in page_referenced() argument
857 .mapcount = total_mapcount(page), in page_referenced()
870 if (!page_rmapping(page)) in page_referenced()
873 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { in page_referenced()
874 we_locked = trylock_page(page); in page_referenced()
888 rmap_walk(page, &rwc); in page_referenced()
892 unlock_page(page); in page_referenced()
897 static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, in page_mkclean_one() argument
901 .page = page, in page_mkclean_one()
915 min(vma->vm_end, address + page_size(page))); in page_mkclean_one()
943 flush_cache_page(vma, address, page_to_pfn(page)); in page_mkclean_one()
979 int page_mkclean(struct page *page) in page_mkclean() argument
989 BUG_ON(!PageLocked(page)); in page_mkclean()
991 if (!page_mapped(page)) in page_mkclean()
994 mapping = page_mapping(page); in page_mkclean()
998 rmap_walk(page, &rwc); in page_mkclean()
1014 void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) in page_move_anon_rmap() argument
1018 page = compound_head(page); in page_move_anon_rmap()
1020 VM_BUG_ON_PAGE(!PageLocked(page), page); in page_move_anon_rmap()
1029 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); in page_move_anon_rmap()
1039 static void __page_set_anon_rmap(struct page *page, in __page_set_anon_rmap() argument
1046 if (PageAnon(page)) in __page_set_anon_rmap()
1058 page->mapping = (struct address_space *) anon_vma; in __page_set_anon_rmap()
1059 page->index = linear_page_index(vma, address); in __page_set_anon_rmap()
1068 static void __page_check_anon_rmap(struct page *page, in __page_check_anon_rmap() argument
1083 VM_BUG_ON_PAGE(page_anon_vma(page)->root != vma->anon_vma->root, page); in __page_check_anon_rmap()
1084 VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address), in __page_check_anon_rmap()
1085 page); in __page_check_anon_rmap()
1100 void page_add_anon_rmap(struct page *page, in page_add_anon_rmap() argument
1103 do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0); in page_add_anon_rmap()
1111 void do_page_add_anon_rmap(struct page *page, in do_page_add_anon_rmap() argument
1117 if (unlikely(PageKsm(page))) in do_page_add_anon_rmap()
1118 lock_page_memcg(page); in do_page_add_anon_rmap()
1120 VM_BUG_ON_PAGE(!PageLocked(page), page); in do_page_add_anon_rmap()
1124 VM_BUG_ON_PAGE(!PageLocked(page), page); in do_page_add_anon_rmap()
1125 VM_BUG_ON_PAGE(!PageTransHuge(page), page); in do_page_add_anon_rmap()
1126 mapcount = compound_mapcount_ptr(page); in do_page_add_anon_rmap()
1129 first = atomic_inc_and_test(&page->_mapcount); in do_page_add_anon_rmap()
1133 int nr = compound ? thp_nr_pages(page) : 1; in do_page_add_anon_rmap()
1141 __inc_lruvec_page_state(page, NR_ANON_THPS); in do_page_add_anon_rmap()
1142 __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); in do_page_add_anon_rmap()
1145 if (unlikely(PageKsm(page))) { in do_page_add_anon_rmap()
1146 unlock_page_memcg(page); in do_page_add_anon_rmap()
1152 __page_set_anon_rmap(page, vma, address, in do_page_add_anon_rmap()
1155 __page_check_anon_rmap(page, vma, address); in do_page_add_anon_rmap()
1169 void page_add_new_anon_rmap(struct page *page, in page_add_new_anon_rmap() argument
1172 int nr = compound ? thp_nr_pages(page) : 1; in page_add_new_anon_rmap()
1175 __SetPageSwapBacked(page); in page_add_new_anon_rmap()
1177 VM_BUG_ON_PAGE(!PageTransHuge(page), page); in page_add_new_anon_rmap()
1179 atomic_set(compound_mapcount_ptr(page), 0); in page_add_new_anon_rmap()
1180 if (hpage_pincount_available(page)) in page_add_new_anon_rmap()
1181 atomic_set(compound_pincount_ptr(page), 0); in page_add_new_anon_rmap()
1183 __inc_lruvec_page_state(page, NR_ANON_THPS); in page_add_new_anon_rmap()
1186 VM_BUG_ON_PAGE(PageTransCompound(page), page); in page_add_new_anon_rmap()
1188 atomic_set(&page->_mapcount, 0); in page_add_new_anon_rmap()
1190 __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); in page_add_new_anon_rmap()
1191 __page_set_anon_rmap(page, vma, address, 1); in page_add_new_anon_rmap()
1201 void page_add_file_rmap(struct page *page, bool compound) in page_add_file_rmap() argument
1205 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); in page_add_file_rmap()
1206 lock_page_memcg(page); in page_add_file_rmap()
1207 if (compound && PageTransHuge(page)) { in page_add_file_rmap()
1208 for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { in page_add_file_rmap()
1209 if (atomic_inc_and_test(&page[i]._mapcount)) in page_add_file_rmap()
1212 if (!atomic_inc_and_test(compound_mapcount_ptr(page))) in page_add_file_rmap()
1214 if (PageSwapBacked(page)) in page_add_file_rmap()
1215 __inc_node_page_state(page, NR_SHMEM_PMDMAPPED); in page_add_file_rmap()
1217 __inc_node_page_state(page, NR_FILE_PMDMAPPED); in page_add_file_rmap()
1219 if (PageTransCompound(page) && page_mapping(page)) { in page_add_file_rmap()
1220 VM_WARN_ON_ONCE(!PageLocked(page)); in page_add_file_rmap()
1222 SetPageDoubleMap(compound_head(page)); in page_add_file_rmap()
1223 if (PageMlocked(page)) in page_add_file_rmap()
1224 clear_page_mlock(compound_head(page)); in page_add_file_rmap()
1226 if (!atomic_inc_and_test(&page->_mapcount)) in page_add_file_rmap()
1229 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr); in page_add_file_rmap()
1231 unlock_page_memcg(page); in page_add_file_rmap()
1234 static void page_remove_file_rmap(struct page *page, bool compound) in page_remove_file_rmap() argument
1238 VM_BUG_ON_PAGE(compound && !PageHead(page), page); in page_remove_file_rmap()
1241 if (unlikely(PageHuge(page))) { in page_remove_file_rmap()
1243 atomic_dec(compound_mapcount_ptr(page)); in page_remove_file_rmap()
1248 if (compound && PageTransHuge(page)) { in page_remove_file_rmap()
1249 for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { in page_remove_file_rmap()
1250 if (atomic_add_negative(-1, &page[i]._mapcount)) in page_remove_file_rmap()
1253 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) in page_remove_file_rmap()
1255 if (PageSwapBacked(page)) in page_remove_file_rmap()
1256 __dec_node_page_state(page, NR_SHMEM_PMDMAPPED); in page_remove_file_rmap()
1258 __dec_node_page_state(page, NR_FILE_PMDMAPPED); in page_remove_file_rmap()
1260 if (!atomic_add_negative(-1, &page->_mapcount)) in page_remove_file_rmap()
1269 __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr); in page_remove_file_rmap()
1271 if (unlikely(PageMlocked(page))) in page_remove_file_rmap()
1272 clear_page_mlock(page); in page_remove_file_rmap()
1275 static void page_remove_anon_compound_rmap(struct page *page) in page_remove_anon_compound_rmap() argument
1279 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) in page_remove_anon_compound_rmap()
1283 if (unlikely(PageHuge(page))) in page_remove_anon_compound_rmap()
1289 __dec_lruvec_page_state(page, NR_ANON_THPS); in page_remove_anon_compound_rmap()
1291 if (TestClearPageDoubleMap(page)) { in page_remove_anon_compound_rmap()
1296 for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { in page_remove_anon_compound_rmap()
1297 if (atomic_add_negative(-1, &page[i]._mapcount)) in page_remove_anon_compound_rmap()
1306 if (nr && nr < thp_nr_pages(page)) in page_remove_anon_compound_rmap()
1307 deferred_split_huge_page(page); in page_remove_anon_compound_rmap()
1309 nr = thp_nr_pages(page); in page_remove_anon_compound_rmap()
1312 if (unlikely(PageMlocked(page))) in page_remove_anon_compound_rmap()
1313 clear_page_mlock(page); in page_remove_anon_compound_rmap()
1316 __mod_lruvec_page_state(page, NR_ANON_MAPPED, -nr); in page_remove_anon_compound_rmap()
1326 void page_remove_rmap(struct page *page, bool compound) in page_remove_rmap() argument
1328 lock_page_memcg(page); in page_remove_rmap()
1330 if (!PageAnon(page)) { in page_remove_rmap()
1331 page_remove_file_rmap(page, compound); in page_remove_rmap()
1336 page_remove_anon_compound_rmap(page); in page_remove_rmap()
1341 if (!atomic_add_negative(-1, &page->_mapcount)) in page_remove_rmap()
1349 __dec_lruvec_page_state(page, NR_ANON_MAPPED); in page_remove_rmap()
1351 if (unlikely(PageMlocked(page))) in page_remove_rmap()
1352 clear_page_mlock(page); in page_remove_rmap()
1354 if (PageTransCompound(page)) in page_remove_rmap()
1355 deferred_split_huge_page(compound_head(page)); in page_remove_rmap()
1367 unlock_page_memcg(page); in page_remove_rmap()
1373 static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, in try_to_unmap_one() argument
1378 .page = page, in try_to_unmap_one()
1383 struct page *subpage; in try_to_unmap_one()
1393 is_zone_device_page(page) && !is_device_private_page(page)) in try_to_unmap_one()
1398 flags & TTU_SPLIT_FREEZE, page); in try_to_unmap_one()
1411 min(vma->vm_end, address + page_size(page))); in try_to_unmap_one()
1412 if (PageHuge(page)) { in try_to_unmap_one()
1426 VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); in try_to_unmap_one()
1428 set_pmd_migration_entry(&pvmw, page); in try_to_unmap_one()
1441 if (!PageTransCompound(page)) { in try_to_unmap_one()
1446 mlock_vma_page(page); in try_to_unmap_one()
1457 VM_BUG_ON_PAGE(!pvmw.pte, page); in try_to_unmap_one()
1459 subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte); in try_to_unmap_one()
1462 if (PageHuge(page) && !PageAnon(page)) { in try_to_unmap_one()
1498 is_zone_device_page(page)) { in try_to_unmap_one()
1509 entry = make_migration_entry(page, 0); in try_to_unmap_one()
1532 subpage = page; in try_to_unmap_one()
1565 set_page_dirty(page); in try_to_unmap_one()
1570 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { in try_to_unmap_one()
1572 if (PageHuge(page)) { in try_to_unmap_one()
1573 hugetlb_count_sub(compound_nr(page), mm); in try_to_unmap_one()
1578 dec_mm_counter(mm, mm_counter(page)); in try_to_unmap_one()
1593 dec_mm_counter(mm, mm_counter(page)); in try_to_unmap_one()
1626 } else if (PageAnon(page)) { in try_to_unmap_one()
1633 if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) { in try_to_unmap_one()
1644 if (!PageSwapBacked(page)) { in try_to_unmap_one()
1645 if (!PageDirty(page)) { in try_to_unmap_one()
1658 SetPageSwapBacked(page); in try_to_unmap_one()
1704 dec_mm_counter(mm, mm_counter_file(page)); in try_to_unmap_one()
1714 page_remove_rmap(subpage, PageHuge(page)); in try_to_unmap_one()
1715 put_page(page); in try_to_unmap_one()
1728 static int page_mapcount_is_zero(struct page *page) in page_mapcount_is_zero() argument
1730 return !total_mapcount(page); in page_mapcount_is_zero()
1743 bool try_to_unmap(struct page *page, enum ttu_flags flags) in try_to_unmap() argument
1761 && !PageKsm(page) && PageAnon(page)) in try_to_unmap()
1765 rmap_walk_locked(page, &rwc); in try_to_unmap()
1767 rmap_walk(page, &rwc); in try_to_unmap()
1769 return !page_mapcount(page) ? true : false; in try_to_unmap()
1772 static int page_not_mapped(struct page *page) in page_not_mapped() argument
1774 return !page_mapped(page); in page_not_mapped()
1786 void try_to_munlock(struct page *page) in try_to_munlock() argument
1796 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page); in try_to_munlock()
1797 VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page); in try_to_munlock()
1799 rmap_walk(page, &rwc); in try_to_munlock()
1811 static struct anon_vma *rmap_walk_anon_lock(struct page *page, in rmap_walk_anon_lock() argument
1817 return rwc->anon_lock(page); in rmap_walk_anon_lock()
1825 anon_vma = page_anon_vma(page); in rmap_walk_anon_lock()
1847 static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, in rmap_walk_anon() argument
1855 anon_vma = page_anon_vma(page); in rmap_walk_anon()
1857 VM_BUG_ON_PAGE(!anon_vma, page); in rmap_walk_anon()
1859 anon_vma = rmap_walk_anon_lock(page, rwc); in rmap_walk_anon()
1864 pgoff_start = page_to_pgoff(page); in rmap_walk_anon()
1865 pgoff_end = pgoff_start + thp_nr_pages(page) - 1; in rmap_walk_anon()
1869 unsigned long address = vma_address(page, vma); in rmap_walk_anon()
1876 if (!rwc->rmap_one(page, vma, address, rwc->arg)) in rmap_walk_anon()
1878 if (rwc->done && rwc->done(page)) in rmap_walk_anon()
1899 static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, in rmap_walk_file() argument
1902 struct address_space *mapping = page_mapping(page); in rmap_walk_file()
1912 VM_BUG_ON_PAGE(!PageLocked(page), page); in rmap_walk_file()
1917 pgoff_start = page_to_pgoff(page); in rmap_walk_file()
1918 pgoff_end = pgoff_start + thp_nr_pages(page) - 1; in rmap_walk_file()
1923 unsigned long address = vma_address(page, vma); in rmap_walk_file()
1930 if (!rwc->rmap_one(page, vma, address, rwc->arg)) in rmap_walk_file()
1932 if (rwc->done && rwc->done(page)) in rmap_walk_file()
1941 void rmap_walk(struct page *page, struct rmap_walk_control *rwc) in rmap_walk() argument
1943 if (unlikely(PageKsm(page))) in rmap_walk()
1944 rmap_walk_ksm(page, rwc); in rmap_walk()
1945 else if (PageAnon(page)) in rmap_walk()
1946 rmap_walk_anon(page, rwc, false); in rmap_walk()
1948 rmap_walk_file(page, rwc, false); in rmap_walk()
1952 void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc) in rmap_walk_locked() argument
1955 VM_BUG_ON_PAGE(PageKsm(page), page); in rmap_walk_locked()
1956 if (PageAnon(page)) in rmap_walk_locked()
1957 rmap_walk_anon(page, rwc, true); in rmap_walk_locked()
1959 rmap_walk_file(page, rwc, true); in rmap_walk_locked()
1968 void hugepage_add_anon_rmap(struct page *page, in hugepage_add_anon_rmap() argument
1974 BUG_ON(!PageLocked(page)); in hugepage_add_anon_rmap()
1977 first = atomic_inc_and_test(compound_mapcount_ptr(page)); in hugepage_add_anon_rmap()
1979 __page_set_anon_rmap(page, vma, address, 0); in hugepage_add_anon_rmap()
1982 void hugepage_add_new_anon_rmap(struct page *page, in hugepage_add_new_anon_rmap() argument
1986 atomic_set(compound_mapcount_ptr(page), 0); in hugepage_add_new_anon_rmap()
1987 if (hpage_pincount_available(page)) in hugepage_add_new_anon_rmap()
1988 atomic_set(compound_pincount_ptr(page), 0); in hugepage_add_new_anon_rmap()
1990 __page_set_anon_rmap(page, vma, address, 1); in hugepage_add_new_anon_rmap()