Lines Matching refs:page
60 static void __page_cache_release(struct page *page) in __page_cache_release() argument
62 if (PageLRU(page)) { in __page_cache_release()
63 pg_data_t *pgdat = page_pgdat(page); in __page_cache_release()
68 lruvec = mem_cgroup_page_lruvec(page, pgdat); in __page_cache_release()
69 VM_BUG_ON_PAGE(!PageLRU(page), page); in __page_cache_release()
70 __ClearPageLRU(page); in __page_cache_release()
71 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in __page_cache_release()
74 __ClearPageWaiters(page); in __page_cache_release()
77 static void __put_single_page(struct page *page) in __put_single_page() argument
79 __page_cache_release(page); in __put_single_page()
80 mem_cgroup_uncharge(page); in __put_single_page()
81 free_unref_page(page); in __put_single_page()
84 static void __put_compound_page(struct page *page) in __put_compound_page() argument
94 if (!PageHuge(page)) in __put_compound_page()
95 __page_cache_release(page); in __put_compound_page()
96 dtor = get_compound_page_dtor(page); in __put_compound_page()
97 (*dtor)(page); in __put_compound_page()
100 void __put_page(struct page *page) in __put_page() argument
102 if (is_zone_device_page(page)) { in __put_page()
103 put_dev_pagemap(page->pgmap); in __put_page()
112 if (unlikely(PageCompound(page))) in __put_page()
113 __put_compound_page(page); in __put_page()
115 __put_single_page(page); in __put_page()
129 struct page *victim; in put_pages_list()
152 struct page **pages) in get_kernel_pages()
179 int get_kernel_page(unsigned long start, int write, struct page **pages) in get_kernel_page()
191 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), in pagevec_lru_move_fn() argument
200 struct page *page = pvec->pages[i]; in pagevec_lru_move_fn() local
201 struct pglist_data *pagepgdat = page_pgdat(page); in pagevec_lru_move_fn()
210 lruvec = mem_cgroup_page_lruvec(page, pgdat); in pagevec_lru_move_fn()
211 (*move_fn)(page, lruvec, arg); in pagevec_lru_move_fn()
219 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, in pagevec_move_tail_fn() argument
224 if (PageLRU(page) && !PageUnevictable(page)) { in pagevec_move_tail_fn()
225 del_page_from_lru_list(page, lruvec, page_lru(page)); in pagevec_move_tail_fn()
226 ClearPageActive(page); in pagevec_move_tail_fn()
227 add_page_to_lru_list_tail(page, lruvec, page_lru(page)); in pagevec_move_tail_fn()
249 void rotate_reclaimable_page(struct page *page) in rotate_reclaimable_page() argument
251 if (!PageLocked(page) && !PageDirty(page) && in rotate_reclaimable_page()
252 !PageUnevictable(page) && PageLRU(page)) { in rotate_reclaimable_page()
256 get_page(page); in rotate_reclaimable_page()
259 if (!pagevec_add(pvec, page) || PageCompound(page)) in rotate_reclaimable_page()
275 static void __activate_page(struct page *page, struct lruvec *lruvec, in __activate_page() argument
278 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in __activate_page()
279 int file = page_is_file_cache(page); in __activate_page()
280 int lru = page_lru_base_type(page); in __activate_page()
282 del_page_from_lru_list(page, lruvec, lru); in __activate_page()
283 SetPageActive(page); in __activate_page()
285 add_page_to_lru_list(page, lruvec, lru); in __activate_page()
286 trace_mm_lru_activate(page); in __activate_page()
307 void activate_page(struct page *page) in activate_page() argument
309 page = compound_head(page); in activate_page()
310 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in activate_page()
313 get_page(page); in activate_page()
314 if (!pagevec_add(pvec, page) || PageCompound(page)) in activate_page()
325 void activate_page(struct page *page) in activate_page() argument
327 pg_data_t *pgdat = page_pgdat(page); in activate_page()
329 page = compound_head(page); in activate_page()
331 __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL); in activate_page()
336 static void __lru_cache_activate_page(struct page *page) in __lru_cache_activate_page() argument
352 struct page *pagevec_page = pvec->pages[i]; in __lru_cache_activate_page()
354 if (pagevec_page == page) { in __lru_cache_activate_page()
355 SetPageActive(page); in __lru_cache_activate_page()
373 void mark_page_accessed(struct page *page) in mark_page_accessed() argument
375 page = compound_head(page); in mark_page_accessed()
376 if (!PageActive(page) && !PageUnevictable(page) && in mark_page_accessed()
377 PageReferenced(page)) { in mark_page_accessed()
385 if (PageLRU(page)) in mark_page_accessed()
386 activate_page(page); in mark_page_accessed()
388 __lru_cache_activate_page(page); in mark_page_accessed()
389 ClearPageReferenced(page); in mark_page_accessed()
390 if (page_is_file_cache(page)) in mark_page_accessed()
391 workingset_activation(page); in mark_page_accessed()
392 } else if (!PageReferenced(page)) { in mark_page_accessed()
393 SetPageReferenced(page); in mark_page_accessed()
395 if (page_is_idle(page)) in mark_page_accessed()
396 clear_page_idle(page); in mark_page_accessed()
400 static void __lru_cache_add(struct page *page) in __lru_cache_add() argument
404 get_page(page); in __lru_cache_add()
405 if (!pagevec_add(pvec, page) || PageCompound(page)) in __lru_cache_add()
414 void lru_cache_add_anon(struct page *page) in lru_cache_add_anon() argument
416 if (PageActive(page)) in lru_cache_add_anon()
417 ClearPageActive(page); in lru_cache_add_anon()
418 __lru_cache_add(page); in lru_cache_add_anon()
421 void lru_cache_add_file(struct page *page) in lru_cache_add_file() argument
423 if (PageActive(page)) in lru_cache_add_file()
424 ClearPageActive(page); in lru_cache_add_file()
425 __lru_cache_add(page); in lru_cache_add_file()
438 void lru_cache_add(struct page *page) in lru_cache_add() argument
440 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); in lru_cache_add()
441 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add()
442 __lru_cache_add(page); in lru_cache_add()
455 void lru_cache_add_active_or_unevictable(struct page *page, in lru_cache_add_active_or_unevictable() argument
458 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add_active_or_unevictable()
461 SetPageActive(page); in lru_cache_add_active_or_unevictable()
462 else if (!TestSetPageMlocked(page)) { in lru_cache_add_active_or_unevictable()
468 __mod_zone_page_state(page_zone(page), NR_MLOCK, in lru_cache_add_active_or_unevictable()
469 hpage_nr_pages(page)); in lru_cache_add_active_or_unevictable()
472 lru_cache_add(page); in lru_cache_add_active_or_unevictable()
496 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, in lru_deactivate_file_fn() argument
502 if (!PageLRU(page)) in lru_deactivate_file_fn()
505 if (PageUnevictable(page)) in lru_deactivate_file_fn()
509 if (page_mapped(page)) in lru_deactivate_file_fn()
512 active = PageActive(page); in lru_deactivate_file_fn()
513 file = page_is_file_cache(page); in lru_deactivate_file_fn()
514 lru = page_lru_base_type(page); in lru_deactivate_file_fn()
516 del_page_from_lru_list(page, lruvec, lru + active); in lru_deactivate_file_fn()
517 ClearPageActive(page); in lru_deactivate_file_fn()
518 ClearPageReferenced(page); in lru_deactivate_file_fn()
520 if (PageWriteback(page) || PageDirty(page)) { in lru_deactivate_file_fn()
526 add_page_to_lru_list(page, lruvec, lru); in lru_deactivate_file_fn()
527 SetPageReclaim(page); in lru_deactivate_file_fn()
533 add_page_to_lru_list_tail(page, lruvec, lru); in lru_deactivate_file_fn()
542 static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, in lru_deactivate_fn() argument
545 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { in lru_deactivate_fn()
546 int file = page_is_file_cache(page); in lru_deactivate_fn()
547 int lru = page_lru_base_type(page); in lru_deactivate_fn()
549 del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); in lru_deactivate_fn()
550 ClearPageActive(page); in lru_deactivate_fn()
551 ClearPageReferenced(page); in lru_deactivate_fn()
552 add_page_to_lru_list(page, lruvec, lru); in lru_deactivate_fn()
554 __count_vm_events(PGDEACTIVATE, hpage_nr_pages(page)); in lru_deactivate_fn()
559 static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, in lru_lazyfree_fn() argument
562 if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && in lru_lazyfree_fn()
563 !PageSwapCache(page) && !PageUnevictable(page)) { in lru_lazyfree_fn()
564 bool active = PageActive(page); in lru_lazyfree_fn()
566 del_page_from_lru_list(page, lruvec, in lru_lazyfree_fn()
568 ClearPageActive(page); in lru_lazyfree_fn()
569 ClearPageReferenced(page); in lru_lazyfree_fn()
575 ClearPageSwapBacked(page); in lru_lazyfree_fn()
576 add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); in lru_lazyfree_fn()
578 __count_vm_events(PGLAZYFREE, hpage_nr_pages(page)); in lru_lazyfree_fn()
579 count_memcg_page_event(page, PGLAZYFREE); in lru_lazyfree_fn()
629 void deactivate_file_page(struct page *page) in deactivate_file_page() argument
635 if (PageUnevictable(page)) in deactivate_file_page()
638 if (likely(get_page_unless_zero(page))) { in deactivate_file_page()
641 if (!pagevec_add(pvec, page) || PageCompound(page)) in deactivate_file_page()
655 void deactivate_page(struct page *page) in deactivate_page() argument
657 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { in deactivate_page()
660 get_page(page); in deactivate_page()
661 if (!pagevec_add(pvec, page) || PageCompound(page)) in deactivate_page()
674 void mark_page_lazyfree(struct page *page) in mark_page_lazyfree() argument
676 if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && in mark_page_lazyfree()
677 !PageSwapCache(page) && !PageUnevictable(page)) { in mark_page_lazyfree()
680 get_page(page); in mark_page_lazyfree()
681 if (!pagevec_add(pvec, page) || PageCompound(page)) in mark_page_lazyfree()
760 void release_pages(struct page **pages, int nr) in release_pages()
770 struct page *page = pages[i]; in release_pages() local
782 if (is_huge_zero_page(page)) in release_pages()
785 if (is_zone_device_page(page)) { in release_pages()
797 if (put_devmap_managed_page(page)) in release_pages()
801 page = compound_head(page); in release_pages()
802 if (!put_page_testzero(page)) in release_pages()
805 if (PageCompound(page)) { in release_pages()
810 __put_compound_page(page); in release_pages()
814 if (PageLRU(page)) { in release_pages()
815 struct pglist_data *pgdat = page_pgdat(page); in release_pages()
826 lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); in release_pages()
827 VM_BUG_ON_PAGE(!PageLRU(page), page); in release_pages()
828 __ClearPageLRU(page); in release_pages()
829 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in release_pages()
833 __ClearPageActive(page); in release_pages()
834 __ClearPageWaiters(page); in release_pages()
836 list_add(&page->lru, &pages_to_free); in release_pages()
869 void lru_add_page_tail(struct page *page, struct page *page_tail, in lru_add_page_tail() argument
874 VM_BUG_ON_PAGE(!PageHead(page), page); in lru_add_page_tail()
875 VM_BUG_ON_PAGE(PageCompound(page_tail), page); in lru_add_page_tail()
876 VM_BUG_ON_PAGE(PageLRU(page_tail), page); in lru_add_page_tail()
882 if (likely(PageLRU(page))) in lru_add_page_tail()
883 list_add_tail(&page_tail->lru, &page->lru); in lru_add_page_tail()
900 if (!PageUnevictable(page)) in lru_add_page_tail()
905 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, in __pagevec_lru_add_fn() argument
909 int was_unevictable = TestClearPageUnevictable(page); in __pagevec_lru_add_fn()
911 VM_BUG_ON_PAGE(PageLRU(page), page); in __pagevec_lru_add_fn()
913 SetPageLRU(page); in __pagevec_lru_add_fn()
942 if (page_evictable(page)) { in __pagevec_lru_add_fn()
943 lru = page_lru(page); in __pagevec_lru_add_fn()
944 update_page_reclaim_stat(lruvec, page_is_file_cache(page), in __pagevec_lru_add_fn()
945 PageActive(page)); in __pagevec_lru_add_fn()
950 ClearPageActive(page); in __pagevec_lru_add_fn()
951 SetPageUnevictable(page); in __pagevec_lru_add_fn()
956 add_page_to_lru_list(page, lruvec, lru); in __pagevec_lru_add_fn()
957 trace_mm_lru_insertion(page, lru); in __pagevec_lru_add_fn()
1014 struct page *page = pvec->pages[i]; in pagevec_remove_exceptionals() local
1015 if (!xa_is_value(page)) in pagevec_remove_exceptionals()
1016 pvec->pages[j++] = page; in pagevec_remove_exceptionals()