79 static void __page_cache_release(struct page *page)  in __page_cache_release()  argument
81 	if (PageLRU(page)) {  in __page_cache_release()
82 		pg_data_t *pgdat = page_pgdat(page);  in __page_cache_release()
87 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in __page_cache_release()
88 		VM_BUG_ON_PAGE(!PageLRU(page), page);  in __page_cache_release()
89 		__ClearPageLRU(page);  in __page_cache_release()
90 		del_page_from_lru_list(page, lruvec, page_off_lru(page));  in __page_cache_release()
93 	__ClearPageWaiters(page);  in __page_cache_release()
96 static void __put_single_page(struct page *page)  in __put_single_page()  argument
98 	__page_cache_release(page);  in __put_single_page()
99 	mem_cgroup_uncharge(page);  in __put_single_page()
100 	free_unref_page(page);  in __put_single_page()
103 static void __put_compound_page(struct page *page)  in __put_compound_page()  argument
111 	if (!PageHuge(page))  in __put_compound_page()
112 		__page_cache_release(page);  in __put_compound_page()
113 	destroy_compound_page(page);  in __put_compound_page()
116 void __put_page(struct page *page)  in __put_page()  argument
118 	if (is_zone_device_page(page)) {  in __put_page()
119 		put_dev_pagemap(page->pgmap);  in __put_page()
128 	if (unlikely(PageCompound(page)))  in __put_page()
129 		__put_compound_page(page);  in __put_page()
131 		__put_single_page(page);  in __put_page()
145 		struct page *victim;  in put_pages_list()
168 		struct page **pages)  in get_kernel_pages()
195 int get_kernel_page(unsigned long start, int write, struct page **pages)  in get_kernel_page()
207 	void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),  in pagevec_lru_move_fn()  argument
216 		struct page *page = pvec->pages[i];  in pagevec_lru_move_fn()  local
217 		struct pglist_data *pagepgdat = page_pgdat(page);  in pagevec_lru_move_fn()
226 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in pagevec_lru_move_fn()
227 		(*move_fn)(page, lruvec, arg);  in pagevec_lru_move_fn()
235 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,  in pagevec_move_tail_fn()  argument
240 	if (PageLRU(page) && !PageUnevictable(page)) {  in pagevec_move_tail_fn()
241 		del_page_from_lru_list(page, lruvec, page_lru(page));  in pagevec_move_tail_fn()
242 		ClearPageActive(page);  in pagevec_move_tail_fn()
243 		add_page_to_lru_list_tail(page, lruvec, page_lru(page));  in pagevec_move_tail_fn()
244 		(*pgmoved) += thp_nr_pages(page);  in pagevec_move_tail_fn()
265 void rotate_reclaimable_page(struct page *page)  in rotate_reclaimable_page()  argument
267 	if (!PageLocked(page) && !PageDirty(page) &&  in rotate_reclaimable_page()
268 	    !PageUnevictable(page) && PageLRU(page)) {  in rotate_reclaimable_page()
272 		get_page(page);  in rotate_reclaimable_page()
275 		if (!pagevec_add(pvec, page) || PageCompound(page))  in rotate_reclaimable_page()
312 void lru_note_cost_page(struct page *page)  in lru_note_cost_page()  argument
314 	lru_note_cost(mem_cgroup_page_lruvec(page, page_pgdat(page)),  in lru_note_cost_page()
315 		      page_is_file_lru(page), thp_nr_pages(page));  in lru_note_cost_page()
318 static void __activate_page(struct page *page, struct lruvec *lruvec,  in __activate_page()  argument
321 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {  in __activate_page()
322 		int lru = page_lru_base_type(page);  in __activate_page()
323 		int nr_pages = thp_nr_pages(page);  in __activate_page()
325 		del_page_from_lru_list(page, lruvec, lru);  in __activate_page()
326 		SetPageActive(page);  in __activate_page()
328 		add_page_to_lru_list(page, lruvec, lru);  in __activate_page()
329 		trace_mm_lru_activate(page);  in __activate_page()
351 static void activate_page(struct page *page)  in activate_page()  argument
353 	page = compound_head(page);  in activate_page()
354 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {  in activate_page()
359 		get_page(page);  in activate_page()
360 		if (!pagevec_add(pvec, page) || PageCompound(page))  in activate_page()
371 static void activate_page(struct page *page)  in activate_page()  argument
373 	pg_data_t *pgdat = page_pgdat(page);  in activate_page()
375 	page = compound_head(page);  in activate_page()
377 	__activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL);  in activate_page()
382 static void __lru_cache_activate_page(struct page *page)  in __lru_cache_activate_page()  argument
401 		struct page *pagevec_page = pvec->pages[i];  in __lru_cache_activate_page()
403 		if (pagevec_page == page) {  in __lru_cache_activate_page()
404 			SetPageActive(page);  in __lru_cache_activate_page()
422 void mark_page_accessed(struct page *page)  in mark_page_accessed()  argument
424 	page = compound_head(page);  in mark_page_accessed()
426 	if (!PageReferenced(page)) {  in mark_page_accessed()
427 		SetPageReferenced(page);  in mark_page_accessed()
428 	} else if (PageUnevictable(page)) {  in mark_page_accessed()
434 	} else if (!PageActive(page)) {  in mark_page_accessed()
441 		if (PageLRU(page))  in mark_page_accessed()
442 			activate_page(page);  in mark_page_accessed()
444 			__lru_cache_activate_page(page);  in mark_page_accessed()
445 		ClearPageReferenced(page);  in mark_page_accessed()
446 		workingset_activation(page);  in mark_page_accessed()
448 	if (page_is_idle(page))  in mark_page_accessed()
449 		clear_page_idle(page);  in mark_page_accessed()
462 void lru_cache_add(struct page *page)  in lru_cache_add()  argument
466 	VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);  in lru_cache_add()
467 	VM_BUG_ON_PAGE(PageLRU(page), page);  in lru_cache_add()
469 	get_page(page);  in lru_cache_add()
472 	if (!pagevec_add(pvec, page) || PageCompound(page))  in lru_cache_add()
486 void lru_cache_add_inactive_or_unevictable(struct page *page,  in lru_cache_add_inactive_or_unevictable()  argument
491 	VM_BUG_ON_PAGE(PageLRU(page), page);  in lru_cache_add_inactive_or_unevictable()
494 	if (unlikely(unevictable) && !TestSetPageMlocked(page)) {  in lru_cache_add_inactive_or_unevictable()
495 		int nr_pages = thp_nr_pages(page);  in lru_cache_add_inactive_or_unevictable()
501 		__mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);  in lru_cache_add_inactive_or_unevictable()
504 	lru_cache_add(page);  in lru_cache_add_inactive_or_unevictable()
528 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,  in lru_deactivate_file_fn()  argument
533 	int nr_pages = thp_nr_pages(page);  in lru_deactivate_file_fn()
535 	if (!PageLRU(page))  in lru_deactivate_file_fn()
538 	if (PageUnevictable(page))  in lru_deactivate_file_fn()
542 	if (page_mapped(page))  in lru_deactivate_file_fn()
545 	active = PageActive(page);  in lru_deactivate_file_fn()
546 	lru = page_lru_base_type(page);  in lru_deactivate_file_fn()
548 	del_page_from_lru_list(page, lruvec, lru + active);  in lru_deactivate_file_fn()
549 	ClearPageActive(page);  in lru_deactivate_file_fn()
550 	ClearPageReferenced(page);  in lru_deactivate_file_fn()
552 	if (PageWriteback(page) || PageDirty(page)) {  in lru_deactivate_file_fn()
558 		add_page_to_lru_list(page, lruvec, lru);  in lru_deactivate_file_fn()
559 		SetPageReclaim(page);  in lru_deactivate_file_fn()
565 		add_page_to_lru_list_tail(page, lruvec, lru);  in lru_deactivate_file_fn()
576 static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,  in lru_deactivate_fn()  argument
579 	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {  in lru_deactivate_fn()
580 		int lru = page_lru_base_type(page);  in lru_deactivate_fn()
581 		int nr_pages = thp_nr_pages(page);  in lru_deactivate_fn()
583 		del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE);  in lru_deactivate_fn()
584 		ClearPageActive(page);  in lru_deactivate_fn()
585 		ClearPageReferenced(page);  in lru_deactivate_fn()
586 		add_page_to_lru_list(page, lruvec, lru);  in lru_deactivate_fn()
594 static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,  in lru_lazyfree_fn()  argument
597 	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&  in lru_lazyfree_fn()
598 	    !PageSwapCache(page) && !PageUnevictable(page)) {  in lru_lazyfree_fn()
599 		bool active = PageActive(page);  in lru_lazyfree_fn()
600 		int nr_pages = thp_nr_pages(page);  in lru_lazyfree_fn()
602 		del_page_from_lru_list(page, lruvec,  in lru_lazyfree_fn()
604 		ClearPageActive(page);  in lru_lazyfree_fn()
605 		ClearPageReferenced(page);  in lru_lazyfree_fn()
611 		ClearPageSwapBacked(page);  in lru_lazyfree_fn()
612 		add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);  in lru_lazyfree_fn()
666 void deactivate_file_page(struct page *page)  in deactivate_file_page()  argument
672 	if (PageUnevictable(page))  in deactivate_file_page()
675 	if (likely(get_page_unless_zero(page))) {  in deactivate_file_page()
681 		if (!pagevec_add(pvec, page) || PageCompound(page))  in deactivate_file_page()
695 void deactivate_page(struct page *page)  in deactivate_page()  argument
697 	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {  in deactivate_page()
702 		get_page(page);  in deactivate_page()
703 		if (!pagevec_add(pvec, page) || PageCompound(page))  in deactivate_page()
716 void mark_page_lazyfree(struct page *page)  in mark_page_lazyfree()  argument
718 	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&  in mark_page_lazyfree()
719 	    !PageSwapCache(page) && !PageUnevictable(page)) {  in mark_page_lazyfree()
724 		get_page(page);  in mark_page_lazyfree()
725 		if (!pagevec_add(pvec, page) || PageCompound(page))  in mark_page_lazyfree()
870 void release_pages(struct page **pages, int nr)  in release_pages()
880 		struct page *page = pages[i];  in release_pages()  local
892 		page = compound_head(page);  in release_pages()
893 		if (is_huge_zero_page(page))  in release_pages()
896 		if (is_zone_device_page(page)) {  in release_pages()
908 			if (page_is_devmap_managed(page)) {  in release_pages()
909 				put_devmap_managed_page(page);  in release_pages()
914 		if (!put_page_testzero(page))  in release_pages()
917 		if (PageCompound(page)) {  in release_pages()
922 			__put_compound_page(page);  in release_pages()
926 		if (PageLRU(page)) {  in release_pages()
927 			struct pglist_data *pgdat = page_pgdat(page);  in release_pages()
938 			lruvec = mem_cgroup_page_lruvec(page, locked_pgdat);  in release_pages()
939 			VM_BUG_ON_PAGE(!PageLRU(page), page);  in release_pages()
940 			__ClearPageLRU(page);  in release_pages()
941 			del_page_from_lru_list(page, lruvec, page_off_lru(page));  in release_pages()
944 		__ClearPageWaiters(page);  in release_pages()
946 		list_add(&page->lru, &pages_to_free);  in release_pages()
979 void lru_add_page_tail(struct page *page, struct page *page_tail,  in lru_add_page_tail()  argument
982 	VM_BUG_ON_PAGE(!PageHead(page), page);  in lru_add_page_tail()
983 	VM_BUG_ON_PAGE(PageCompound(page_tail), page);  in lru_add_page_tail()
984 	VM_BUG_ON_PAGE(PageLRU(page_tail), page);  in lru_add_page_tail()
990 	if (likely(PageLRU(page)))  in lru_add_page_tail()
991 		list_add_tail(&page_tail->lru, &page->lru);  in lru_add_page_tail()
1010 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,  in __pagevec_lru_add_fn()  argument
1014 	int was_unevictable = TestClearPageUnevictable(page);  in __pagevec_lru_add_fn()
1015 	int nr_pages = thp_nr_pages(page);  in __pagevec_lru_add_fn()
1017 	VM_BUG_ON_PAGE(PageLRU(page), page);  in __pagevec_lru_add_fn()
1045 	SetPageLRU(page);  in __pagevec_lru_add_fn()
1048 	if (page_evictable(page)) {  in __pagevec_lru_add_fn()
1049 		lru = page_lru(page);  in __pagevec_lru_add_fn()
1054 		ClearPageActive(page);  in __pagevec_lru_add_fn()
1055 		SetPageUnevictable(page);  in __pagevec_lru_add_fn()
1060 	add_page_to_lru_list(page, lruvec, lru);  in __pagevec_lru_add_fn()
1061 	trace_mm_lru_insertion(page, lru);  in __pagevec_lru_add_fn()
1121 		struct page *page = pvec->pages[i];  in pagevec_remove_exceptionals()  local
1122 		if (!xa_is_value(page))  in pagevec_remove_exceptionals()
1123 			pvec->pages[j++] = page;  in pagevec_remove_exceptionals()
1195 void put_devmap_managed_page(struct page *page)  in put_devmap_managed_page()  argument
1199 	if (WARN_ON_ONCE(!page_is_devmap_managed(page)))  in put_devmap_managed_page()
1202 	count = page_ref_dec_return(page);  in put_devmap_managed_page()
1210 		free_devmap_managed_page(page);  in put_devmap_managed_page()
1212 		__put_page(page);  in put_devmap_managed_page()