338 unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)  in lruvec_lru_size()  argument
344 		lru_size = mem_cgroup_get_lru_size(lruvec, lru);  in lruvec_lru_size()
346 		lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);  in lruvec_lru_size()
349 		struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];  in lruvec_lru_size()
356 			size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);  in lruvec_lru_size()
358 			size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],  in lruvec_lru_size()
1618 static __always_inline void update_lru_sizes(struct lruvec *lruvec,  in update_lru_sizes()  argument
1627 		__update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);  in update_lru_sizes()
1629 		mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);  in update_lru_sizes()
1656 		struct lruvec *lruvec, struct list_head *dst,  in isolate_lru_pages()  argument
1660 	struct list_head *src = &lruvec->lists[lru];  in isolate_lru_pages()
1732 	update_lru_sizes(lruvec, lru, nr_zone_taken);  in isolate_lru_pages()
1771 		struct lruvec *lruvec;  in isolate_lru_page()  local
1774 		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);  in isolate_lru_page()
1779 			del_page_from_lru_list(page, lruvec, lru);  in isolate_lru_page()
1825 putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)  in putback_inactive_pages()  argument
1827 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;  in putback_inactive_pages()
1828 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in putback_inactive_pages()
1847 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in putback_inactive_pages()
1851 		add_page_to_lru_list(page, lruvec, lru);  in putback_inactive_pages()
1861 			del_page_from_lru_list(page, lruvec, lru);  in putback_inactive_pages()
1897 shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,  in shrink_inactive_list()  argument
1907 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_inactive_list()
1908 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;  in shrink_inactive_list()
1931 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,  in shrink_inactive_list()
1940 		count_memcg_events(lruvec_memcg(lruvec), PGSCAN_KSWAPD,  in shrink_inactive_list()
1945 		count_memcg_events(lruvec_memcg(lruvec), PGSCAN_DIRECT,  in shrink_inactive_list()
1961 		count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_KSWAPD,  in shrink_inactive_list()
1966 		count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_DIRECT,  in shrink_inactive_list()
1970 	putback_inactive_pages(lruvec, &page_list);  in shrink_inactive_list()
2027 static unsigned move_active_pages_to_lru(struct lruvec *lruvec,  in move_active_pages_to_lru()  argument
2032 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in move_active_pages_to_lru()
2039 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in move_active_pages_to_lru()
2045 		update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);  in move_active_pages_to_lru()
2046 		list_move(&page->lru, &lruvec->lists[lru]);  in move_active_pages_to_lru()
2051 			del_page_from_lru_list(page, lruvec, lru);  in move_active_pages_to_lru()
2067 		count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,  in move_active_pages_to_lru()
2075 			       struct lruvec *lruvec,  in shrink_active_list()  argument
2086 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;  in shrink_active_list()
2091 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_active_list()
2100 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,  in shrink_active_list()
2107 	count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);  in shrink_active_list()
2163 	nr_activate = move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);  in shrink_active_list()
2164 	nr_deactivate = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);  in shrink_active_list()
2202 static bool inactive_list_is_low(struct lruvec *lruvec, bool file,  in inactive_list_is_low()  argument
2207 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in inactive_list_is_low()
2221 	inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);  in inactive_list_is_low()
2222 	active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);  in inactive_list_is_low()
2234 	if (file && actual_reclaim && lruvec->refaults != refaults) {  in inactive_list_is_low()
2246 			lruvec_lru_size(lruvec, inactive_lru, MAX_NR_ZONES), inactive,  in inactive_list_is_low()
2247 			lruvec_lru_size(lruvec, active_lru, MAX_NR_ZONES), active,  in inactive_list_is_low()
2254 				 struct lruvec *lruvec, struct mem_cgroup *memcg,  in shrink_list()  argument
2258 		if (inactive_list_is_low(lruvec, is_file_lru(lru),  in shrink_list()
2260 			shrink_active_list(nr_to_scan, lruvec, sc, lru);  in shrink_list()
2264 	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);  in shrink_list()
2283 static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,  in get_scan_count()  argument
2288 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;  in get_scan_count()
2291 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in get_scan_count()
2359 			if (!inactive_list_is_low(lruvec, false, memcg, sc, false) &&  in get_scan_count()
2360 			    lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx)  in get_scan_count()
2377 	if (!inactive_list_is_low(lruvec, true, memcg, sc, false) &&  in get_scan_count()
2378 	    lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {  in get_scan_count()
2404 	anon  = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +  in get_scan_count()
2405 		lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, MAX_NR_ZONES);  in get_scan_count()
2406 	file  = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +  in get_scan_count()
2407 		lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);  in get_scan_count()
2442 		size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);  in get_scan_count()
2487 	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);  in shrink_node_memcg()  local
2497 	get_scan_count(lruvec, memcg, sc, nr, lru_pages);  in shrink_node_memcg()
2528 							    lruvec, memcg, sc);  in shrink_node_memcg()
2595 	if (inactive_list_is_low(lruvec, false, memcg, sc, true))  in shrink_node_memcg()
2596 		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,  in shrink_node_memcg()
2991 		struct lruvec *lruvec;  in snapshot_refaults()  local
2998 		lruvec = mem_cgroup_lruvec(pgdat, memcg);  in snapshot_refaults()
2999 		lruvec->refaults = refaults;  in snapshot_refaults()
3353 		struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);  in age_active_anon()  local
3355 		if (inactive_list_is_low(lruvec, false, memcg, sc, true))  in age_active_anon()
3356 			shrink_active_list(SWAP_CLUSTER_MAX, lruvec,  in age_active_anon()
4176 	struct lruvec *lruvec;  in check_move_unevictable_pages()  local
4193 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in check_move_unevictable_pages()
4203 			del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);  in check_move_unevictable_pages()
4204 			add_page_to_lru_list(page, lruvec, lru);  in check_move_unevictable_pages()