1336 				     struct pglist_data *pgdat)  in demote_page_list()  argument
1338 	int target_nid = next_demotion_node(pgdat->node_id);  in demote_page_list()
1365 				     struct pglist_data *pgdat,  in shrink_page_list()  argument
1379 	do_demote_pass = can_demote(pgdat->node_id, sc);  in shrink_page_list()
1484 			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {  in shrink_page_list()
1636 			     !test_bit(PGDAT_DIRTY, &pgdat->flags))) {  in shrink_page_list()
1798 	nr_reclaimed += demote_page_list(&demote_pages, pgdat);  in shrink_page_list()
2118 static int too_many_isolated(struct pglist_data *pgdat, int file,  in too_many_isolated()  argument
2130 		inactive = node_page_state(pgdat, NR_INACTIVE_FILE);  in too_many_isolated()
2131 		isolated = node_page_state(pgdat, NR_ISOLATED_FILE);  in too_many_isolated()
2133 		inactive = node_page_state(pgdat, NR_INACTIVE_ANON);  in too_many_isolated()
2134 		isolated = node_page_state(pgdat, NR_ISOLATED_ANON);  in too_many_isolated()
2246 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_inactive_list()  local
2249 	while (unlikely(too_many_isolated(pgdat, file, sc))) {  in shrink_inactive_list()
2269 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);  in shrink_inactive_list()
2281 	nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);  in shrink_inactive_list()
2286 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);  in shrink_inactive_list()
2321 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,  in shrink_inactive_list()
2358 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_active_list()  local
2367 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);  in shrink_active_list()
2429 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);  in shrink_active_list()
2434 	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,  in shrink_active_list()
2577 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in get_scan_count()  local
2588 	if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {  in get_scan_count()
2775 static bool can_age_anon_pages(struct pglist_data *pgdat,  in can_age_anon_pages()  argument
2783 	return can_demote(pgdat->node_id, sc);  in can_age_anon_pages()
2919 static inline bool should_continue_reclaim(struct pglist_data *pgdat,  in should_continue_reclaim()  argument
2946 		struct zone *zone = &pgdat->node_zones[z];  in should_continue_reclaim()
2965 	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);  in should_continue_reclaim()
2966 	if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))  in should_continue_reclaim()
2967 		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);  in should_continue_reclaim()
2972 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)  in shrink_node_memcgs()  argument
2979 		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in shrink_node_memcgs()
3018 		shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,  in shrink_node_memcgs()
3029 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)  in shrink_node()  argument
3037 	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);  in shrink_node()
3114 		free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);  in shrink_node()
3115 		file = node_page_state(pgdat, NR_ACTIVE_FILE) +  in shrink_node()
3116 			   node_page_state(pgdat, NR_INACTIVE_FILE);  in shrink_node()
3119 			struct zone *zone = &pgdat->node_zones[z];  in shrink_node()
3131 		anon = node_page_state(pgdat, NR_INACTIVE_ANON);  in shrink_node()
3139 	shrink_node_memcgs(pgdat, sc);  in shrink_node()
3173 			set_bit(PGDAT_WRITEBACK, &pgdat->flags);  in shrink_node()
3177 			set_bit(PGDAT_DIRTY, &pgdat->flags);  in shrink_node()
3213 	if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,  in shrink_node()
3224 		pgdat->kswapd_failures = 0;  in shrink_node()
3352 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)  in snapshot_refaults()  argument
3357 	target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);  in snapshot_refaults()
3468 static bool allow_direct_reclaim(pg_data_t *pgdat)  in allow_direct_reclaim()  argument
3476 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)  in allow_direct_reclaim()
3480 		zone = &pgdat->node_zones[i];  in allow_direct_reclaim()
3498 	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {  in allow_direct_reclaim()
3499 		if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)  in allow_direct_reclaim()
3500 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);  in allow_direct_reclaim()
3502 		wake_up_interruptible(&pgdat->kswapd_wait);  in allow_direct_reclaim()
3522 	pg_data_t *pgdat = NULL;  in throttle_direct_reclaim()  local
3561 		pgdat = zone->zone_pgdat;  in throttle_direct_reclaim()
3562 		if (allow_direct_reclaim(pgdat))  in throttle_direct_reclaim()
3568 	if (!pgdat)  in throttle_direct_reclaim()
3583 		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,  in throttle_direct_reclaim()
3584 			allow_direct_reclaim(pgdat), HZ);  in throttle_direct_reclaim()
3588 			allow_direct_reclaim(pgdat));  in throttle_direct_reclaim()
3645 						pg_data_t *pgdat,  in mem_cgroup_shrink_node()  argument
3648 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in mem_cgroup_shrink_node()
3721 static void age_active_anon(struct pglist_data *pgdat,  in age_active_anon()  argument
3727 	if (!can_age_anon_pages(pgdat, sc))  in age_active_anon()
3730 	lruvec = mem_cgroup_lruvec(NULL, pgdat);  in age_active_anon()
3736 		lruvec = mem_cgroup_lruvec(memcg, pgdat);  in age_active_anon()
3743 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)  in pgdat_watermark_boosted()  argument
3756 		zone = pgdat->node_zones + i;  in pgdat_watermark_boosted()
3771 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)  in pgdat_balanced()  argument
3782 		zone = pgdat->node_zones + i;  in pgdat_balanced()
3804 static void clear_pgdat_congested(pg_data_t *pgdat)  in clear_pgdat_congested()  argument
3806 	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);  in clear_pgdat_congested()
3809 	clear_bit(PGDAT_DIRTY, &pgdat->flags);  in clear_pgdat_congested()
3810 	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);  in clear_pgdat_congested()
3819 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,  in prepare_kswapd_sleep()  argument
3835 	if (waitqueue_active(&pgdat->pfmemalloc_wait))  in prepare_kswapd_sleep()
3836 		wake_up_all(&pgdat->pfmemalloc_wait);  in prepare_kswapd_sleep()
3839 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)  in prepare_kswapd_sleep()
3842 	if (pgdat_balanced(pgdat, order, highest_zoneidx)) {  in prepare_kswapd_sleep()
3843 		clear_pgdat_congested(pgdat);  in prepare_kswapd_sleep()
3858 static bool kswapd_shrink_node(pg_data_t *pgdat,  in kswapd_shrink_node()  argument
3867 		zone = pgdat->node_zones + z;  in kswapd_shrink_node()
3878 	shrink_node(pgdat, sc);  in kswapd_shrink_node()
3895 update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)  in update_reclaim_active()  argument
3901 		zone = pgdat->node_zones + i;  in update_reclaim_active()
3914 set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)  in set_reclaim_active()  argument
3916 	update_reclaim_active(pgdat, highest_zoneidx, true);  in set_reclaim_active()
3920 clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)  in clear_reclaim_active()  argument
3922 	update_reclaim_active(pgdat, highest_zoneidx, false);  in clear_reclaim_active()
3938 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)  in balance_pgdat()  argument
3967 		zone = pgdat->node_zones + i;  in balance_pgdat()
3977 	set_reclaim_active(pgdat, highest_zoneidx);  in balance_pgdat()
3999 				zone = pgdat->node_zones + i;  in balance_pgdat()
4015 		balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);  in balance_pgdat()
4048 		age_active_anon(pgdat, &sc);  in balance_pgdat()
4060 		nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,  in balance_pgdat()
4069 		if (kswapd_shrink_node(pgdat, &sc))  in balance_pgdat()
4077 		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&  in balance_pgdat()
4078 				allow_direct_reclaim(pgdat))  in balance_pgdat()
4079 			wake_up_all(&pgdat->pfmemalloc_wait);  in balance_pgdat()
4108 		pgdat->kswapd_failures++;  in balance_pgdat()
4111 	clear_reclaim_active(pgdat, highest_zoneidx);  in balance_pgdat()
4122 			zone = pgdat->node_zones + i;  in balance_pgdat()
4132 		wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);  in balance_pgdat()
4135 	snapshot_refaults(NULL, pgdat);  in balance_pgdat()
4156 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,  in kswapd_highest_zoneidx()  argument
4159 	enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);  in kswapd_highest_zoneidx()
4164 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,  in kswapd_try_to_sleep()  argument
4173 	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);  in kswapd_try_to_sleep()
4182 	if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {  in kswapd_try_to_sleep()
4189 		reset_isolation_suitable(pgdat);  in kswapd_try_to_sleep()
4195 		wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);  in kswapd_try_to_sleep()
4205 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx,  in kswapd_try_to_sleep()
4206 					kswapd_highest_zoneidx(pgdat,  in kswapd_try_to_sleep()
4209 			if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)  in kswapd_try_to_sleep()
4210 				WRITE_ONCE(pgdat->kswapd_order, reclaim_order);  in kswapd_try_to_sleep()
4213 		finish_wait(&pgdat->kswapd_wait, &wait);  in kswapd_try_to_sleep()
4214 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);  in kswapd_try_to_sleep()
4222 	    prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {  in kswapd_try_to_sleep()
4223 		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);  in kswapd_try_to_sleep()
4233 		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);  in kswapd_try_to_sleep()
4238 		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);  in kswapd_try_to_sleep()
4245 	finish_wait(&pgdat->kswapd_wait, &wait);  in kswapd_try_to_sleep()
4265 	pg_data_t *pgdat = (pg_data_t *)p;  in kswapd()  local
4267 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);  in kswapd()
4287 	WRITE_ONCE(pgdat->kswapd_order, 0);  in kswapd()
4288 	WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);  in kswapd()
4292 		alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);  in kswapd()
4293 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,  in kswapd()
4297 		kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,  in kswapd()
4301 		alloc_order = READ_ONCE(pgdat->kswapd_order);  in kswapd()
4302 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,  in kswapd()
4304 		WRITE_ONCE(pgdat->kswapd_order, 0);  in kswapd()
4305 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);  in kswapd()
4326 		trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,  in kswapd()
4328 		reclaim_order = balance_pgdat(pgdat, alloc_order,  in kswapd()
4349 	pg_data_t *pgdat;  in wakeup_kswapd()  local
4358 	pgdat = zone->zone_pgdat;  in wakeup_kswapd()
4359 	curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);  in wakeup_kswapd()
4362 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);  in wakeup_kswapd()
4364 	if (READ_ONCE(pgdat->kswapd_order) < order)  in wakeup_kswapd()
4365 		WRITE_ONCE(pgdat->kswapd_order, order);  in wakeup_kswapd()
4367 	if (!waitqueue_active(&pgdat->kswapd_wait))  in wakeup_kswapd()
4371 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||  in wakeup_kswapd()
4372 	    (pgdat_balanced(pgdat, order, highest_zoneidx) &&  in wakeup_kswapd()
4373 	     !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {  in wakeup_kswapd()
4382 			wakeup_kcompactd(pgdat, order, highest_zoneidx);  in wakeup_kswapd()
4386 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,  in wakeup_kswapd()
4388 	wake_up_interruptible(&pgdat->kswapd_wait);  in wakeup_kswapd()
4436 	pg_data_t *pgdat = NODE_DATA(nid);  in kswapd_run()  local
4438 	if (pgdat->kswapd)  in kswapd_run()
4441 	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);  in kswapd_run()
4442 	if (IS_ERR(pgdat->kswapd)) {  in kswapd_run()
4446 		pgdat->kswapd = NULL;  in kswapd_run()
4504 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)  in node_unmapped_file_pages()  argument
4506 	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);  in node_unmapped_file_pages()
4507 	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +  in node_unmapped_file_pages()
4508 		node_page_state(pgdat, NR_ACTIVE_FILE);  in node_unmapped_file_pages()
4519 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)  in node_pagecache_reclaimable()  argument
4531 		nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);  in node_pagecache_reclaimable()
4533 		nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);  in node_pagecache_reclaimable()
4537 		delta += node_page_state(pgdat, NR_FILE_DIRTY);  in node_pagecache_reclaimable()
4549 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)  in __node_reclaim()  argument
4567 	trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,  in __node_reclaim()
4582 	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {  in __node_reclaim()
4588 			shrink_node(pgdat, &sc);  in __node_reclaim()
4603 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)  in node_reclaim()  argument
4617 	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&  in node_reclaim()
4618 	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=  in node_reclaim()
4619 	    pgdat->min_slab_pages)  in node_reclaim()
4634 	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())  in node_reclaim()
4637 	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))  in node_reclaim()
4640 	ret = __node_reclaim(pgdat, gfp_mask, order);  in node_reclaim()
4641 	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);  in node_reclaim()