1073 				     struct pglist_data *pgdat,  in shrink_page_list()  argument
1189 			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {  in shrink_page_list()
1327 			     !test_bit(PGDAT_DIRTY, &pgdat->flags))) {  in shrink_page_list()
1770 		pg_data_t *pgdat = page_pgdat(page);  in isolate_lru_page()  local
1773 		spin_lock_irq(&pgdat->lru_lock);  in isolate_lru_page()
1774 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in isolate_lru_page()
1782 		spin_unlock_irq(&pgdat->lru_lock);  in isolate_lru_page()
1794 static int too_many_isolated(struct pglist_data *pgdat, int file,  in too_many_isolated()  argument
1806 		inactive = node_page_state(pgdat, NR_INACTIVE_FILE);  in too_many_isolated()
1807 		isolated = node_page_state(pgdat, NR_ISOLATED_FILE);  in too_many_isolated()
1809 		inactive = node_page_state(pgdat, NR_INACTIVE_ANON);  in too_many_isolated()
1810 		isolated = node_page_state(pgdat, NR_ISOLATED_ANON);  in too_many_isolated()
1847 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in move_pages_to_lru()  local
1858 			spin_unlock_irq(&pgdat->lru_lock);  in move_pages_to_lru()
1860 			spin_lock_irq(&pgdat->lru_lock);  in move_pages_to_lru()
1863 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in move_pages_to_lru()
1878 				spin_unlock_irq(&pgdat->lru_lock);  in move_pages_to_lru()
1880 				spin_lock_irq(&pgdat->lru_lock);  in move_pages_to_lru()
1926 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_inactive_list()  local
1929 	while (unlikely(too_many_isolated(pgdat, file, sc))) {  in shrink_inactive_list()
1944 	spin_lock_irq(&pgdat->lru_lock);  in shrink_inactive_list()
1949 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);  in shrink_inactive_list()
1956 	spin_unlock_irq(&pgdat->lru_lock);  in shrink_inactive_list()
1961 	nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0,  in shrink_inactive_list()
1964 	spin_lock_irq(&pgdat->lru_lock);  in shrink_inactive_list()
1968 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);  in shrink_inactive_list()
1976 	spin_unlock_irq(&pgdat->lru_lock);  in shrink_inactive_list()
2004 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,  in shrink_inactive_list()
2024 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_active_list()  local
2028 	spin_lock_irq(&pgdat->lru_lock);  in shrink_active_list()
2033 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);  in shrink_active_list()
2039 	spin_unlock_irq(&pgdat->lru_lock);  in shrink_active_list()
2085 	spin_lock_irq(&pgdat->lru_lock);  in shrink_active_list()
2095 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);  in shrink_active_list()
2096 	spin_unlock_irq(&pgdat->lru_lock);  in shrink_active_list()
2100 	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,  in shrink_active_list()
2557 static inline bool should_continue_reclaim(struct pglist_data *pgdat,  in should_continue_reclaim()  argument
2584 		struct zone *zone = &pgdat->node_zones[z];  in should_continue_reclaim()
2603 	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);  in should_continue_reclaim()
2605 		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);  in should_continue_reclaim()
2610 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)  in shrink_node_memcgs()  argument
2617 		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in shrink_node_memcgs()
2656 		shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,  in shrink_node_memcgs()
2667 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)  in shrink_node()  argument
2675 	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);  in shrink_node()
2686 	spin_lock_irq(&pgdat->lru_lock);  in shrink_node()
2689 	spin_unlock_irq(&pgdat->lru_lock);  in shrink_node()
2746 		free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);  in shrink_node()
2747 		file = node_page_state(pgdat, NR_ACTIVE_FILE) +  in shrink_node()
2748 			   node_page_state(pgdat, NR_INACTIVE_FILE);  in shrink_node()
2751 			struct zone *zone = &pgdat->node_zones[z];  in shrink_node()
2763 		anon = node_page_state(pgdat, NR_INACTIVE_ANON);  in shrink_node()
2771 	shrink_node_memcgs(pgdat, sc);  in shrink_node()
2805 			set_bit(PGDAT_WRITEBACK, &pgdat->flags);  in shrink_node()
2809 			set_bit(PGDAT_DIRTY, &pgdat->flags);  in shrink_node()
2845 	if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,  in shrink_node()
2856 		pgdat->kswapd_failures = 0;  in shrink_node()
2984 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)  in snapshot_refaults()  argument
2989 	target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);  in snapshot_refaults()
3100 static bool allow_direct_reclaim(pg_data_t *pgdat)  in allow_direct_reclaim()  argument
3108 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)  in allow_direct_reclaim()
3112 		zone = &pgdat->node_zones[i];  in allow_direct_reclaim()
3130 	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {  in allow_direct_reclaim()
3131 		if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)  in allow_direct_reclaim()
3132 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);  in allow_direct_reclaim()
3134 		wake_up_interruptible(&pgdat->kswapd_wait);  in allow_direct_reclaim()
3154 	pg_data_t *pgdat = NULL;  in throttle_direct_reclaim()  local
3193 		pgdat = zone->zone_pgdat;  in throttle_direct_reclaim()
3194 		if (allow_direct_reclaim(pgdat))  in throttle_direct_reclaim()
3200 	if (!pgdat)  in throttle_direct_reclaim()
3215 		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,  in throttle_direct_reclaim()
3216 			allow_direct_reclaim(pgdat), HZ);  in throttle_direct_reclaim()
3223 		allow_direct_reclaim(pgdat));  in throttle_direct_reclaim()
3281 						pg_data_t *pgdat,  in mem_cgroup_shrink_node()  argument
3284 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in mem_cgroup_shrink_node()
3357 static void age_active_anon(struct pglist_data *pgdat,  in age_active_anon()  argument
3366 	lruvec = mem_cgroup_lruvec(NULL, pgdat);  in age_active_anon()
3372 		lruvec = mem_cgroup_lruvec(memcg, pgdat);  in age_active_anon()
3379 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)  in pgdat_watermark_boosted()  argument
3392 		zone = pgdat->node_zones + i;  in pgdat_watermark_boosted()
3407 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)  in pgdat_balanced()  argument
3418 		zone = pgdat->node_zones + i;  in pgdat_balanced()
3440 static void clear_pgdat_congested(pg_data_t *pgdat)  in clear_pgdat_congested()  argument
3442 	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);  in clear_pgdat_congested()
3445 	clear_bit(PGDAT_DIRTY, &pgdat->flags);  in clear_pgdat_congested()
3446 	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);  in clear_pgdat_congested()
3455 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,  in prepare_kswapd_sleep()  argument
3471 	if (waitqueue_active(&pgdat->pfmemalloc_wait))  in prepare_kswapd_sleep()
3472 		wake_up_all(&pgdat->pfmemalloc_wait);  in prepare_kswapd_sleep()
3475 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)  in prepare_kswapd_sleep()
3478 	if (pgdat_balanced(pgdat, order, highest_zoneidx)) {  in prepare_kswapd_sleep()
3479 		clear_pgdat_congested(pgdat);  in prepare_kswapd_sleep()
3494 static bool kswapd_shrink_node(pg_data_t *pgdat,  in kswapd_shrink_node()  argument
3503 		zone = pgdat->node_zones + z;  in kswapd_shrink_node()
3514 	shrink_node(pgdat, sc);  in kswapd_shrink_node()
3542 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)  in balance_pgdat()  argument
3571 		zone = pgdat->node_zones + i;  in balance_pgdat()
3602 				zone = pgdat->node_zones + i;  in balance_pgdat()
3618 		balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);  in balance_pgdat()
3651 		age_active_anon(pgdat, &sc);  in balance_pgdat()
3663 		nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,  in balance_pgdat()
3672 		if (kswapd_shrink_node(pgdat, &sc))  in balance_pgdat()
3680 		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&  in balance_pgdat()
3681 				allow_direct_reclaim(pgdat))  in balance_pgdat()
3682 			wake_up_all(&pgdat->pfmemalloc_wait);  in balance_pgdat()
3711 		pgdat->kswapd_failures++;  in balance_pgdat()
3723 			zone = pgdat->node_zones + i;  in balance_pgdat()
3733 		wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);  in balance_pgdat()
3736 	snapshot_refaults(NULL, pgdat);  in balance_pgdat()
3757 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,  in kswapd_highest_zoneidx()  argument
3760 	enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);  in kswapd_highest_zoneidx()
3765 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,  in kswapd_try_to_sleep()  argument
3774 	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);  in kswapd_try_to_sleep()
3783 	if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {  in kswapd_try_to_sleep()
3790 		reset_isolation_suitable(pgdat);  in kswapd_try_to_sleep()
3796 		wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);  in kswapd_try_to_sleep()
3806 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx,  in kswapd_try_to_sleep()
3807 					kswapd_highest_zoneidx(pgdat,  in kswapd_try_to_sleep()
3810 			if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)  in kswapd_try_to_sleep()
3811 				WRITE_ONCE(pgdat->kswapd_order, reclaim_order);  in kswapd_try_to_sleep()
3814 		finish_wait(&pgdat->kswapd_wait, &wait);  in kswapd_try_to_sleep()
3815 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);  in kswapd_try_to_sleep()
3823 	    prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {  in kswapd_try_to_sleep()
3824 		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);  in kswapd_try_to_sleep()
3834 		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);  in kswapd_try_to_sleep()
3839 		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);  in kswapd_try_to_sleep()
3846 	finish_wait(&pgdat->kswapd_wait, &wait);  in kswapd_try_to_sleep()
3866 	pg_data_t *pgdat = (pg_data_t*)p;  in kswapd()  local
3868 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);  in kswapd()
3888 	WRITE_ONCE(pgdat->kswapd_order, 0);  in kswapd()
3889 	WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);  in kswapd()
3893 		alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);  in kswapd()
3894 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,  in kswapd()
3898 		kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,  in kswapd()
3902 		alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);  in kswapd()
3903 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,  in kswapd()
3905 		WRITE_ONCE(pgdat->kswapd_order, 0);  in kswapd()
3906 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);  in kswapd()
3927 		trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,  in kswapd()
3929 		reclaim_order = balance_pgdat(pgdat, alloc_order,  in kswapd()
3950 	pg_data_t *pgdat;  in wakeup_kswapd()  local
3959 	pgdat = zone->zone_pgdat;  in wakeup_kswapd()
3960 	curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);  in wakeup_kswapd()
3963 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);  in wakeup_kswapd()
3965 	if (READ_ONCE(pgdat->kswapd_order) < order)  in wakeup_kswapd()
3966 		WRITE_ONCE(pgdat->kswapd_order, order);  in wakeup_kswapd()
3968 	if (!waitqueue_active(&pgdat->kswapd_wait))  in wakeup_kswapd()
3972 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||  in wakeup_kswapd()
3973 	    (pgdat_balanced(pgdat, order, highest_zoneidx) &&  in wakeup_kswapd()
3974 	     !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {  in wakeup_kswapd()
3983 			wakeup_kcompactd(pgdat, order, highest_zoneidx);  in wakeup_kswapd()
3987 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,  in wakeup_kswapd()
3989 	wake_up_interruptible(&pgdat->kswapd_wait);  in wakeup_kswapd()
4037 	pg_data_t *pgdat = NODE_DATA(nid);  in kswapd_run()  local
4040 	if (pgdat->kswapd)  in kswapd_run()
4043 	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);  in kswapd_run()
4044 	if (IS_ERR(pgdat->kswapd)) {  in kswapd_run()
4048 		ret = PTR_ERR(pgdat->kswapd);  in kswapd_run()
4049 		pgdat->kswapd = NULL;  in kswapd_run()
4111 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)  in node_unmapped_file_pages()  argument
4113 	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);  in node_unmapped_file_pages()
4114 	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +  in node_unmapped_file_pages()
4115 		node_page_state(pgdat, NR_ACTIVE_FILE);  in node_unmapped_file_pages()
4126 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)  in node_pagecache_reclaimable()  argument
4138 		nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);  in node_pagecache_reclaimable()
4140 		nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);  in node_pagecache_reclaimable()
4144 		delta += node_page_state(pgdat, NR_FILE_DIRTY);  in node_pagecache_reclaimable()
4156 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)  in __node_reclaim()  argument
4173 	trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,  in __node_reclaim()
4187 	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {  in __node_reclaim()
4193 			shrink_node(pgdat, &sc);  in __node_reclaim()
4207 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)  in node_reclaim()  argument
4221 	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&  in node_reclaim()
4222 	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=  in node_reclaim()
4223 	    pgdat->min_slab_pages)  in node_reclaim()
4238 	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())  in node_reclaim()
4241 	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))  in node_reclaim()
4244 	ret = __node_reclaim(pgdat, gfp_mask, order);  in node_reclaim()
4245 	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);  in node_reclaim()
4266 	struct pglist_data *pgdat = NULL;  in check_move_unevictable_pages()  local
4282 		if (pagepgdat != pgdat) {  in check_move_unevictable_pages()
4283 			if (pgdat)  in check_move_unevictable_pages()
4284 				spin_unlock_irq(&pgdat->lru_lock);  in check_move_unevictable_pages()
4285 			pgdat = pagepgdat;  in check_move_unevictable_pages()
4286 			spin_lock_irq(&pgdat->lru_lock);  in check_move_unevictable_pages()
4288 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in check_move_unevictable_pages()
4304 	if (pgdat) {  in check_move_unevictable_pages()
4307 		spin_unlock_irq(&pgdat->lru_lock);  in check_move_unevictable_pages()