1161 static bool skip_throttle_noprogress(pg_data_t *pgdat)  in skip_throttle_noprogress()  argument
1170 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)  in skip_throttle_noprogress()
1179 		struct zone *zone = pgdat->node_zones + i;  in skip_throttle_noprogress()
1194 void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)  in reclaim_throttle()  argument
1196 	wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason];  in reclaim_throttle()
1225 		if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) {  in reclaim_throttle()
1226 			WRITE_ONCE(pgdat->nr_reclaim_start,  in reclaim_throttle()
1227 				node_page_state(pgdat, NR_THROTTLED_WRITTEN));  in reclaim_throttle()
1234 		if (skip_throttle_noprogress(pgdat)) {  in reclaim_throttle()
1256 		atomic_dec(&pgdat->nr_writeback_throttled);  in reclaim_throttle()
1258 	trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout),  in reclaim_throttle()
1268 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,  in __acct_reclaim_writeback()  argument
1282 	nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) -  in __acct_reclaim_writeback()
1283 		READ_ONCE(pgdat->nr_reclaim_start);  in __acct_reclaim_writeback()
1286 		wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]);  in __acct_reclaim_writeback()
1650 				     struct pglist_data *pgdat)  in demote_folio_list()  argument
1652 	int target_nid = next_demotion_node(pgdat->node_id);  in demote_folio_list()
1674 	node_get_allowed_targets(pgdat, &allowed_mask);  in demote_folio_list()
1706 		struct pglist_data *pgdat, struct scan_control *sc,  in shrink_folio_list()  argument
1719 	do_demote_pass = can_demote(pgdat->node_id, sc);  in shrink_folio_list()
1824 			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {  in shrink_folio_list()
1988 			     !test_bit(PGDAT_DIRTY, &pgdat->flags))) {  in shrink_folio_list()
2155 	nr_reclaimed += demote_folio_list(&demote_folios, pgdat);  in shrink_folio_list()
2447 static int too_many_isolated(struct pglist_data *pgdat, int file,  in too_many_isolated()  argument
2460 		inactive = node_page_state(pgdat, NR_INACTIVE_FILE);  in too_many_isolated()
2461 		isolated = node_page_state(pgdat, NR_ISOLATED_FILE);  in too_many_isolated()
2463 		inactive = node_page_state(pgdat, NR_INACTIVE_ANON);  in too_many_isolated()
2464 		isolated = node_page_state(pgdat, NR_ISOLATED_ANON);  in too_many_isolated()
2479 		wake_throttle_isolated(pgdat);  in too_many_isolated()
2579 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_inactive_list()  local
2582 	while (unlikely(too_many_isolated(pgdat, file, sc))) {  in shrink_inactive_list()
2588 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);  in shrink_inactive_list()
2602 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);  in shrink_inactive_list()
2614 	nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false);  in shrink_inactive_list()
2619 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);  in shrink_inactive_list()
2654 			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);  in shrink_inactive_list()
2666 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,  in shrink_inactive_list()
2702 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_active_list()  local
2711 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);  in shrink_active_list()
2776 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);  in shrink_active_list()
2783 	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,  in shrink_active_list()
2788 				      struct pglist_data *pgdat)  in reclaim_folio_list()  argument
2801 	nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false);  in reclaim_folio_list()
2912 static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc)  in prepare_scan_count()  argument
2920 	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);  in prepare_scan_count()
2991 		free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);  in prepare_scan_count()
2992 		file = node_page_state(pgdat, NR_ACTIVE_FILE) +  in prepare_scan_count()
2993 			   node_page_state(pgdat, NR_INACTIVE_FILE);  in prepare_scan_count()
2996 			struct zone *zone = &pgdat->node_zones[z];  in prepare_scan_count()
3009 		anon = node_page_state(pgdat, NR_INACTIVE_ANON);  in prepare_scan_count()
3028 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in get_scan_count()  local
3039 	if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {  in get_scan_count()
3226 static bool can_age_anon_pages(struct pglist_data *pgdat,  in can_age_anon_pages()  argument
3234 	return can_demote(pgdat->node_id, sc);  in can_age_anon_pages()
3282 	struct pglist_data *pgdat = NODE_DATA(nid);  in get_lruvec()  local
3289 		if (!lruvec->pgdat)  in get_lruvec()
3290 			lruvec->pgdat = pgdat;  in get_lruvec()
3297 	return &pgdat->__lruvec;  in get_lruvec()
3303 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in get_swappiness()  local
3308 	if (!can_demote(pgdat->node_id, sc) &&  in get_swappiness()
3563 	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);  in should_skip_mm()  local
3564 	int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);  in should_skip_mm()
3973 				   struct pglist_data *pgdat, bool can_swap)  in get_pfn_folio()  argument
3978 	if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))  in get_pfn_folio()
3982 	if (folio_nid(folio) != pgdat->node_id)  in get_pfn_folio()
4014 	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);  in walk_pte_range()  local
4044 		folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);  in walk_pte_range()
4082 	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);  in walk_pmd_range_locked()  local
4125 		folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);  in walk_pmd_range_locked()
4194 			struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);  in walk_pmd_range()  local
4204 			if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))  in walk_pmd_range()
4327 static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc)  in set_mm_walk()  argument
4331 	if (pgdat && current_is_kswapd()) {  in set_mm_walk()
4334 		walk = &pgdat->mm_walk;  in set_mm_walk()
4600 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)  in lru_gen_age_node()  argument
4613 		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in lru_gen_age_node()
4662 	struct pglist_data *pgdat = folio_pgdat(folio);  in lru_gen_look_around()  local
4663 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in lru_gen_look_around()
4709 		folio = get_pfn_folio(pfn, memcg, pgdat, can_swap);  in lru_gen_look_around()
4772 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in lru_gen_rotate_memcg()  local
4774 	spin_lock_irqsave(&pgdat->memcg_lru.lock, flags);  in lru_gen_rotate_memcg()
4787 		new = get_memcg_gen(pgdat->memcg_lru.seq);  in lru_gen_rotate_memcg()
4789 		new = get_memcg_gen(pgdat->memcg_lru.seq + 1);  in lru_gen_rotate_memcg()
4796 		hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);  in lru_gen_rotate_memcg()
4798 		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);  in lru_gen_rotate_memcg()
4800 	pgdat->memcg_lru.nr_memcgs[old]--;  in lru_gen_rotate_memcg()
4801 	pgdat->memcg_lru.nr_memcgs[new]++;  in lru_gen_rotate_memcg()
4806 	if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq))  in lru_gen_rotate_memcg()
4807 		WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);  in lru_gen_rotate_memcg()
4809 	spin_unlock_irqrestore(&pgdat->memcg_lru.lock, flags);  in lru_gen_rotate_memcg()
4819 		struct pglist_data *pgdat = NODE_DATA(nid);  in lru_gen_online_memcg()  local
4822 		spin_lock_irq(&pgdat->memcg_lru.lock);  in lru_gen_online_memcg()
4826 		gen = get_memcg_gen(pgdat->memcg_lru.seq);  in lru_gen_online_memcg()
4828 		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);  in lru_gen_online_memcg()
4829 		pgdat->memcg_lru.nr_memcgs[gen]++;  in lru_gen_online_memcg()
4833 		spin_unlock_irq(&pgdat->memcg_lru.lock);  in lru_gen_online_memcg()
4854 		struct pglist_data *pgdat = NODE_DATA(nid);  in lru_gen_release_memcg()  local
4857 		spin_lock_irq(&pgdat->memcg_lru.lock);  in lru_gen_release_memcg()
4865 		pgdat->memcg_lru.nr_memcgs[gen]--;  in lru_gen_release_memcg()
4867 		if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq))  in lru_gen_release_memcg()
4868 			WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);  in lru_gen_release_memcg()
4870 		spin_unlock_irq(&pgdat->memcg_lru.lock);  in lru_gen_release_memcg()
5179 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in evict_folios()  local
5195 	reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false);  in evict_folios()
5396 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in shrink_one()  local
5417 	shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);  in shrink_one()
5430 static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)  in shrink_many()  argument
5446 	gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));  in shrink_many()
5450 	hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {  in shrink_many()
5520 static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)  in shrink_many()  argument
5532 static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc)  in set_initial_priority()  argument
5536 	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);  in set_initial_priority()
5545 	reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE);  in set_initial_priority()
5547 		reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON);  in set_initial_priority()
5557 static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)  in lru_gen_shrink_node()  argument
5576 	set_mm_walk(pgdat, sc->proactive);  in lru_gen_shrink_node()
5578 	set_initial_priority(pgdat, sc);  in lru_gen_shrink_node()
5584 		shrink_one(&pgdat->__lruvec, sc);  in lru_gen_shrink_node()
5586 		shrink_many(pgdat, sc);  in lru_gen_shrink_node()
5596 	pgdat->kswapd_failures = 0;  in lru_gen_shrink_node()
6200 void lru_gen_init_pgdat(struct pglist_data *pgdat)  in lru_gen_init_pgdat()  argument
6204 	spin_lock_init(&pgdat->memcg_lru.lock);  in lru_gen_init_pgdat()
6208 			INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i);  in lru_gen_init_pgdat()
6259 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)  in lru_gen_age_node()  argument
6267 static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)  in lru_gen_shrink_node()  argument
6409 static inline bool should_continue_reclaim(struct pglist_data *pgdat,  in should_continue_reclaim()  argument
6436 		struct zone *zone = &pgdat->node_zones[z];  in should_continue_reclaim()
6454 	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);  in should_continue_reclaim()
6455 	if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))  in should_continue_reclaim()
6456 		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);  in should_continue_reclaim()
6461 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)  in shrink_node_memcgs()  argument
6468 		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in shrink_node_memcgs()
6507 		shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,  in shrink_node_memcgs()
6519 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)  in shrink_node()  argument
6526 		lru_gen_shrink_node(pgdat, sc);  in shrink_node()
6530 	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);  in shrink_node()
6538 	prepare_scan_count(pgdat, sc);  in shrink_node()
6540 	shrink_node_memcgs(pgdat, sc);  in shrink_node()
6573 			set_bit(PGDAT_WRITEBACK, &pgdat->flags);  in shrink_node()
6577 			set_bit(PGDAT_DIRTY, &pgdat->flags);  in shrink_node()
6587 			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);  in shrink_node()
6615 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED);  in shrink_node()
6617 	if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc))  in shrink_node()
6627 		pgdat->kswapd_failures = 0;  in shrink_node()
6662 static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc)  in consider_reclaim_throttle()  argument
6671 		wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS];  in consider_reclaim_throttle()
6689 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS);  in consider_reclaim_throttle()
6792 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)  in snapshot_refaults()  argument
6800 	target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);  in snapshot_refaults()
6912 static bool allow_direct_reclaim(pg_data_t *pgdat)  in allow_direct_reclaim()  argument
6920 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)  in allow_direct_reclaim()
6924 		zone = &pgdat->node_zones[i];  in allow_direct_reclaim()
6942 	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {  in allow_direct_reclaim()
6943 		if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)  in allow_direct_reclaim()
6944 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);  in allow_direct_reclaim()
6946 		wake_up_interruptible(&pgdat->kswapd_wait);  in allow_direct_reclaim()
6966 	pg_data_t *pgdat = NULL;  in throttle_direct_reclaim()  local
7005 		pgdat = zone->zone_pgdat;  in throttle_direct_reclaim()
7006 		if (allow_direct_reclaim(pgdat))  in throttle_direct_reclaim()
7012 	if (!pgdat)  in throttle_direct_reclaim()
7027 		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,  in throttle_direct_reclaim()
7028 			allow_direct_reclaim(pgdat), HZ);  in throttle_direct_reclaim()
7032 			allow_direct_reclaim(pgdat));  in throttle_direct_reclaim()
7089 						pg_data_t *pgdat,  in mem_cgroup_shrink_node()  argument
7092 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in mem_cgroup_shrink_node()
7166 static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc)  in kswapd_age_node()  argument
7172 		lru_gen_age_node(pgdat, sc);  in kswapd_age_node()
7176 	if (!can_age_anon_pages(pgdat, sc))  in kswapd_age_node()
7179 	lruvec = mem_cgroup_lruvec(NULL, pgdat);  in kswapd_age_node()
7185 		lruvec = mem_cgroup_lruvec(memcg, pgdat);  in kswapd_age_node()
7192 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)  in pgdat_watermark_boosted()  argument
7205 		zone = pgdat->node_zones + i;  in pgdat_watermark_boosted()
7220 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)  in pgdat_balanced()  argument
7231 		zone = pgdat->node_zones + i;  in pgdat_balanced()
7256 static void clear_pgdat_congested(pg_data_t *pgdat)  in clear_pgdat_congested()  argument
7258 	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);  in clear_pgdat_congested()
7262 	clear_bit(PGDAT_DIRTY, &pgdat->flags);  in clear_pgdat_congested()
7263 	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);  in clear_pgdat_congested()
7272 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,  in prepare_kswapd_sleep()  argument
7288 	if (waitqueue_active(&pgdat->pfmemalloc_wait))  in prepare_kswapd_sleep()
7289 		wake_up_all(&pgdat->pfmemalloc_wait);  in prepare_kswapd_sleep()
7292 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)  in prepare_kswapd_sleep()
7295 	if (pgdat_balanced(pgdat, order, highest_zoneidx)) {  in prepare_kswapd_sleep()
7296 		clear_pgdat_congested(pgdat);  in prepare_kswapd_sleep()
7311 static bool kswapd_shrink_node(pg_data_t *pgdat,  in kswapd_shrink_node()  argument
7320 		zone = pgdat->node_zones + z;  in kswapd_shrink_node()
7331 	shrink_node(pgdat, sc);  in kswapd_shrink_node()
7348 update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)  in update_reclaim_active()  argument
7354 		zone = pgdat->node_zones + i;  in update_reclaim_active()
7367 set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)  in set_reclaim_active()  argument
7369 	update_reclaim_active(pgdat, highest_zoneidx, true);  in set_reclaim_active()
7373 clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)  in clear_reclaim_active()  argument
7375 	update_reclaim_active(pgdat, highest_zoneidx, false);  in clear_reclaim_active()
7391 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)  in balance_pgdat()  argument
7420 		zone = pgdat->node_zones + i;  in balance_pgdat()
7430 	set_reclaim_active(pgdat, highest_zoneidx);  in balance_pgdat()
7452 				zone = pgdat->node_zones + i;  in balance_pgdat()
7468 		balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);  in balance_pgdat()
7500 		kswapd_age_node(pgdat, &sc);  in balance_pgdat()
7512 		nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,  in balance_pgdat()
7521 		if (kswapd_shrink_node(pgdat, &sc))  in balance_pgdat()
7529 		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&  in balance_pgdat()
7530 				allow_direct_reclaim(pgdat))  in balance_pgdat()
7531 			wake_up_all(&pgdat->pfmemalloc_wait);  in balance_pgdat()
7560 		pgdat->kswapd_failures++;  in balance_pgdat()
7563 	clear_reclaim_active(pgdat, highest_zoneidx);  in balance_pgdat()
7574 			zone = pgdat->node_zones + i;  in balance_pgdat()
7584 		wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);  in balance_pgdat()
7587 	snapshot_refaults(NULL, pgdat);  in balance_pgdat()
7608 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,  in kswapd_highest_zoneidx()  argument
7611 	enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);  in kswapd_highest_zoneidx()
7616 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,  in kswapd_try_to_sleep()  argument
7625 	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);  in kswapd_try_to_sleep()
7634 	if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {  in kswapd_try_to_sleep()
7641 		reset_isolation_suitable(pgdat);  in kswapd_try_to_sleep()
7647 		wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);  in kswapd_try_to_sleep()
7657 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx,  in kswapd_try_to_sleep()
7658 					kswapd_highest_zoneidx(pgdat,  in kswapd_try_to_sleep()
7661 			if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)  in kswapd_try_to_sleep()
7662 				WRITE_ONCE(pgdat->kswapd_order, reclaim_order);  in kswapd_try_to_sleep()
7665 		finish_wait(&pgdat->kswapd_wait, &wait);  in kswapd_try_to_sleep()
7666 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);  in kswapd_try_to_sleep()
7674 	    prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {  in kswapd_try_to_sleep()
7675 		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);  in kswapd_try_to_sleep()
7685 		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);  in kswapd_try_to_sleep()
7690 		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);  in kswapd_try_to_sleep()
7697 	finish_wait(&pgdat->kswapd_wait, &wait);  in kswapd_try_to_sleep()
7717 	pg_data_t *pgdat = (pg_data_t *)p;  in kswapd()  local
7719 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);  in kswapd()
7739 	WRITE_ONCE(pgdat->kswapd_order, 0);  in kswapd()
7740 	WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);  in kswapd()
7741 	atomic_set(&pgdat->nr_writeback_throttled, 0);  in kswapd()
7745 		alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);  in kswapd()
7746 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,  in kswapd()
7750 		kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,  in kswapd()
7754 		alloc_order = READ_ONCE(pgdat->kswapd_order);  in kswapd()
7755 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,  in kswapd()
7757 		WRITE_ONCE(pgdat->kswapd_order, 0);  in kswapd()
7758 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);  in kswapd()
7779 		trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,  in kswapd()
7781 		reclaim_order = balance_pgdat(pgdat, alloc_order,  in kswapd()
7802 	pg_data_t *pgdat;  in wakeup_kswapd()  local
7811 	pgdat = zone->zone_pgdat;  in wakeup_kswapd()
7812 	curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);  in wakeup_kswapd()
7815 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);  in wakeup_kswapd()
7817 	if (READ_ONCE(pgdat->kswapd_order) < order)  in wakeup_kswapd()
7818 		WRITE_ONCE(pgdat->kswapd_order, order);  in wakeup_kswapd()
7820 	if (!waitqueue_active(&pgdat->kswapd_wait))  in wakeup_kswapd()
7824 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||  in wakeup_kswapd()
7825 	    (pgdat_balanced(pgdat, order, highest_zoneidx) &&  in wakeup_kswapd()
7826 	     !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {  in wakeup_kswapd()
7835 			wakeup_kcompactd(pgdat, order, highest_zoneidx);  in wakeup_kswapd()
7839 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,  in wakeup_kswapd()
7841 	wake_up_interruptible(&pgdat->kswapd_wait);  in wakeup_kswapd()
7888 	pg_data_t *pgdat = NODE_DATA(nid);  in kswapd_run()  local
7890 	pgdat_kswapd_lock(pgdat);  in kswapd_run()
7891 	if (!pgdat->kswapd) {  in kswapd_run()
7892 		pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);  in kswapd_run()
7893 		if (IS_ERR(pgdat->kswapd)) {  in kswapd_run()
7897 			pgdat->kswapd = NULL;  in kswapd_run()
7900 	pgdat_kswapd_unlock(pgdat);  in kswapd_run()
7909 	pg_data_t *pgdat = NODE_DATA(nid);  in kswapd_stop()  local
7912 	pgdat_kswapd_lock(pgdat);  in kswapd_stop()
7913 	kswapd = pgdat->kswapd;  in kswapd_stop()
7916 		pgdat->kswapd = NULL;  in kswapd_stop()
7918 	pgdat_kswapd_unlock(pgdat);  in kswapd_stop()
7961 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)  in node_unmapped_file_pages()  argument
7963 	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);  in node_unmapped_file_pages()
7964 	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +  in node_unmapped_file_pages()
7965 		node_page_state(pgdat, NR_ACTIVE_FILE);  in node_unmapped_file_pages()
7976 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)  in node_pagecache_reclaimable()  argument
7988 		nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);  in node_pagecache_reclaimable()
7990 		nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);  in node_pagecache_reclaimable()
7994 		delta += node_page_state(pgdat, NR_FILE_DIRTY);  in node_pagecache_reclaimable()
8006 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)  in __node_reclaim()  argument
8024 	trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,  in __node_reclaim()
8036 	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages ||  in __node_reclaim()
8037 	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) {  in __node_reclaim()
8043 			shrink_node(pgdat, &sc);  in __node_reclaim()
8057 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)  in node_reclaim()  argument
8071 	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&  in node_reclaim()
8072 	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=  in node_reclaim()
8073 	    pgdat->min_slab_pages)  in node_reclaim()
8088 	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())  in node_reclaim()
8091 	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))  in node_reclaim()
8094 	ret = __node_reclaim(pgdat, gfp_mask, order);  in node_reclaim()
8095 	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);  in node_reclaim()