Lines Matching refs:lruvec
654 static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, in lruvec_lru_size() argument
661 struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid]; in lruvec_lru_size()
667 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); in lruvec_lru_size()
2250 static __always_inline void update_lru_sizes(struct lruvec *lruvec, in update_lru_sizes() argument
2259 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); in update_lru_sizes()
2305 struct lruvec *lruvec, struct list_head *dst, in isolate_lru_folios() argument
2309 struct list_head *src = &lruvec->lists[lru]; in isolate_lru_folios()
2394 update_lru_sizes(lruvec, lru, nr_zone_taken); in isolate_lru_folios()
2428 struct lruvec *lruvec; in folio_isolate_lru() local
2431 lruvec = folio_lruvec_lock_irq(folio); in folio_isolate_lru()
2432 lruvec_del_folio(lruvec, folio); in folio_isolate_lru()
2433 unlock_page_lruvec_irq(lruvec); in folio_isolate_lru()
2490 static unsigned int move_folios_to_lru(struct lruvec *lruvec, in move_folios_to_lru() argument
2502 spin_unlock_irq(&lruvec->lru_lock); in move_folios_to_lru()
2504 spin_lock_irq(&lruvec->lru_lock); in move_folios_to_lru()
2525 spin_unlock_irq(&lruvec->lru_lock); in move_folios_to_lru()
2527 spin_lock_irq(&lruvec->lru_lock); in move_folios_to_lru()
2538 VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); in move_folios_to_lru()
2539 lruvec_add_folio(lruvec, folio); in move_folios_to_lru()
2543 workingset_age_nonresident(lruvec, nr_pages); in move_folios_to_lru()
2569 struct lruvec *lruvec, struct scan_control *sc, in shrink_inactive_list() argument
2579 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in shrink_inactive_list()
2597 spin_lock_irq(&lruvec->lru_lock); in shrink_inactive_list()
2599 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, in shrink_inactive_list()
2606 __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); in shrink_inactive_list()
2609 spin_unlock_irq(&lruvec->lru_lock); in shrink_inactive_list()
2616 spin_lock_irq(&lruvec->lru_lock); in shrink_inactive_list()
2617 move_folios_to_lru(lruvec, &folio_list); in shrink_inactive_list()
2623 __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); in shrink_inactive_list()
2625 spin_unlock_irq(&lruvec->lru_lock); in shrink_inactive_list()
2627 lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed); in shrink_inactive_list()
2689 struct lruvec *lruvec, in shrink_active_list() argument
2702 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in shrink_active_list()
2706 spin_lock_irq(&lruvec->lru_lock); in shrink_active_list()
2708 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, in shrink_active_list()
2715 __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); in shrink_active_list()
2717 spin_unlock_irq(&lruvec->lru_lock); in shrink_active_list()
2766 spin_lock_irq(&lruvec->lru_lock); in shrink_active_list()
2768 nr_activate = move_folios_to_lru(lruvec, &l_active); in shrink_active_list()
2769 nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); in shrink_active_list()
2774 __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); in shrink_active_list()
2777 spin_unlock_irq(&lruvec->lru_lock); in shrink_active_list()
2780 lru_note_cost(lruvec, file, 0, nr_rotated); in shrink_active_list()
2845 struct lruvec *lruvec, struct scan_control *sc) in shrink_list() argument
2849 shrink_active_list(nr_to_scan, lruvec, sc, lru); in shrink_list()
2855 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); in shrink_list()
2886 static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) in inactive_is_low() argument
2893 inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); in inactive_is_low()
2894 active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); in inactive_is_low()
2915 struct lruvec *target_lruvec; in prepare_scan_count()
3025 static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, in get_scan_count() argument
3028 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in get_scan_count()
3029 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in get_scan_count()
3120 lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); in get_scan_count()
3263 #define DEFINE_MAX_SEQ(lruvec) \ argument
3264 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
3266 #define DEFINE_MIN_SEQ(lruvec) \ argument
3268 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
3269 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
3280 static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) in get_lruvec()
3286 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; in get_lruvec() local
3289 if (!lruvec->pgdat) in get_lruvec()
3290 lruvec->pgdat = pgdat; in get_lruvec()
3292 return lruvec; in get_lruvec()
3300 static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) in get_swappiness() argument
3302 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in get_swappiness()
3303 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in get_swappiness()
3315 static int get_nr_gens(struct lruvec *lruvec, int type) in get_nr_gens() argument
3317 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; in get_nr_gens()
3320 static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) in seq_is_valid() argument
3323 return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && in seq_is_valid()
3324 get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && in seq_is_valid()
3325 get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; in seq_is_valid()
3371 static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) in test_bloom_filter() argument
3377 filter = READ_ONCE(lruvec->mm_state.filters[gen]); in test_bloom_filter()
3386 static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) in update_bloom_filter() argument
3392 filter = READ_ONCE(lruvec->mm_state.filters[gen]); in update_bloom_filter()
3404 static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) in reset_bloom_filter() argument
3409 filter = lruvec->mm_state.filters[gen]; in reset_bloom_filter()
3417 WRITE_ONCE(lruvec->mm_state.filters[gen], filter); in reset_bloom_filter()
3454 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_add_mm() local
3457 if (lruvec->mm_state.tail == &mm_list->fifo) in lru_gen_add_mm()
3458 lruvec->mm_state.tail = &mm->lru_gen.list; in lru_gen_add_mm()
3483 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_del_mm() local
3486 if (lruvec->mm_state.head == &mm->lru_gen.list) in lru_gen_del_mm()
3487 lruvec->mm_state.head = lruvec->mm_state.head->prev; in lru_gen_del_mm()
3490 if (lruvec->mm_state.tail == &mm->lru_gen.list) in lru_gen_del_mm()
3491 lruvec->mm_state.tail = lruvec->mm_state.tail->next; in lru_gen_del_mm()
3534 static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) in reset_mm_stats() argument
3539 lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); in reset_mm_stats()
3545 WRITE_ONCE(lruvec->mm_state.stats[hist][i], in reset_mm_stats()
3546 lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); in reset_mm_stats()
3552 hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); in reset_mm_stats()
3555 WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); in reset_mm_stats()
3563 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); in should_skip_mm()
3583 static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, in iterate_mm_list() argument
3589 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in iterate_mm_list()
3591 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; in iterate_mm_list()
3636 reset_mm_stats(lruvec, walk, last); in iterate_mm_list()
3641 reset_bloom_filter(lruvec, walk->max_seq + 1); in iterate_mm_list()
3651 static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) in iterate_mm_list_nowalk() argument
3654 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in iterate_mm_list_nowalk()
3656 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; in iterate_mm_list_nowalk()
3666 reset_mm_stats(lruvec, NULL, true); in iterate_mm_list_nowalk()
3704 static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, in read_ctrl_pos() argument
3707 struct lru_gen_folio *lrugen = &lruvec->lrugen; in read_ctrl_pos()
3719 static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) in reset_ctrl_pos() argument
3722 struct lru_gen_folio *lrugen = &lruvec->lrugen; in reset_ctrl_pos()
3726 lockdep_assert_held(&lruvec->lru_lock); in reset_ctrl_pos()
3796 static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) in folio_inc_gen() argument
3799 struct lru_gen_folio *lrugen = &lruvec->lrugen; in folio_inc_gen()
3820 lru_gen_update_size(lruvec, folio, old_gen, new_gen); in folio_inc_gen()
3841 static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) in reset_batch_size() argument
3844 struct lru_gen_folio *lrugen = &lruvec->lrugen; in reset_batch_size()
3859 if (lru_gen_is_active(lruvec, gen)) in reset_batch_size()
3861 __update_lru_size(lruvec, lru, zone, delta); in reset_batch_size()
4013 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); in walk_pte_range()
4014 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); in walk_pte_range()
4081 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); in walk_pmd_range_locked()
4082 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); in walk_pmd_range_locked()
4194 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); in walk_pmd_range()
4220 if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) in walk_pmd_range()
4231 update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); in walk_pmd_range()
4282 static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) in walk_mm() argument
4291 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in walk_mm()
4296 DEFINE_MAX_SEQ(lruvec); in walk_mm()
4318 spin_lock_irq(&lruvec->lru_lock); in walk_mm()
4319 reset_batch_size(lruvec, walk); in walk_mm()
4320 spin_unlock_irq(&lruvec->lru_lock); in walk_mm()
4359 static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) in inc_min_seq() argument
4363 struct lru_gen_folio *lrugen = &lruvec->lrugen; in inc_min_seq()
4381 new_gen = folio_inc_gen(lruvec, folio, false); in inc_min_seq()
4389 reset_ctrl_pos(lruvec, type, true); in inc_min_seq()
4395 static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) in try_to_inc_min_seq() argument
4399 struct lru_gen_folio *lrugen = &lruvec->lrugen; in try_to_inc_min_seq()
4400 DEFINE_MIN_SEQ(lruvec); in try_to_inc_min_seq()
4402 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); in try_to_inc_min_seq()
4430 reset_ctrl_pos(lruvec, type, true); in try_to_inc_min_seq()
4438 static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) in inc_max_seq() argument
4442 struct lru_gen_folio *lrugen = &lruvec->lrugen; in inc_max_seq()
4444 spin_lock_irq(&lruvec->lru_lock); in inc_max_seq()
4446 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); in inc_max_seq()
4449 if (get_nr_gens(lruvec, type) != MAX_NR_GENS) in inc_max_seq()
4454 if (inc_min_seq(lruvec, type, can_swap)) in inc_max_seq()
4457 spin_unlock_irq(&lruvec->lru_lock); in inc_max_seq()
4480 __update_lru_size(lruvec, lru, zone, delta); in inc_max_seq()
4481 __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); in inc_max_seq()
4486 reset_ctrl_pos(lruvec, type, false); in inc_max_seq()
4492 spin_unlock_irq(&lruvec->lru_lock); in inc_max_seq()
4495 static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, in try_to_inc_max_seq() argument
4501 struct lru_gen_folio *lrugen = &lruvec->lrugen; in try_to_inc_max_seq()
4506 if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { in try_to_inc_max_seq()
4518 success = iterate_mm_list_nowalk(lruvec, max_seq); in try_to_inc_max_seq()
4524 success = iterate_mm_list_nowalk(lruvec, max_seq); in try_to_inc_max_seq()
4528 walk->lruvec = lruvec; in try_to_inc_max_seq()
4534 success = iterate_mm_list(lruvec, walk, &mm); in try_to_inc_max_seq()
4536 walk_mm(lruvec, mm, walk); in try_to_inc_max_seq()
4540 inc_max_seq(lruvec, can_swap, force_scan); in try_to_inc_max_seq()
4549 static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc) in lruvec_is_sizable() argument
4553 bool can_swap = get_swappiness(lruvec, sc); in lruvec_is_sizable()
4554 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lruvec_is_sizable()
4555 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in lruvec_is_sizable()
4556 DEFINE_MAX_SEQ(lruvec); in lruvec_is_sizable()
4557 DEFINE_MIN_SEQ(lruvec); in lruvec_is_sizable()
4574 static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc, in lruvec_is_reclaimable() argument
4579 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in lruvec_is_reclaimable()
4580 DEFINE_MIN_SEQ(lruvec); in lruvec_is_reclaimable()
4584 birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); in lruvec_is_reclaimable()
4589 if (!lruvec_is_sizable(lruvec, sc)) in lruvec_is_reclaimable()
4613 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); in lru_gen_age_node() local
4615 if (lruvec_is_reclaimable(lruvec, sc, min_ttl)) { in lru_gen_age_node()
4663 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); in lru_gen_look_around() local
4664 DEFINE_MAX_SEQ(lruvec); in lru_gen_look_around()
4743 update_bloom_filter(lruvec, max_seq, pvmw->pmd); in lru_gen_look_around()
4761 static int lru_gen_memcg_seg(struct lruvec *lruvec) in lru_gen_memcg_seg() argument
4763 return READ_ONCE(lruvec->lrugen.seg); in lru_gen_memcg_seg()
4766 static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) in lru_gen_rotate_memcg() argument
4772 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in lru_gen_rotate_memcg()
4776 VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list)); in lru_gen_rotate_memcg()
4779 new = old = lruvec->lrugen.gen; in lru_gen_rotate_memcg()
4793 hlist_nulls_del_rcu(&lruvec->lrugen.list); in lru_gen_rotate_memcg()
4796 hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); in lru_gen_rotate_memcg()
4798 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); in lru_gen_rotate_memcg()
4803 lruvec->lrugen.gen = new; in lru_gen_rotate_memcg()
4804 WRITE_ONCE(lruvec->lrugen.seg, seg); in lru_gen_rotate_memcg()
4820 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_online_memcg() local
4824 VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list)); in lru_gen_online_memcg()
4828 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]); in lru_gen_online_memcg()
4831 lruvec->lrugen.gen = gen; in lru_gen_online_memcg()
4842 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_offline_memcg() local
4844 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD); in lru_gen_offline_memcg()
4855 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_release_memcg() local
4859 if (hlist_nulls_unhashed(&lruvec->lrugen.list)) in lru_gen_release_memcg()
4862 gen = lruvec->lrugen.gen; in lru_gen_release_memcg()
4864 hlist_nulls_del_init_rcu(&lruvec->lrugen.list); in lru_gen_release_memcg()
4876 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_soft_reclaim() local
4879 if (lru_gen_memcg_seg(lruvec) != MEMCG_LRU_HEAD) in lru_gen_soft_reclaim()
4880 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD); in lru_gen_soft_reclaim()
4885 static int lru_gen_memcg_seg(struct lruvec *lruvec) in lru_gen_memcg_seg() argument
4896 static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, in sort_folio() argument
4906 struct lru_gen_folio *lrugen = &lruvec->lrugen; in sort_folio()
4912 success = lru_gen_del_folio(lruvec, folio, true); in sort_folio()
4915 lruvec_add_folio(lruvec, folio); in sort_folio()
4922 success = lru_gen_del_folio(lruvec, folio, true); in sort_folio()
4925 lruvec_add_folio_tail(lruvec, folio); in sort_folio()
4939 gen = folio_inc_gen(lruvec, folio, false); in sort_folio()
4949 gen = folio_inc_gen(lruvec, folio, false); in sort_folio()
4957 gen = folio_inc_gen(lruvec, folio, true); in sort_folio()
4965 static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) in isolate_folio() argument
4993 success = lru_gen_del_folio(lruvec, folio, true); in isolate_folio()
4999 static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, in scan_folios() argument
5009 struct lru_gen_folio *lrugen = &lruvec->lrugen; in scan_folios()
5010 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in scan_folios()
5014 if (get_nr_gens(lruvec, type) == MIN_NR_GENS) in scan_folios()
5036 if (sort_folio(lruvec, folio, sc, tier)) in scan_folios()
5038 else if (isolate_folio(lruvec, folio, sc)) { in scan_folios()
5075 static int get_tier_idx(struct lruvec *lruvec, int type) in get_tier_idx() argument
5085 read_ctrl_pos(lruvec, type, 0, 1, &sp); in get_tier_idx()
5087 read_ctrl_pos(lruvec, type, tier, 2, &pv); in get_tier_idx()
5095 static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) in get_type_to_scan() argument
5107 read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); in get_type_to_scan()
5108 read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); in get_type_to_scan()
5111 read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); in get_type_to_scan()
5113 read_ctrl_pos(lruvec, type, tier, gain[type], &pv); in get_type_to_scan()
5123 static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, in isolate_folios() argument
5130 DEFINE_MIN_SEQ(lruvec); in isolate_folios()
5146 type = get_type_to_scan(lruvec, swappiness, &tier); in isolate_folios()
5150 tier = get_tier_idx(lruvec, type); in isolate_folios()
5152 scanned = scan_folios(lruvec, sc, type, tier, list); in isolate_folios()
5165 static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness) in evict_folios() argument
5178 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in evict_folios()
5179 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in evict_folios()
5181 spin_lock_irq(&lruvec->lru_lock); in evict_folios()
5183 scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); in evict_folios()
5185 scanned += try_to_inc_min_seq(lruvec, swappiness); in evict_folios()
5187 if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) in evict_folios()
5190 spin_unlock_irq(&lruvec->lru_lock); in evict_folios()
5227 spin_lock_irq(&lruvec->lru_lock); in evict_folios()
5229 move_folios_to_lru(lruvec, &list); in evict_folios()
5233 reset_batch_size(lruvec, walk); in evict_folios()
5241 spin_unlock_irq(&lruvec->lru_lock); in evict_folios()
5257 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, in should_run_aging() argument
5264 struct lru_gen_folio *lrugen = &lruvec->lrugen; in should_run_aging()
5265 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in should_run_aging()
5266 DEFINE_MIN_SEQ(lruvec); in should_run_aging()
5324 static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap) in get_nr_to_scan() argument
5327 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in get_nr_to_scan()
5328 DEFINE_MAX_SEQ(lruvec); in get_nr_to_scan()
5333 if (!should_run_aging(lruvec, max_seq, sc, can_swap, &nr_to_scan)) in get_nr_to_scan()
5341 return try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false) ? -1 : 0; in get_nr_to_scan()
5353 static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) in try_to_shrink_lruvec() argument
5358 int swappiness = get_swappiness(lruvec, sc); in try_to_shrink_lruvec()
5367 nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness); in try_to_shrink_lruvec()
5371 delta = evict_folios(lruvec, sc, swappiness); in try_to_shrink_lruvec()
5389 static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) in shrink_one() argument
5394 int seg = lru_gen_memcg_seg(lruvec); in shrink_one()
5395 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in shrink_one()
5396 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in shrink_one()
5399 if (!lruvec_is_sizable(lruvec, sc)) in shrink_one()
5415 success = try_to_shrink_lruvec(lruvec, sc); in shrink_one()
5436 struct lruvec *lruvec; in shrink_many() local
5452 lru_gen_rotate_memcg(lruvec, op); in shrink_many()
5458 lruvec = container_of(lrugen, struct lruvec, lrugen); in shrink_many()
5459 memcg = lruvec_memcg(lruvec); in shrink_many()
5469 op = shrink_one(lruvec, sc); in shrink_many()
5480 lru_gen_rotate_memcg(lruvec, op); in shrink_many()
5497 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) in lru_gen_shrink_lruvec() argument
5510 if (try_to_shrink_lruvec(lruvec, sc)) in lru_gen_shrink_lruvec()
5511 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG); in lru_gen_shrink_lruvec()
5525 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) in lru_gen_shrink_lruvec() argument
5536 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); in set_initial_priority() local
5546 if (get_swappiness(lruvec, sc)) in set_initial_priority()
5603 static bool __maybe_unused state_is_valid(struct lruvec *lruvec) in state_is_valid() argument
5605 struct lru_gen_folio *lrugen = &lruvec->lrugen; in state_is_valid()
5611 if (!list_empty(&lruvec->lists[lru])) in state_is_valid()
5626 static bool fill_evictable(struct lruvec *lruvec) in fill_evictable() argument
5634 struct list_head *head = &lruvec->lists[lru]; in fill_evictable()
5645 lruvec_del_folio(lruvec, folio); in fill_evictable()
5646 success = lru_gen_add_folio(lruvec, folio, false); in fill_evictable()
5657 static bool drain_evictable(struct lruvec *lruvec) in drain_evictable() argument
5663 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; in drain_evictable()
5674 success = lru_gen_del_folio(lruvec, folio, false); in drain_evictable()
5676 lruvec_add_folio(lruvec, folio); in drain_evictable()
5710 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_change_state() local
5712 spin_lock_irq(&lruvec->lru_lock); in lru_gen_change_state()
5714 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); in lru_gen_change_state()
5715 VM_WARN_ON_ONCE(!state_is_valid(lruvec)); in lru_gen_change_state()
5717 lruvec->lrugen.enabled = enabled; in lru_gen_change_state()
5719 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { in lru_gen_change_state()
5720 spin_unlock_irq(&lruvec->lru_lock); in lru_gen_change_state()
5722 spin_lock_irq(&lruvec->lru_lock); in lru_gen_change_state()
5725 spin_unlock_irq(&lruvec->lru_lock); in lru_gen_change_state()
5873 static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, in lru_gen_seq_show_full() argument
5880 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lru_gen_seq_show_full()
5913 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); in lru_gen_seq_show_full()
5916 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); in lru_gen_seq_show_full()
5929 struct lruvec *lruvec = v; in lru_gen_seq_show() local
5930 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lru_gen_seq_show()
5931 int nid = lruvec_pgdat(lruvec)->node_id; in lru_gen_seq_show()
5932 struct mem_cgroup *memcg = lruvec_memcg(lruvec); in lru_gen_seq_show()
5933 DEFINE_MAX_SEQ(lruvec); in lru_gen_seq_show()
5934 DEFINE_MIN_SEQ(lruvec); in lru_gen_seq_show()
5958 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); in lru_gen_seq_show()
5975 lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); in lru_gen_seq_show()
5988 static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, in run_aging() argument
5991 DEFINE_MAX_SEQ(lruvec); in run_aging()
5992 DEFINE_MIN_SEQ(lruvec); in run_aging()
6003 try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); in run_aging()
6008 static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, in run_eviction() argument
6011 DEFINE_MAX_SEQ(lruvec); in run_eviction()
6019 DEFINE_MIN_SEQ(lruvec); in run_eviction()
6027 if (!evict_folios(lruvec, sc, swappiness)) in run_eviction()
6039 struct lruvec *lruvec; in run_cmd() local
6062 lruvec = get_lruvec(memcg, nid); in run_cmd()
6065 swappiness = get_swappiness(lruvec, sc); in run_cmd()
6071 err = run_aging(lruvec, seq, sc, swappiness, opt); in run_cmd()
6074 err = run_eviction(lruvec, seq, sc, swappiness, opt); in run_cmd()
6180 void lru_gen_init_lruvec(struct lruvec *lruvec) in lru_gen_init_lruvec() argument
6184 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lru_gen_init_lruvec()
6195 lruvec->mm_state.seq = MIN_NR_GENS; in lru_gen_init_lruvec()
6226 struct lruvec *lruvec = get_lruvec(memcg, nid); in lru_gen_exit_memcg() local
6228 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, in lru_gen_exit_memcg()
6229 sizeof(lruvec->lrugen.nr_pages))); in lru_gen_exit_memcg()
6231 lruvec->lrugen.list.next = LIST_POISON1; in lru_gen_exit_memcg()
6234 bitmap_free(lruvec->mm_state.filters[i]); in lru_gen_exit_memcg()
6235 lruvec->mm_state.filters[i] = NULL; in lru_gen_exit_memcg()
6263 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) in lru_gen_shrink_lruvec() argument
6273 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) in shrink_lruvec() argument
6285 lru_gen_shrink_lruvec(lruvec, sc); in shrink_lruvec()
6289 get_scan_count(lruvec, sc, nr); in shrink_lruvec()
6320 lruvec, sc); in shrink_lruvec()
6385 if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && in shrink_lruvec()
6386 inactive_is_low(lruvec, LRU_INACTIVE_ANON)) in shrink_lruvec()
6387 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, in shrink_lruvec()
6468 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); in shrink_node_memcgs() local
6505 shrink_lruvec(lruvec, sc); in shrink_node_memcgs()
6522 struct lruvec *target_lruvec; in shrink_node()
6794 struct lruvec *target_lruvec; in snapshot_refaults()
6867 struct lruvec *lruvec; in do_try_to_free_pages() local
6869 lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, in do_try_to_free_pages()
6871 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); in do_try_to_free_pages()
7092 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); in mem_cgroup_shrink_node() local
7117 shrink_lruvec(lruvec, &sc); in mem_cgroup_shrink_node()
7169 struct lruvec *lruvec; in kswapd_age_node() local
7179 lruvec = mem_cgroup_lruvec(NULL, pgdat); in kswapd_age_node()
7180 if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) in kswapd_age_node()
7185 lruvec = mem_cgroup_lruvec(memcg, pgdat); in kswapd_age_node()
7186 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, in kswapd_age_node()
7258 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); in clear_pgdat_congested() local
7260 clear_bit(LRUVEC_NODE_CONGESTED, &lruvec->flags); in clear_pgdat_congested()
7261 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); in clear_pgdat_congested()
8115 struct lruvec *lruvec = NULL; in check_move_unevictable_folios() local
8130 lruvec = folio_lruvec_relock_irq(folio, lruvec); in check_move_unevictable_folios()
8132 lruvec_del_folio(lruvec, folio); in check_move_unevictable_folios()
8134 lruvec_add_folio(lruvec, folio); in check_move_unevictable_folios()
8140 if (lruvec) { in check_move_unevictable_folios()
8143 unlock_page_lruvec_irq(lruvec); in check_move_unevictable_folios()