208 static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,  in shrinker_info_protected()  argument
211 	return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,  in shrinker_info_protected()
215 static int expand_one_shrinker_info(struct mem_cgroup *memcg,  in expand_one_shrinker_info()  argument
226 		pn = memcg->nodeinfo[nid];  in expand_one_shrinker_info()
227 		old = shrinker_info_protected(memcg, nid);  in expand_one_shrinker_info()
259 void free_shrinker_info(struct mem_cgroup *memcg)  in free_shrinker_info()  argument
266 		pn = memcg->nodeinfo[nid];  in free_shrinker_info()
273 int alloc_shrinker_info(struct mem_cgroup *memcg)  in alloc_shrinker_info()  argument
286 			free_shrinker_info(memcg);  in alloc_shrinker_info()
293 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);  in alloc_shrinker_info()
306 	struct mem_cgroup *memcg;  in expand_shrinker_info()  local
318 	memcg = mem_cgroup_iter(NULL, NULL, NULL);  in expand_shrinker_info()
320 		ret = expand_one_shrinker_info(memcg, map_size, defer_size,  in expand_shrinker_info()
324 			mem_cgroup_iter_break(NULL, memcg);  in expand_shrinker_info()
327 	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);  in expand_shrinker_info()
335 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)  in set_shrinker_bit()  argument
337 	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {  in set_shrinker_bit()
341 		info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);  in set_shrinker_bit()
391 				   struct mem_cgroup *memcg)  in xchg_nr_deferred_memcg()  argument
395 	info = shrinker_info_protected(memcg, nid);  in xchg_nr_deferred_memcg()
400 				  struct mem_cgroup *memcg)  in add_nr_deferred_memcg()  argument
404 	info = shrinker_info_protected(memcg, nid);  in add_nr_deferred_memcg()
408 void reparent_shrinker_deferred(struct mem_cgroup *memcg)  in reparent_shrinker_deferred()  argument
415 	parent = parent_mem_cgroup(memcg);  in reparent_shrinker_deferred()
422 		child_info = shrinker_info_protected(memcg, nid);  in reparent_shrinker_deferred()
481 				   struct mem_cgroup *memcg)  in xchg_nr_deferred_memcg()  argument
487 				  struct mem_cgroup *memcg)  in add_nr_deferred_memcg()  argument
568 	if (sc->memcg &&  in xchg_nr_deferred()
571 					      sc->memcg);  in xchg_nr_deferred()
585 	if (sc->memcg &&  in add_nr_deferred()
588 					     sc->memcg);  in add_nr_deferred()
605 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,  in can_reclaim_anon_pages()  argument
609 	if (memcg == NULL) {  in can_reclaim_anon_pages()
618 		if (mem_cgroup_get_nr_swap_pages(memcg) > 0)  in can_reclaim_anon_pages()
933 			struct mem_cgroup *memcg, int priority)  in shrink_slab_memcg()  argument
939 	if (!mem_cgroup_online(memcg))  in shrink_slab_memcg()
945 	info = shrinker_info_protected(memcg, nid);  in shrink_slab_memcg()
953 			.memcg = memcg,  in shrink_slab_memcg()
992 				set_shrinker_bit(memcg, nid, i);  in shrink_slab_memcg()
1007 			struct mem_cgroup *memcg, int priority)  in shrink_slab_memcg()  argument
1034 				 struct mem_cgroup *memcg,  in shrink_slab()  argument
1047 	if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))  in shrink_slab()
1048 		return shrink_slab_memcg(gfp_mask, nid, memcg, priority);  in shrink_slab()
1057 			.memcg = memcg,  in shrink_slab()
1084 	struct mem_cgroup *memcg = NULL;  in drop_slab_node()  local
1086 	memcg = mem_cgroup_iter(NULL, NULL, NULL);  in drop_slab_node()
1088 		freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);  in drop_slab_node()
1089 	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);  in drop_slab_node()
3029 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in get_scan_count()  local
3031 	int swappiness = mem_cgroup_swappiness(memcg);  in get_scan_count()
3039 	if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {  in get_scan_count()
3121 		mem_cgroup_protection(sc->target_mem_cgroup, memcg,  in get_scan_count()
3154 			unsigned long cgroup_size = mem_cgroup_size(memcg);  in get_scan_count()
3187 		if (!scan && !mem_cgroup_online(memcg))  in get_scan_count()
3202 			scan = mem_cgroup_online(memcg) ?  in get_scan_count()
3280 static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)  in get_lruvec()  argument
3285 	if (memcg) {  in get_lruvec()
3286 		struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;  in get_lruvec()
3302 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in get_swappiness()  local
3309 	    mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)  in get_swappiness()
3312 	return mem_cgroup_swappiness(memcg);  in get_swappiness()
3424 static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg)  in get_mm_list()  argument
3432 	if (memcg)  in get_mm_list()
3433 		return &memcg->mm_list;  in get_mm_list()
3443 	struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);  in lru_gen_add_mm()  local
3444 	struct lru_gen_mm_list *mm_list = get_mm_list(memcg);  in lru_gen_add_mm()
3448 	VM_WARN_ON_ONCE(mm->lru_gen.memcg);  in lru_gen_add_mm()
3449 	mm->lru_gen.memcg = memcg;  in lru_gen_add_mm()
3454 		struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_add_mm()
3470 	struct mem_cgroup *memcg = NULL;  in lru_gen_del_mm()  local
3476 	memcg = mm->lru_gen.memcg;  in lru_gen_del_mm()
3478 	mm_list = get_mm_list(memcg);  in lru_gen_del_mm()
3483 		struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_del_mm()
3499 	mem_cgroup_put(mm->lru_gen.memcg);  in lru_gen_del_mm()
3500 	mm->lru_gen.memcg = NULL;  in lru_gen_del_mm()
3507 	struct mem_cgroup *memcg;  in lru_gen_migrate_mm()  local
3518 	if (!mm->lru_gen.memcg)  in lru_gen_migrate_mm()
3522 	memcg = mem_cgroup_from_task(task);  in lru_gen_migrate_mm()
3524 	if (memcg == mm->lru_gen.memcg)  in lru_gen_migrate_mm()
3589 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in iterate_mm_list()  local
3590 	struct lru_gen_mm_list *mm_list = get_mm_list(memcg);  in iterate_mm_list()
3654 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in iterate_mm_list_nowalk()  local
3655 	struct lru_gen_mm_list *mm_list = get_mm_list(memcg);  in iterate_mm_list_nowalk()
3972 static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg,  in get_pfn_folio()  argument
3985 	if (folio_memcg_rcu(folio) != memcg)  in get_pfn_folio()
4013 	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);  in walk_pte_range()  local
4044 		folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);  in walk_pte_range()
4081 	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);  in walk_pmd_range_locked()  local
4125 		folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);  in walk_pmd_range_locked()
4291 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in walk_mm()  local
4305 		if (!mem_cgroup_trylock_pages(memcg))  in walk_mm()
4555 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in lruvec_is_sizable()  local
4571 	return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;  in lruvec_is_sizable()
4579 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in lruvec_is_reclaimable()  local
4592 	mem_cgroup_calculate_protection(NULL, memcg);  in lruvec_is_reclaimable()
4594 	return !mem_cgroup_below_min(NULL, memcg);  in lruvec_is_reclaimable()
4602 	struct mem_cgroup *memcg;  in lru_gen_age_node()  local
4611 	memcg = mem_cgroup_iter(NULL, NULL, NULL);  in lru_gen_age_node()
4613 		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in lru_gen_age_node()
4616 			mem_cgroup_iter_break(NULL, memcg);  in lru_gen_age_node()
4621 	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));  in lru_gen_age_node()
4661 	struct mem_cgroup *memcg = folio_memcg(folio);  in lru_gen_look_around()  local
4663 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in lru_gen_look_around()
4691 	if (!mem_cgroup_trylock_pages(memcg))  in lru_gen_look_around()
4709 		folio = get_pfn_folio(pfn, memcg, pgdat, can_swap);  in lru_gen_look_around()
4812 void lru_gen_online_memcg(struct mem_cgroup *memcg)  in lru_gen_online_memcg()  argument
4820 		struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_online_memcg()
4837 void lru_gen_offline_memcg(struct mem_cgroup *memcg)  in lru_gen_offline_memcg()  argument
4842 		struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_offline_memcg()
4848 void lru_gen_release_memcg(struct mem_cgroup *memcg)  in lru_gen_release_memcg()  argument
4855 		struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_release_memcg()
4874 void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)  in lru_gen_soft_reclaim()  argument
4876 	struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_soft_reclaim()
5010 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in scan_folios()  local
5064 	__count_memcg_events(memcg, item, isolated);  in scan_folios()
5065 	__count_memcg_events(memcg, PGREFILL, sorted);  in scan_folios()
5178 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in evict_folios()  local
5238 	__count_memcg_events(memcg, item, reclaimed);  in evict_folios()
5265 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in should_run_aging()  local
5294 	*nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total;  in should_run_aging()
5327 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in get_nr_to_scan()  local
5330 	if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg))  in get_nr_to_scan()
5395 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in shrink_one()  local
5402 	mem_cgroup_calculate_protection(NULL, memcg);  in shrink_one()
5404 	if (mem_cgroup_below_min(NULL, memcg))  in shrink_one()
5407 	if (mem_cgroup_below_low(NULL, memcg)) {  in shrink_one()
5412 		memcg_memory_event(memcg, MEMCG_LOW);  in shrink_one()
5417 	shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);  in shrink_one()
5420 		vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned,  in shrink_one()
5438 	struct mem_cgroup *memcg;  in shrink_many()  local
5445 	memcg = NULL;  in shrink_many()
5456 		mem_cgroup_put(memcg);  in shrink_many()
5459 		memcg = lruvec_memcg(lruvec);  in shrink_many()
5461 		if (!mem_cgroup_tryget(memcg)) {  in shrink_many()
5462 			lru_gen_release_memcg(memcg);  in shrink_many()
5463 			memcg = NULL;  in shrink_many()
5482 	mem_cgroup_put(memcg);  in shrink_many()
5690 	struct mem_cgroup *memcg;  in lru_gen_change_state()  local
5705 	memcg = mem_cgroup_iter(NULL, NULL, NULL);  in lru_gen_change_state()
5710 			struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_change_state()
5729 	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));  in lru_gen_change_state()
5825 	struct mem_cgroup *memcg;  in lru_gen_seq_start()  local
5832 	memcg = mem_cgroup_iter(NULL, NULL, NULL);  in lru_gen_seq_start()
5838 				return get_lruvec(memcg, nid);  in lru_gen_seq_start()
5840 	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));  in lru_gen_seq_start()
5857 	struct mem_cgroup *memcg = lruvec_memcg(v);  in lru_gen_seq_next()  local
5863 		memcg = mem_cgroup_iter(NULL, memcg, NULL);  in lru_gen_seq_next()
5864 		if (!memcg)  in lru_gen_seq_next()
5870 	return get_lruvec(memcg, nid);  in lru_gen_seq_next()
5932 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);  in lru_gen_seq_show()  local
5937 		const char *path = memcg ? m->private : "";  in lru_gen_seq_show()
5940 		if (memcg)  in lru_gen_seq_show()
5941 			cgroup_path(memcg->css.cgroup, m->private, PATH_MAX);  in lru_gen_seq_show()
5943 		seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path);  in lru_gen_seq_show()
6041 	struct mem_cgroup *memcg = NULL;  in run_cmd()  local
6049 		memcg = mem_cgroup_from_id(memcg_id);  in run_cmd()
6050 		if (!mem_cgroup_tryget(memcg))  in run_cmd()
6051 			memcg = NULL;  in run_cmd()
6055 		if (!memcg)  in run_cmd()
6059 	if (memcg_id != mem_cgroup_id(memcg))  in run_cmd()
6062 	lruvec = get_lruvec(memcg, nid);  in run_cmd()
6078 	mem_cgroup_put(memcg);  in run_cmd()
6212 void lru_gen_init_memcg(struct mem_cgroup *memcg)  in lru_gen_init_memcg()  argument
6214 	INIT_LIST_HEAD(&memcg->mm_list.fifo);  in lru_gen_init_memcg()
6215 	spin_lock_init(&memcg->mm_list.lock);  in lru_gen_init_memcg()
6218 void lru_gen_exit_memcg(struct mem_cgroup *memcg)  in lru_gen_exit_memcg()  argument
6223 	VM_WARN_ON_ONCE(!list_empty(&memcg->mm_list.fifo));  in lru_gen_exit_memcg()
6226 		struct lruvec *lruvec = get_lruvec(memcg, nid);  in lru_gen_exit_memcg()
6464 	struct mem_cgroup *memcg;  in shrink_node_memcgs()  local
6466 	memcg = mem_cgroup_iter(target_memcg, NULL, NULL);  in shrink_node_memcgs()
6468 		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in shrink_node_memcgs()
6480 		mem_cgroup_calculate_protection(target_memcg, memcg);  in shrink_node_memcgs()
6482 		if (mem_cgroup_below_min(target_memcg, memcg)) {  in shrink_node_memcgs()
6488 		} else if (mem_cgroup_below_low(target_memcg, memcg)) {  in shrink_node_memcgs()
6499 			memcg_memory_event(memcg, MEMCG_LOW);  in shrink_node_memcgs()
6507 		shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,  in shrink_node_memcgs()
6512 			vmpressure(sc->gfp_mask, memcg, false,  in shrink_node_memcgs()
6516 	} while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));  in shrink_node_memcgs()
7087 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,  in mem_cgroup_shrink_node()  argument
7092 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);  in mem_cgroup_shrink_node()
7095 		.target_mem_cgroup = memcg,  in mem_cgroup_shrink_node()
7126 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,  in try_to_free_mem_cgroup_pages()  argument
7138 		.target_mem_cgroup = memcg,  in try_to_free_mem_cgroup_pages()
7168 	struct mem_cgroup *memcg;  in kswapd_age_node()  local
7183 	memcg = mem_cgroup_iter(NULL, NULL, NULL);  in kswapd_age_node()
7185 		lruvec = mem_cgroup_lruvec(memcg, pgdat);  in kswapd_age_node()
7188 		memcg = mem_cgroup_iter(NULL, memcg, NULL);  in kswapd_age_node()
7189 	} while (memcg);  in kswapd_age_node()