Lines Matching refs:memcg
138 struct mem_cgroup *memcg; member
152 int (*register_event)(struct mem_cgroup *memcg,
159 void (*unregister_event)(struct mem_cgroup *memcg,
171 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
172 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
240 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) in memcg_to_vmpressure() argument
242 if (!memcg) in memcg_to_vmpressure()
243 memcg = root_mem_cgroup; in memcg_to_vmpressure()
244 return &memcg->vmpressure; in memcg_to_vmpressure()
324 static void memcg_reparent_objcgs(struct mem_cgroup *memcg, in memcg_reparent_objcgs() argument
329 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); in memcg_reparent_objcgs()
334 list_add(&objcg->list, &memcg->objcg_list); in memcg_reparent_objcgs()
336 list_for_each_entry(iter, &memcg->objcg_list, list) in memcg_reparent_objcgs()
337 WRITE_ONCE(iter->memcg, parent); in memcg_reparent_objcgs()
339 list_splice(&memcg->objcg_list, &parent->objcg_list); in memcg_reparent_objcgs()
372 struct mem_cgroup *memcg = folio_memcg(folio); in mem_cgroup_css_from_folio() local
374 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) in mem_cgroup_css_from_folio()
375 memcg = root_mem_cgroup; in mem_cgroup_css_from_folio()
377 return &memcg->css; in mem_cgroup_css_from_folio()
395 struct mem_cgroup *memcg; in page_cgroup_ino() local
400 memcg = folio_memcg_check(page_folio(page)); in page_cgroup_ino()
402 while (memcg && !(memcg->css.flags & CSS_ONLINE)) in page_cgroup_ino()
403 memcg = parent_mem_cgroup(memcg); in page_cgroup_ino()
404 if (memcg) in page_cgroup_ino()
405 ino = cgroup_ino(memcg->css.cgroup); in page_cgroup_ino()
468 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) in soft_limit_excess() argument
470 unsigned long nr_pages = page_counter_read(&memcg->memory); in soft_limit_excess()
471 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); in soft_limit_excess()
480 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid) in mem_cgroup_update_tree() argument
487 if (soft_limit_excess(memcg)) in mem_cgroup_update_tree()
488 lru_gen_soft_reclaim(memcg, nid); in mem_cgroup_update_tree()
499 for (; memcg; memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_update_tree()
500 mz = memcg->nodeinfo[nid]; in mem_cgroup_update_tree()
501 excess = soft_limit_excess(memcg); in mem_cgroup_update_tree()
523 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) in mem_cgroup_remove_from_trees() argument
530 mz = memcg->nodeinfo[nid]; in mem_cgroup_remove_from_trees()
555 if (!soft_limit_excess(mz->memcg) || in __mem_cgroup_largest_soft_limit_node()
556 !css_tryget(&mz->memcg->css)) in __mem_cgroup_largest_soft_limit_node()
619 static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) in memcg_rstat_updated() argument
626 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); in memcg_rstat_updated()
754 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) in memcg_page_state() argument
756 long x = READ_ONCE(memcg->vmstats->state[idx]); in memcg_page_state()
770 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) in __mod_memcg_state() argument
775 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); in __mod_memcg_state()
776 memcg_rstat_updated(memcg, val); in __mod_memcg_state()
780 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) in memcg_page_state_local() argument
782 long x = READ_ONCE(memcg->vmstats->state_local[idx]); in memcg_page_state_local()
795 struct mem_cgroup *memcg; in __mod_memcg_lruvec_state() local
798 memcg = pn->memcg; in __mod_memcg_lruvec_state()
822 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); in __mod_memcg_lruvec_state()
827 memcg_rstat_updated(memcg, val); in __mod_memcg_lruvec_state()
856 struct mem_cgroup *memcg; in __mod_lruvec_page_state() local
861 memcg = page_memcg(head); in __mod_lruvec_page_state()
863 if (!memcg) { in __mod_lruvec_page_state()
869 lruvec = mem_cgroup_lruvec(memcg, pgdat); in __mod_lruvec_page_state()
878 struct mem_cgroup *memcg; in __mod_lruvec_kmem_state() local
882 memcg = mem_cgroup_from_slab_obj(p); in __mod_lruvec_kmem_state()
890 if (!memcg) { in __mod_lruvec_kmem_state()
893 lruvec = mem_cgroup_lruvec(memcg, pgdat); in __mod_lruvec_kmem_state()
905 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, in __count_memcg_events() argument
914 __this_cpu_add(memcg->vmstats_percpu->events[index], count); in __count_memcg_events()
915 memcg_rstat_updated(memcg, count); in __count_memcg_events()
919 static unsigned long memcg_events(struct mem_cgroup *memcg, int event) in memcg_events() argument
925 return READ_ONCE(memcg->vmstats->events[index]); in memcg_events()
928 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) in memcg_events_local() argument
935 return READ_ONCE(memcg->vmstats->events_local[index]); in memcg_events_local()
938 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, in mem_cgroup_charge_statistics() argument
943 __count_memcg_events(memcg, PGPGIN, 1); in mem_cgroup_charge_statistics()
945 __count_memcg_events(memcg, PGPGOUT, 1); in mem_cgroup_charge_statistics()
949 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages); in mem_cgroup_charge_statistics()
952 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, in mem_cgroup_event_ratelimit() argument
957 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events); in mem_cgroup_event_ratelimit()
958 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]); in mem_cgroup_event_ratelimit()
971 __this_cpu_write(memcg->vmstats_percpu->targets[target], next); in mem_cgroup_event_ratelimit()
981 static void memcg_check_events(struct mem_cgroup *memcg, int nid) in memcg_check_events() argument
987 if (unlikely(mem_cgroup_event_ratelimit(memcg, in memcg_check_events()
991 do_softlimit = mem_cgroup_event_ratelimit(memcg, in memcg_check_events()
993 mem_cgroup_threshold(memcg); in memcg_check_events()
995 mem_cgroup_update_tree(memcg, nid); in memcg_check_events()
1034 struct mem_cgroup *memcg; in get_mem_cgroup_from_mm() local
1049 memcg = active_memcg(); in get_mem_cgroup_from_mm()
1050 if (unlikely(memcg)) { in get_mem_cgroup_from_mm()
1052 css_get(&memcg->css); in get_mem_cgroup_from_mm()
1053 return memcg; in get_mem_cgroup_from_mm()
1062 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); in get_mem_cgroup_from_mm()
1063 if (unlikely(!memcg)) in get_mem_cgroup_from_mm()
1064 memcg = root_mem_cgroup; in get_mem_cgroup_from_mm()
1065 } while (!css_tryget(&memcg->css)); in get_mem_cgroup_from_mm()
1067 return memcg; in get_mem_cgroup_from_mm()
1107 struct mem_cgroup *memcg = NULL; in mem_cgroup_iter() local
1174 memcg = mem_cgroup_from_css(css); in mem_cgroup_iter()
1185 (void)cmpxchg(&iter->position, pos, memcg); in mem_cgroup_iter()
1190 if (!memcg) in mem_cgroup_iter()
1199 return memcg; in mem_cgroup_iter()
1232 struct mem_cgroup *memcg = dead_memcg; in invalidate_reclaim_iterators() local
1236 __invalidate_reclaim_iterators(memcg, dead_memcg); in invalidate_reclaim_iterators()
1237 last = memcg; in invalidate_reclaim_iterators()
1238 } while ((memcg = parent_mem_cgroup(memcg))); in invalidate_reclaim_iterators()
1264 void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, in mem_cgroup_scan_tasks() argument
1270 BUG_ON(mem_cgroup_is_root(memcg)); in mem_cgroup_scan_tasks()
1272 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_scan_tasks()
1281 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_scan_tasks()
1290 struct mem_cgroup *memcg; in lruvec_memcg_debug() local
1295 memcg = folio_memcg(folio); in lruvec_memcg_debug()
1297 if (!memcg) in lruvec_memcg_debug()
1300 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio); in lruvec_memcg_debug()
1419 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) in mem_cgroup_margin() argument
1425 count = page_counter_read(&memcg->memory); in mem_cgroup_margin()
1426 limit = READ_ONCE(memcg->memory.max); in mem_cgroup_margin()
1431 count = page_counter_read(&memcg->memsw); in mem_cgroup_margin()
1432 limit = READ_ONCE(memcg->memsw.max); in mem_cgroup_margin()
1449 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) in mem_cgroup_under_move() argument
1464 ret = mem_cgroup_is_descendant(from, memcg) || in mem_cgroup_under_move()
1465 mem_cgroup_is_descendant(to, memcg); in mem_cgroup_under_move()
1471 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) in mem_cgroup_wait_acct_move() argument
1474 if (mem_cgroup_under_move(memcg)) { in mem_cgroup_wait_acct_move()
1559 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, in memcg_page_state_output() argument
1562 return memcg_page_state(memcg, item) * memcg_page_state_unit(item); in memcg_page_state_output()
1565 static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) in memcg_stat_format() argument
1584 size = memcg_page_state_output(memcg, memory_stats[i].idx); in memcg_stat_format()
1588 size += memcg_page_state_output(memcg, in memcg_stat_format()
1596 memcg_events(memcg, PGSCAN_KSWAPD) + in memcg_stat_format()
1597 memcg_events(memcg, PGSCAN_DIRECT) + in memcg_stat_format()
1598 memcg_events(memcg, PGSCAN_KHUGEPAGED)); in memcg_stat_format()
1600 memcg_events(memcg, PGSTEAL_KSWAPD) + in memcg_stat_format()
1601 memcg_events(memcg, PGSTEAL_DIRECT) + in memcg_stat_format()
1602 memcg_events(memcg, PGSTEAL_KHUGEPAGED)); in memcg_stat_format()
1611 memcg_events(memcg, memcg_vm_event_stat[i])); in memcg_stat_format()
1618 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s);
1620 static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) in memory_stat_format() argument
1623 memcg_stat_format(memcg, s); in memory_stat_format()
1625 memcg1_stat_format(memcg, s); in memory_stat_format()
1638 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) in mem_cgroup_print_oom_context() argument
1642 if (memcg) { in mem_cgroup_print_oom_context()
1644 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_context()
1659 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) in mem_cgroup_print_oom_meminfo() argument
1668 K((u64)page_counter_read(&memcg->memory)), in mem_cgroup_print_oom_meminfo()
1669 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt); in mem_cgroup_print_oom_meminfo()
1672 K((u64)page_counter_read(&memcg->swap)), in mem_cgroup_print_oom_meminfo()
1673 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt); in mem_cgroup_print_oom_meminfo()
1676 K((u64)page_counter_read(&memcg->memsw)), in mem_cgroup_print_oom_meminfo()
1677 K((u64)memcg->memsw.max), memcg->memsw.failcnt); in mem_cgroup_print_oom_meminfo()
1679 K((u64)page_counter_read(&memcg->kmem)), in mem_cgroup_print_oom_meminfo()
1680 K((u64)memcg->kmem.max), memcg->kmem.failcnt); in mem_cgroup_print_oom_meminfo()
1684 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_meminfo()
1687 memory_stat_format(memcg, &s); in mem_cgroup_print_oom_meminfo()
1694 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) in mem_cgroup_get_max() argument
1696 unsigned long max = READ_ONCE(memcg->memory.max); in mem_cgroup_get_max()
1699 if (mem_cgroup_swappiness(memcg)) { in mem_cgroup_get_max()
1701 unsigned long swap = READ_ONCE(memcg->memsw.max) - max; in mem_cgroup_get_max()
1706 if (mem_cgroup_swappiness(memcg)) in mem_cgroup_get_max()
1707 max += min(READ_ONCE(memcg->swap.max), in mem_cgroup_get_max()
1713 unsigned long mem_cgroup_size(struct mem_cgroup *memcg) in mem_cgroup_size() argument
1715 return page_counter_read(&memcg->memory); in mem_cgroup_size()
1718 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, in mem_cgroup_out_of_memory() argument
1724 .memcg = memcg, in mem_cgroup_out_of_memory()
1733 if (mem_cgroup_margin(memcg) >= (1 << order)) in mem_cgroup_out_of_memory()
1809 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) in mem_cgroup_oom_trylock() argument
1815 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_oom_trylock()
1822 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_oom_trylock()
1833 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_oom_trylock()
1835 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_oom_trylock()
1848 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) in mem_cgroup_oom_unlock() argument
1854 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_oom_unlock()
1859 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) in mem_cgroup_mark_under_oom() argument
1864 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_mark_under_oom()
1869 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) in mem_cgroup_unmark_under_oom() argument
1878 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_unmark_under_oom()
1887 struct mem_cgroup *memcg; member
1899 oom_wait_memcg = oom_wait_info->memcg; in memcg_oom_wake_function()
1907 static void memcg_oom_recover(struct mem_cgroup *memcg) in memcg_oom_recover() argument
1917 if (memcg && memcg->under_oom) in memcg_oom_recover()
1918 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); in memcg_oom_recover()
1925 static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) in mem_cgroup_oom() argument
1932 memcg_memory_event(memcg, MEMCG_OOM); in mem_cgroup_oom()
1952 if (READ_ONCE(memcg->oom_kill_disable)) { in mem_cgroup_oom()
1954 css_get(&memcg->css); in mem_cgroup_oom()
1955 current->memcg_in_oom = memcg; in mem_cgroup_oom()
1962 mem_cgroup_mark_under_oom(memcg); in mem_cgroup_oom()
1964 locked = mem_cgroup_oom_trylock(memcg); in mem_cgroup_oom()
1967 mem_cgroup_oom_notify(memcg); in mem_cgroup_oom()
1969 mem_cgroup_unmark_under_oom(memcg); in mem_cgroup_oom()
1970 ret = mem_cgroup_out_of_memory(memcg, mask, order); in mem_cgroup_oom()
1973 mem_cgroup_oom_unlock(memcg); in mem_cgroup_oom()
1997 struct mem_cgroup *memcg = current->memcg_in_oom; in mem_cgroup_oom_synchronize() local
2002 if (!memcg) in mem_cgroup_oom_synchronize()
2008 owait.memcg = memcg; in mem_cgroup_oom_synchronize()
2015 mem_cgroup_mark_under_oom(memcg); in mem_cgroup_oom_synchronize()
2017 locked = mem_cgroup_oom_trylock(memcg); in mem_cgroup_oom_synchronize()
2020 mem_cgroup_oom_notify(memcg); in mem_cgroup_oom_synchronize()
2023 mem_cgroup_unmark_under_oom(memcg); in mem_cgroup_oom_synchronize()
2027 mem_cgroup_oom_unlock(memcg); in mem_cgroup_oom_synchronize()
2030 css_put(&memcg->css); in mem_cgroup_oom_synchronize()
2048 struct mem_cgroup *memcg; in mem_cgroup_get_oom_group() local
2058 memcg = mem_cgroup_from_task(victim); in mem_cgroup_get_oom_group()
2059 if (mem_cgroup_is_root(memcg)) in mem_cgroup_get_oom_group()
2067 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain))) in mem_cgroup_get_oom_group()
2075 for (; memcg; memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_get_oom_group()
2076 if (READ_ONCE(memcg->oom_group)) in mem_cgroup_get_oom_group()
2077 oom_group = memcg; in mem_cgroup_get_oom_group()
2079 if (memcg == oom_domain) in mem_cgroup_get_oom_group()
2091 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) in mem_cgroup_print_oom_group() argument
2094 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_group()
2110 struct mem_cgroup *memcg; in folio_memcg_lock() local
2123 memcg = folio_memcg(folio); in folio_memcg_lock()
2124 if (unlikely(!memcg)) in folio_memcg_lock()
2129 might_lock(&memcg->move_lock); in folio_memcg_lock()
2133 if (atomic_read(&memcg->moving_account) <= 0) in folio_memcg_lock()
2136 spin_lock_irqsave(&memcg->move_lock, flags); in folio_memcg_lock()
2137 if (memcg != folio_memcg(folio)) { in folio_memcg_lock()
2138 spin_unlock_irqrestore(&memcg->move_lock, flags); in folio_memcg_lock()
2148 memcg->move_lock_task = current; in folio_memcg_lock()
2149 memcg->move_lock_flags = flags; in folio_memcg_lock()
2152 static void __folio_memcg_unlock(struct mem_cgroup *memcg) in __folio_memcg_unlock() argument
2154 if (memcg && memcg->move_lock_task == current) { in __folio_memcg_unlock()
2155 unsigned long flags = memcg->move_lock_flags; in __folio_memcg_unlock()
2157 memcg->move_lock_task = NULL; in __folio_memcg_unlock()
2158 memcg->move_lock_flags = 0; in __folio_memcg_unlock()
2160 spin_unlock_irqrestore(&memcg->move_lock, flags); in __folio_memcg_unlock()
2205 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages);
2217 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) in memcg_account_kmem() argument
2233 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in consume_stock() argument
2245 if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) { in consume_stock()
2303 static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in __refill_stock() argument
2308 if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */ in __refill_stock()
2310 css_get(&memcg->css); in __refill_stock()
2311 WRITE_ONCE(stock->cached, memcg); in __refill_stock()
2319 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in refill_stock() argument
2324 __refill_stock(memcg, nr_pages); in refill_stock()
2349 struct mem_cgroup *memcg; in drain_all_stock() local
2353 memcg = READ_ONCE(stock->cached); in drain_all_stock()
2354 if (memcg && stock->nr_pages && in drain_all_stock()
2355 mem_cgroup_is_descendant(memcg, root_memcg)) in drain_all_stock()
2383 static unsigned long reclaim_high(struct mem_cgroup *memcg, in reclaim_high() argument
2392 if (page_counter_read(&memcg->memory) <= in reclaim_high()
2393 READ_ONCE(memcg->memory.high)) in reclaim_high()
2396 memcg_memory_event(memcg, MEMCG_HIGH); in reclaim_high()
2399 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages, in reclaim_high()
2403 } while ((memcg = parent_mem_cgroup(memcg)) && in reclaim_high()
2404 !mem_cgroup_is_root(memcg)); in reclaim_high()
2411 struct mem_cgroup *memcg; in high_work_func() local
2413 memcg = container_of(work, struct mem_cgroup, high_work); in high_work_func()
2414 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); in high_work_func()
2488 static u64 mem_find_max_overage(struct mem_cgroup *memcg) in mem_find_max_overage() argument
2493 overage = calculate_overage(page_counter_read(&memcg->memory), in mem_find_max_overage()
2494 READ_ONCE(memcg->memory.high)); in mem_find_max_overage()
2496 } while ((memcg = parent_mem_cgroup(memcg)) && in mem_find_max_overage()
2497 !mem_cgroup_is_root(memcg)); in mem_find_max_overage()
2502 static u64 swap_find_max_overage(struct mem_cgroup *memcg) in swap_find_max_overage() argument
2507 overage = calculate_overage(page_counter_read(&memcg->swap), in swap_find_max_overage()
2508 READ_ONCE(memcg->swap.high)); in swap_find_max_overage()
2510 memcg_memory_event(memcg, MEMCG_SWAP_HIGH); in swap_find_max_overage()
2512 } while ((memcg = parent_mem_cgroup(memcg)) && in swap_find_max_overage()
2513 !mem_cgroup_is_root(memcg)); in swap_find_max_overage()
2522 static unsigned long calculate_high_delay(struct mem_cgroup *memcg, in calculate_high_delay() argument
2565 struct mem_cgroup *memcg; in mem_cgroup_handle_over_high() local
2571 memcg = get_mem_cgroup_from_mm(current->mm); in mem_cgroup_handle_over_high()
2584 nr_reclaimed = reclaim_high(memcg, in mem_cgroup_handle_over_high()
2592 penalty_jiffies = calculate_high_delay(memcg, nr_pages, in mem_cgroup_handle_over_high()
2593 mem_find_max_overage(memcg)); in mem_cgroup_handle_over_high()
2595 penalty_jiffies += calculate_high_delay(memcg, nr_pages, in mem_cgroup_handle_over_high()
2596 swap_find_max_overage(memcg)); in mem_cgroup_handle_over_high()
2634 css_put(&memcg->css); in mem_cgroup_handle_over_high()
2637 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, in try_charge_memcg() argument
2652 if (consume_stock(memcg, nr_pages)) in try_charge_memcg()
2656 page_counter_try_charge(&memcg->memsw, batch, &counter)) { in try_charge_memcg()
2657 if (page_counter_try_charge(&memcg->memory, batch, &counter)) in try_charge_memcg()
2660 page_counter_uncharge(&memcg->memsw, batch); in try_charge_memcg()
2767 page_counter_charge(&memcg->memory, nr_pages); in try_charge_memcg()
2769 page_counter_charge(&memcg->memsw, nr_pages); in try_charge_memcg()
2775 refill_stock(memcg, batch - nr_pages); in try_charge_memcg()
2789 mem_high = page_counter_read(&memcg->memory) > in try_charge_memcg()
2790 READ_ONCE(memcg->memory.high); in try_charge_memcg()
2791 swap_high = page_counter_read(&memcg->swap) > in try_charge_memcg()
2792 READ_ONCE(memcg->swap.high); in try_charge_memcg()
2797 schedule_work(&memcg->high_work); in try_charge_memcg()
2817 } while ((memcg = parent_mem_cgroup(memcg))); in try_charge_memcg()
2827 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, in try_charge() argument
2830 if (mem_cgroup_is_root(memcg)) in try_charge()
2833 return try_charge_memcg(memcg, gfp_mask, nr_pages); in try_charge()
2836 static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) in cancel_charge() argument
2838 if (mem_cgroup_is_root(memcg)) in cancel_charge()
2841 page_counter_uncharge(&memcg->memory, nr_pages); in cancel_charge()
2843 page_counter_uncharge(&memcg->memsw, nr_pages); in cancel_charge()
2846 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) in commit_charge() argument
2858 folio->memcg_data = (unsigned long)memcg; in commit_charge()
2877 struct mem_cgroup *memcg; in mod_objcg_mlstate() local
2881 memcg = obj_cgroup_memcg(objcg); in mod_objcg_mlstate()
2882 lruvec = mem_cgroup_lruvec(memcg, pgdat); in mod_objcg_mlstate()
3004 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) in __get_obj_cgroup_from_memcg() argument
3008 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) { in __get_obj_cgroup_from_memcg()
3009 objcg = rcu_dereference(memcg->objcg); in __get_obj_cgroup_from_memcg()
3020 struct mem_cgroup *memcg; in get_obj_cgroup_from_current() local
3027 memcg = active_memcg(); in get_obj_cgroup_from_current()
3029 memcg = mem_cgroup_from_task(current); in get_obj_cgroup_from_current()
3030 objcg = __get_obj_cgroup_from_memcg(memcg); in get_obj_cgroup_from_current()
3046 struct mem_cgroup *memcg; in get_obj_cgroup_from_folio() local
3049 memcg = __folio_memcg(folio); in get_obj_cgroup_from_folio()
3050 if (memcg) in get_obj_cgroup_from_folio()
3051 objcg = __get_obj_cgroup_from_memcg(memcg); in get_obj_cgroup_from_folio()
3059 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) in memcg_account_kmem() argument
3061 mod_memcg_state(memcg, MEMCG_KMEM, nr_pages); in memcg_account_kmem()
3064 page_counter_charge(&memcg->kmem, nr_pages); in memcg_account_kmem()
3066 page_counter_uncharge(&memcg->kmem, -nr_pages); in memcg_account_kmem()
3079 struct mem_cgroup *memcg; in obj_cgroup_uncharge_pages() local
3081 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_uncharge_pages()
3083 memcg_account_kmem(memcg, -nr_pages); in obj_cgroup_uncharge_pages()
3084 refill_stock(memcg, nr_pages); in obj_cgroup_uncharge_pages()
3086 css_put(&memcg->css); in obj_cgroup_uncharge_pages()
3100 struct mem_cgroup *memcg; in obj_cgroup_charge_pages() local
3103 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_charge_pages()
3105 ret = try_charge_memcg(memcg, gfp, nr_pages); in obj_cgroup_charge_pages()
3109 memcg_account_kmem(memcg, nr_pages); in obj_cgroup_charge_pages()
3111 css_put(&memcg->css); in obj_cgroup_charge_pages()
3259 struct mem_cgroup *memcg; in drain_obj_stock() local
3261 memcg = get_mem_cgroup_from_objcg(old); in drain_obj_stock()
3263 memcg_account_kmem(memcg, -nr_pages); in drain_obj_stock()
3264 __refill_stock(memcg, nr_pages); in drain_obj_stock()
3266 css_put(&memcg->css); in drain_obj_stock()
3314 struct mem_cgroup *memcg; in obj_stock_flush_required() local
3317 memcg = obj_cgroup_memcg(objcg); in obj_stock_flush_required()
3318 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) in obj_stock_flush_required()
3416 struct mem_cgroup *memcg = folio_memcg(folio); in split_page_memcg() local
3419 if (mem_cgroup_disabled() || !memcg) in split_page_memcg()
3428 css_get_many(&memcg->css, nr - 1); in split_page_memcg()
3471 static int mem_cgroup_resize_max(struct mem_cgroup *memcg, in mem_cgroup_resize_max() argument
3478 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory; in mem_cgroup_resize_max()
3491 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) : in mem_cgroup_resize_max()
3492 max <= memcg->memsw.max; in mem_cgroup_resize_max()
3507 drain_all_stock(memcg); in mem_cgroup_resize_max()
3512 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, in mem_cgroup_resize_max()
3520 memcg_oom_recover(memcg); in mem_cgroup_resize_max()
3565 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, in mem_cgroup_soft_limit_reclaim()
3578 excess = soft_limit_excess(mz->memcg); in mem_cgroup_soft_limit_reclaim()
3590 css_put(&mz->memcg->css); in mem_cgroup_soft_limit_reclaim()
3603 css_put(&next_mz->memcg->css); in mem_cgroup_soft_limit_reclaim()
3612 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) in mem_cgroup_force_empty() argument
3619 drain_all_stock(memcg); in mem_cgroup_force_empty()
3622 while (nr_retries && page_counter_read(&memcg->memory)) { in mem_cgroup_force_empty()
3626 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, in mem_cgroup_force_empty()
3638 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_force_empty_write() local
3640 if (mem_cgroup_is_root(memcg)) in mem_cgroup_force_empty_write()
3642 return mem_cgroup_force_empty(memcg) ?: nbytes; in mem_cgroup_force_empty_write()
3664 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) in mem_cgroup_usage() argument
3668 if (mem_cgroup_is_root(memcg)) { in mem_cgroup_usage()
3679 val = page_counter_read(&memcg->memory); in mem_cgroup_usage()
3681 val = page_counter_read(&memcg->memsw); in mem_cgroup_usage()
3697 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_read_u64() local
3702 counter = &memcg->memory; in mem_cgroup_read_u64()
3705 counter = &memcg->memsw; in mem_cgroup_read_u64()
3708 counter = &memcg->kmem; in mem_cgroup_read_u64()
3711 counter = &memcg->tcpmem; in mem_cgroup_read_u64()
3719 if (counter == &memcg->memory) in mem_cgroup_read_u64()
3720 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; in mem_cgroup_read_u64()
3721 if (counter == &memcg->memsw) in mem_cgroup_read_u64()
3722 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; in mem_cgroup_read_u64()
3731 return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE; in mem_cgroup_read_u64()
3748 static int memcg_online_kmem(struct mem_cgroup *memcg) in memcg_online_kmem() argument
3755 if (unlikely(mem_cgroup_is_root(memcg))) in memcg_online_kmem()
3762 objcg->memcg = memcg; in memcg_online_kmem()
3763 rcu_assign_pointer(memcg->objcg, objcg); in memcg_online_kmem()
3767 memcg->kmemcg_id = memcg->id.id; in memcg_online_kmem()
3772 static void memcg_offline_kmem(struct mem_cgroup *memcg) in memcg_offline_kmem() argument
3779 if (unlikely(mem_cgroup_is_root(memcg))) in memcg_offline_kmem()
3782 parent = parent_mem_cgroup(memcg); in memcg_offline_kmem()
3786 memcg_reparent_objcgs(memcg, parent); in memcg_offline_kmem()
3794 memcg_reparent_list_lrus(memcg, parent); in memcg_offline_kmem()
3797 static int memcg_online_kmem(struct mem_cgroup *memcg) in memcg_online_kmem() argument
3801 static void memcg_offline_kmem(struct mem_cgroup *memcg) in memcg_offline_kmem() argument
3806 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max) in memcg_update_tcp_max() argument
3812 ret = page_counter_set_max(&memcg->tcpmem, max); in memcg_update_tcp_max()
3816 if (!memcg->tcpmem_active) { in memcg_update_tcp_max()
3834 memcg->tcpmem_active = true; in memcg_update_tcp_max()
3848 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_write() local
3859 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ in mem_cgroup_write()
3865 ret = mem_cgroup_resize_max(memcg, nr_pages, false); in mem_cgroup_write()
3868 ret = mem_cgroup_resize_max(memcg, nr_pages, true); in mem_cgroup_write()
3878 ret = memcg_update_tcp_max(memcg, nr_pages); in mem_cgroup_write()
3886 WRITE_ONCE(memcg->soft_limit, nr_pages); in mem_cgroup_write()
3897 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_reset() local
3902 counter = &memcg->memory; in mem_cgroup_reset()
3905 counter = &memcg->memsw; in mem_cgroup_reset()
3908 counter = &memcg->kmem; in mem_cgroup_reset()
3911 counter = &memcg->tcpmem; in mem_cgroup_reset()
3941 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_move_charge_write() local
3956 memcg->move_charge_at_immigrate = val; in mem_cgroup_move_charge_write()
3973 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, in mem_cgroup_node_nr_lru_pages() argument
3976 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); in mem_cgroup_node_nr_lru_pages()
3993 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, in mem_cgroup_nr_lru_pages() argument
4004 nr += memcg_page_state(memcg, NR_LRU_BASE + lru); in mem_cgroup_nr_lru_pages()
4006 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru); in mem_cgroup_nr_lru_pages()
4026 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memcg_numa_stat_show() local
4032 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, in memcg_numa_stat_show()
4036 mem_cgroup_node_nr_lru_pages(memcg, nid, in memcg_numa_stat_show()
4044 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, in memcg_numa_stat_show()
4048 mem_cgroup_node_nr_lru_pages(memcg, nid, in memcg_numa_stat_show()
4095 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) in memcg1_stat_format() argument
4110 nr = memcg_page_state_local(memcg, memcg1_stats[i]); in memcg1_stat_format()
4117 memcg_events_local(memcg, memcg1_events[i])); in memcg1_stat_format()
4121 memcg_page_state_local(memcg, NR_LRU_BASE + i) * in memcg1_stat_format()
4126 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { in memcg1_stat_format()
4141 nr = memcg_page_state(memcg, memcg1_stats[i]); in memcg1_stat_format()
4149 (u64)memcg_events(memcg, memcg1_events[i])); in memcg1_stat_format()
4153 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) * in memcg1_stat_format()
4164 mz = memcg->nodeinfo[pgdat->node_id]; in memcg1_stat_format()
4178 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_swappiness_read() local
4180 return mem_cgroup_swappiness(memcg); in mem_cgroup_swappiness_read()
4186 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_swappiness_write() local
4191 if (!mem_cgroup_is_root(memcg)) in mem_cgroup_swappiness_write()
4192 WRITE_ONCE(memcg->swappiness, val); in mem_cgroup_swappiness_write()
4199 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) in __mem_cgroup_threshold() argument
4207 t = rcu_dereference(memcg->thresholds.primary); in __mem_cgroup_threshold()
4209 t = rcu_dereference(memcg->memsw_thresholds.primary); in __mem_cgroup_threshold()
4214 usage = mem_cgroup_usage(memcg, swap); in __mem_cgroup_threshold()
4250 static void mem_cgroup_threshold(struct mem_cgroup *memcg) in mem_cgroup_threshold() argument
4252 while (memcg) { in mem_cgroup_threshold()
4253 __mem_cgroup_threshold(memcg, false); in mem_cgroup_threshold()
4255 __mem_cgroup_threshold(memcg, true); in mem_cgroup_threshold()
4257 memcg = parent_mem_cgroup(memcg); in mem_cgroup_threshold()
4275 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) in mem_cgroup_oom_notify_cb() argument
4281 list_for_each_entry(ev, &memcg->oom_notify, list) in mem_cgroup_oom_notify_cb()
4288 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) in mem_cgroup_oom_notify() argument
4292 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_oom_notify()
4296 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, in __mem_cgroup_usage_register_event() argument
4309 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
4312 thresholds = &memcg->thresholds; in __mem_cgroup_usage_register_event()
4313 usage = mem_cgroup_usage(memcg, false); in __mem_cgroup_usage_register_event()
4315 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_register_event()
4316 usage = mem_cgroup_usage(memcg, true); in __mem_cgroup_usage_register_event()
4322 __mem_cgroup_threshold(memcg, type == _MEMSWAP); in __mem_cgroup_usage_register_event()
4371 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
4376 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, in mem_cgroup_usage_register_event() argument
4379 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); in mem_cgroup_usage_register_event()
4382 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, in memsw_cgroup_usage_register_event() argument
4385 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); in memsw_cgroup_usage_register_event()
4388 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in __mem_cgroup_usage_unregister_event() argument
4396 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
4399 thresholds = &memcg->thresholds; in __mem_cgroup_usage_unregister_event()
4400 usage = mem_cgroup_usage(memcg, false); in __mem_cgroup_usage_unregister_event()
4402 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_unregister_event()
4403 usage = mem_cgroup_usage(memcg, true); in __mem_cgroup_usage_unregister_event()
4411 __mem_cgroup_threshold(memcg, type == _MEMSWAP); in __mem_cgroup_usage_unregister_event()
4470 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
4473 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in mem_cgroup_usage_unregister_event() argument
4476 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); in mem_cgroup_usage_unregister_event()
4479 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in memsw_cgroup_usage_unregister_event() argument
4482 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); in memsw_cgroup_usage_unregister_event()
4485 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, in mem_cgroup_oom_register_event() argument
4497 list_add(&event->list, &memcg->oom_notify); in mem_cgroup_oom_register_event()
4500 if (memcg->under_oom) in mem_cgroup_oom_register_event()
4507 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, in mem_cgroup_oom_unregister_event() argument
4514 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { in mem_cgroup_oom_unregister_event()
4526 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf); in mem_cgroup_oom_control_read() local
4528 seq_printf(sf, "oom_kill_disable %d\n", READ_ONCE(memcg->oom_kill_disable)); in mem_cgroup_oom_control_read()
4529 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); in mem_cgroup_oom_control_read()
4531 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); in mem_cgroup_oom_control_read()
4538 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_oom_control_write() local
4541 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1))) in mem_cgroup_oom_control_write()
4544 WRITE_ONCE(memcg->oom_kill_disable, val); in mem_cgroup_oom_control_write()
4546 memcg_oom_recover(memcg); in mem_cgroup_oom_control_write()
4555 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) in memcg_wb_domain_init() argument
4557 return wb_domain_init(&memcg->cgwb_domain, gfp); in memcg_wb_domain_init()
4560 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) in memcg_wb_domain_exit() argument
4562 wb_domain_exit(&memcg->cgwb_domain); in memcg_wb_domain_exit()
4565 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) in memcg_wb_domain_size_changed() argument
4567 wb_domain_size_changed(&memcg->cgwb_domain); in memcg_wb_domain_size_changed()
4572 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_wb_domain() local
4574 if (!memcg->css.parent) in mem_cgroup_wb_domain()
4577 return &memcg->cgwb_domain; in mem_cgroup_wb_domain()
4602 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_wb_stats() local
4607 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); in mem_cgroup_wb_stats()
4608 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); in mem_cgroup_wb_stats()
4609 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) + in mem_cgroup_wb_stats()
4610 memcg_page_state(memcg, NR_ACTIVE_FILE); in mem_cgroup_wb_stats()
4613 while ((parent = parent_mem_cgroup(memcg))) { in mem_cgroup_wb_stats()
4614 unsigned long ceiling = min(READ_ONCE(memcg->memory.max), in mem_cgroup_wb_stats()
4615 READ_ONCE(memcg->memory.high)); in mem_cgroup_wb_stats()
4616 unsigned long used = page_counter_read(&memcg->memory); in mem_cgroup_wb_stats()
4619 memcg = parent; in mem_cgroup_wb_stats()
4670 struct mem_cgroup *memcg = folio_memcg(folio); in mem_cgroup_track_foreign_dirty_slowpath() local
4685 frn = &memcg->cgwb_frn[i]; in mem_cgroup_track_foreign_dirty_slowpath()
4712 frn = &memcg->cgwb_frn[oldest]; in mem_cgroup_track_foreign_dirty_slowpath()
4722 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_flush_foreign() local
4728 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i]; in mem_cgroup_flush_foreign()
4749 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) in memcg_wb_domain_init() argument
4754 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) in memcg_wb_domain_exit() argument
4758 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) in memcg_wb_domain_size_changed() argument
4786 struct mem_cgroup *memcg = event->memcg; in memcg_event_remove() local
4790 event->unregister_event(memcg, event->eventfd); in memcg_event_remove()
4797 css_put(&memcg->css); in memcg_event_remove()
4810 struct mem_cgroup *memcg = event->memcg; in memcg_event_wake() local
4823 spin_lock(&memcg->event_list_lock); in memcg_event_wake()
4832 spin_unlock(&memcg->event_list_lock); in memcg_event_wake()
4860 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memcg_write_event_control() local
4890 event->memcg = memcg; in memcg_write_event_control()
4972 ret = event->register_event(memcg, event->eventfd, buf); in memcg_write_event_control()
4978 spin_lock_irq(&memcg->event_list_lock); in memcg_write_event_control()
4979 list_add(&event->list, &memcg->event_list); in memcg_write_event_control()
4980 spin_unlock_irq(&memcg->event_list_lock); in memcg_write_event_control()
5170 static void mem_cgroup_id_remove(struct mem_cgroup *memcg) in mem_cgroup_id_remove() argument
5172 if (memcg->id.id > 0) { in mem_cgroup_id_remove()
5173 idr_remove(&mem_cgroup_idr, memcg->id.id); in mem_cgroup_id_remove()
5174 memcg->id.id = 0; in mem_cgroup_id_remove()
5178 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg, in mem_cgroup_id_get_many() argument
5181 refcount_add(n, &memcg->id.ref); in mem_cgroup_id_get_many()
5184 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) in mem_cgroup_id_put_many() argument
5186 if (refcount_sub_and_test(n, &memcg->id.ref)) { in mem_cgroup_id_put_many()
5187 mem_cgroup_id_remove(memcg); in mem_cgroup_id_put_many()
5190 css_put(&memcg->css); in mem_cgroup_id_put_many()
5194 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) in mem_cgroup_id_put() argument
5196 mem_cgroup_id_put_many(memcg, 1); in mem_cgroup_id_put()
5216 struct mem_cgroup *memcg; in mem_cgroup_get_from_ino() local
5224 memcg = container_of(css, struct mem_cgroup, css); in mem_cgroup_get_from_ino()
5226 memcg = ERR_PTR(-ENOENT); in mem_cgroup_get_from_ino()
5230 return memcg; in mem_cgroup_get_from_ino()
5234 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) in alloc_mem_cgroup_per_node_info() argument
5250 pn->memcg = memcg; in alloc_mem_cgroup_per_node_info()
5252 memcg->nodeinfo[node] = pn; in alloc_mem_cgroup_per_node_info()
5256 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) in free_mem_cgroup_per_node_info() argument
5258 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; in free_mem_cgroup_per_node_info()
5267 static void __mem_cgroup_free(struct mem_cgroup *memcg) in __mem_cgroup_free() argument
5272 free_mem_cgroup_per_node_info(memcg, node); in __mem_cgroup_free()
5273 kfree(memcg->vmstats); in __mem_cgroup_free()
5274 free_percpu(memcg->vmstats_percpu); in __mem_cgroup_free()
5275 kfree(memcg); in __mem_cgroup_free()
5278 static void mem_cgroup_free(struct mem_cgroup *memcg) in mem_cgroup_free() argument
5280 lru_gen_exit_memcg(memcg); in mem_cgroup_free()
5281 memcg_wb_domain_exit(memcg); in mem_cgroup_free()
5282 __mem_cgroup_free(memcg); in mem_cgroup_free()
5287 struct mem_cgroup *memcg; in mem_cgroup_alloc() local
5292 memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL); in mem_cgroup_alloc()
5293 if (!memcg) in mem_cgroup_alloc()
5296 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, in mem_cgroup_alloc()
5298 if (memcg->id.id < 0) { in mem_cgroup_alloc()
5299 error = memcg->id.id; in mem_cgroup_alloc()
5303 memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL); in mem_cgroup_alloc()
5304 if (!memcg->vmstats) in mem_cgroup_alloc()
5307 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu, in mem_cgroup_alloc()
5309 if (!memcg->vmstats_percpu) in mem_cgroup_alloc()
5313 if (alloc_mem_cgroup_per_node_info(memcg, node)) in mem_cgroup_alloc()
5316 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) in mem_cgroup_alloc()
5319 INIT_WORK(&memcg->high_work, high_work_func); in mem_cgroup_alloc()
5320 INIT_LIST_HEAD(&memcg->oom_notify); in mem_cgroup_alloc()
5321 mutex_init(&memcg->thresholds_lock); in mem_cgroup_alloc()
5322 spin_lock_init(&memcg->move_lock); in mem_cgroup_alloc()
5323 vmpressure_init(&memcg->vmpressure); in mem_cgroup_alloc()
5324 INIT_LIST_HEAD(&memcg->event_list); in mem_cgroup_alloc()
5325 spin_lock_init(&memcg->event_list_lock); in mem_cgroup_alloc()
5326 memcg->socket_pressure = jiffies; in mem_cgroup_alloc()
5328 memcg->kmemcg_id = -1; in mem_cgroup_alloc()
5329 INIT_LIST_HEAD(&memcg->objcg_list); in mem_cgroup_alloc()
5332 INIT_LIST_HEAD(&memcg->cgwb_list); in mem_cgroup_alloc()
5334 memcg->cgwb_frn[i].done = in mem_cgroup_alloc()
5338 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock); in mem_cgroup_alloc()
5339 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue); in mem_cgroup_alloc()
5340 memcg->deferred_split_queue.split_queue_len = 0; in mem_cgroup_alloc()
5342 lru_gen_init_memcg(memcg); in mem_cgroup_alloc()
5343 return memcg; in mem_cgroup_alloc()
5345 mem_cgroup_id_remove(memcg); in mem_cgroup_alloc()
5346 __mem_cgroup_free(memcg); in mem_cgroup_alloc()
5354 struct mem_cgroup *memcg, *old_memcg; in mem_cgroup_css_alloc() local
5357 memcg = mem_cgroup_alloc(); in mem_cgroup_css_alloc()
5359 if (IS_ERR(memcg)) in mem_cgroup_css_alloc()
5360 return ERR_CAST(memcg); in mem_cgroup_css_alloc()
5362 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); in mem_cgroup_css_alloc()
5363 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX); in mem_cgroup_css_alloc()
5365 memcg->zswap_max = PAGE_COUNTER_MAX; in mem_cgroup_css_alloc()
5367 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); in mem_cgroup_css_alloc()
5369 WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); in mem_cgroup_css_alloc()
5370 WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable)); in mem_cgroup_css_alloc()
5372 page_counter_init(&memcg->memory, &parent->memory); in mem_cgroup_css_alloc()
5373 page_counter_init(&memcg->swap, &parent->swap); in mem_cgroup_css_alloc()
5374 page_counter_init(&memcg->kmem, &parent->kmem); in mem_cgroup_css_alloc()
5375 page_counter_init(&memcg->tcpmem, &parent->tcpmem); in mem_cgroup_css_alloc()
5378 page_counter_init(&memcg->memory, NULL); in mem_cgroup_css_alloc()
5379 page_counter_init(&memcg->swap, NULL); in mem_cgroup_css_alloc()
5380 page_counter_init(&memcg->kmem, NULL); in mem_cgroup_css_alloc()
5381 page_counter_init(&memcg->tcpmem, NULL); in mem_cgroup_css_alloc()
5383 root_mem_cgroup = memcg; in mem_cgroup_css_alloc()
5384 return &memcg->css; in mem_cgroup_css_alloc()
5395 return &memcg->css; in mem_cgroup_css_alloc()
5400 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_online() local
5402 if (memcg_online_kmem(memcg)) in mem_cgroup_css_online()
5410 if (alloc_shrinker_info(memcg)) in mem_cgroup_css_online()
5413 if (unlikely(mem_cgroup_is_root(memcg))) in mem_cgroup_css_online()
5416 lru_gen_online_memcg(memcg); in mem_cgroup_css_online()
5419 refcount_set(&memcg->id.ref, 1); in mem_cgroup_css_online()
5432 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); in mem_cgroup_css_online()
5436 memcg_offline_kmem(memcg); in mem_cgroup_css_online()
5438 mem_cgroup_id_remove(memcg); in mem_cgroup_css_online()
5444 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_offline() local
5452 spin_lock_irq(&memcg->event_list_lock); in mem_cgroup_css_offline()
5453 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { in mem_cgroup_css_offline()
5457 spin_unlock_irq(&memcg->event_list_lock); in mem_cgroup_css_offline()
5459 page_counter_set_min(&memcg->memory, 0); in mem_cgroup_css_offline()
5460 page_counter_set_low(&memcg->memory, 0); in mem_cgroup_css_offline()
5462 memcg_offline_kmem(memcg); in mem_cgroup_css_offline()
5463 reparent_shrinker_deferred(memcg); in mem_cgroup_css_offline()
5464 wb_memcg_offline(memcg); in mem_cgroup_css_offline()
5465 lru_gen_offline_memcg(memcg); in mem_cgroup_css_offline()
5467 drain_all_stock(memcg); in mem_cgroup_css_offline()
5469 mem_cgroup_id_put(memcg); in mem_cgroup_css_offline()
5474 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_released() local
5476 invalidate_reclaim_iterators(memcg); in mem_cgroup_css_released()
5477 lru_gen_release_memcg(memcg); in mem_cgroup_css_released()
5482 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_free() local
5487 wb_wait_for_completion(&memcg->cgwb_frn[i].done); in mem_cgroup_css_free()
5492 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) in mem_cgroup_css_free()
5500 vmpressure_cleanup(&memcg->vmpressure); in mem_cgroup_css_free()
5501 cancel_work_sync(&memcg->high_work); in mem_cgroup_css_free()
5502 mem_cgroup_remove_from_trees(memcg); in mem_cgroup_css_free()
5503 free_shrinker_info(memcg); in mem_cgroup_css_free()
5504 mem_cgroup_free(memcg); in mem_cgroup_css_free()
5522 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_reset() local
5524 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5525 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5526 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5527 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5528 page_counter_set_min(&memcg->memory, 0); in mem_cgroup_css_reset()
5529 page_counter_set_low(&memcg->memory, 0); in mem_cgroup_css_reset()
5530 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5531 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5532 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5533 memcg_wb_domain_size_changed(memcg); in mem_cgroup_css_reset()
5538 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_rstat_flush() local
5539 struct mem_cgroup *parent = parent_mem_cgroup(memcg); in mem_cgroup_css_rstat_flush()
5544 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); in mem_cgroup_css_rstat_flush()
5552 delta = memcg->vmstats->state_pending[i]; in mem_cgroup_css_rstat_flush()
5554 memcg->vmstats->state_pending[i] = 0; in mem_cgroup_css_rstat_flush()
5567 memcg->vmstats->state_local[i] += delta_cpu; in mem_cgroup_css_rstat_flush()
5570 memcg->vmstats->state[i] += delta; in mem_cgroup_css_rstat_flush()
5577 delta = memcg->vmstats->events_pending[i]; in mem_cgroup_css_rstat_flush()
5579 memcg->vmstats->events_pending[i] = 0; in mem_cgroup_css_rstat_flush()
5590 memcg->vmstats->events_local[i] += delta_cpu; in mem_cgroup_css_rstat_flush()
5593 memcg->vmstats->events[i] += delta; in mem_cgroup_css_rstat_flush()
5600 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; in mem_cgroup_css_rstat_flush()
6134 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ in mem_cgroup_can_attach() local
6155 memcg = mem_cgroup_from_css(css); in mem_cgroup_can_attach()
6165 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); in mem_cgroup_can_attach()
6171 VM_BUG_ON(from == memcg); in mem_cgroup_can_attach()
6187 mc.to = memcg; in mem_cgroup_can_attach()
6417 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memory_current_read() local
6419 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; in memory_current_read()
6425 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memory_peak_read() local
6427 return (u64)memcg->memory.watermark * PAGE_SIZE; in memory_peak_read()
6439 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_min_write() local
6448 page_counter_set_min(&memcg->memory, min); in memory_min_write()
6462 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_low_write() local
6471 page_counter_set_low(&memcg->memory, low); in memory_low_write()
6485 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_high_write() local
6496 page_counter_set_high(&memcg->memory, high); in memory_high_write()
6499 unsigned long nr_pages = page_counter_read(&memcg->memory); in memory_high_write()
6509 drain_all_stock(memcg); in memory_high_write()
6514 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high, in memory_high_write()
6521 memcg_wb_domain_size_changed(memcg); in memory_high_write()
6534 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_max_write() local
6545 xchg(&memcg->memory.max, max); in memory_max_write()
6548 unsigned long nr_pages = page_counter_read(&memcg->memory); in memory_max_write()
6557 drain_all_stock(memcg); in memory_max_write()
6563 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, in memory_max_write()
6569 memcg_memory_event(memcg, MEMCG_OOM); in memory_max_write()
6570 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) in memory_max_write()
6574 memcg_wb_domain_size_changed(memcg); in memory_max_write()
6592 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_events_show() local
6594 __memory_events_show(m, memcg->memory_events); in memory_events_show()
6600 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_events_local_show() local
6602 __memory_events_show(m, memcg->memory_events_local); in memory_events_local_show()
6608 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_stat_show() local
6615 memory_stat_format(memcg, &s); in memory_stat_show()
6631 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_numa_stat_show() local
6646 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); in memory_numa_stat_show()
6660 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_oom_group_show() local
6662 seq_printf(m, "%d\n", READ_ONCE(memcg->oom_group)); in memory_oom_group_show()
6670 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_oom_group_write() local
6684 WRITE_ONCE(memcg->oom_group, oom_group); in memory_oom_group_write()
6692 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_reclaim() local
6718 reclaimed = try_to_free_mem_cgroup_pages(memcg, in memory_reclaim()
6944 struct mem_cgroup *memcg) in mem_cgroup_calculate_protection() argument
6962 if (memcg == root) in mem_cgroup_calculate_protection()
6965 usage = page_counter_read(&memcg->memory); in mem_cgroup_calculate_protection()
6969 parent = parent_mem_cgroup(memcg); in mem_cgroup_calculate_protection()
6972 memcg->memory.emin = READ_ONCE(memcg->memory.min); in mem_cgroup_calculate_protection()
6973 memcg->memory.elow = READ_ONCE(memcg->memory.low); in mem_cgroup_calculate_protection()
6979 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage, in mem_cgroup_calculate_protection()
6980 READ_ONCE(memcg->memory.min), in mem_cgroup_calculate_protection()
6984 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage, in mem_cgroup_calculate_protection()
6985 READ_ONCE(memcg->memory.low), in mem_cgroup_calculate_protection()
6990 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, in charge_memcg() argument
6996 ret = try_charge(memcg, gfp, nr_pages); in charge_memcg()
7000 css_get(&memcg->css); in charge_memcg()
7001 commit_charge(folio, memcg); in charge_memcg()
7004 mem_cgroup_charge_statistics(memcg, nr_pages); in charge_memcg()
7005 memcg_check_events(memcg, folio_nid(folio)); in charge_memcg()
7013 struct mem_cgroup *memcg; in __mem_cgroup_charge() local
7016 memcg = get_mem_cgroup_from_mm(mm); in __mem_cgroup_charge()
7017 ret = charge_memcg(folio, memcg, gfp); in __mem_cgroup_charge()
7018 css_put(&memcg->css); in __mem_cgroup_charge()
7038 struct mem_cgroup *memcg; in mem_cgroup_swapin_charge_folio() local
7047 memcg = mem_cgroup_from_id(id); in mem_cgroup_swapin_charge_folio()
7048 if (!memcg || !css_tryget_online(&memcg->css)) in mem_cgroup_swapin_charge_folio()
7049 memcg = get_mem_cgroup_from_mm(mm); in mem_cgroup_swapin_charge_folio()
7052 ret = charge_memcg(folio, memcg, gfp); in mem_cgroup_swapin_charge_folio()
7054 css_put(&memcg->css); in mem_cgroup_swapin_charge_folio()
7092 struct mem_cgroup *memcg; member
7109 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory); in uncharge_batch()
7111 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory); in uncharge_batch()
7113 memcg_account_kmem(ug->memcg, -ug->nr_kmem); in uncharge_batch()
7114 memcg_oom_recover(ug->memcg); in uncharge_batch()
7118 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); in uncharge_batch()
7119 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory); in uncharge_batch()
7120 memcg_check_events(ug->memcg, ug->nid); in uncharge_batch()
7124 css_put(&ug->memcg->css); in uncharge_batch()
7130 struct mem_cgroup *memcg; in uncharge_folio() local
7146 memcg = get_mem_cgroup_from_objcg(objcg); in uncharge_folio()
7148 memcg = __folio_memcg(folio); in uncharge_folio()
7151 if (!memcg) in uncharge_folio()
7154 if (ug->memcg != memcg) { in uncharge_folio()
7155 if (ug->memcg) { in uncharge_folio()
7159 ug->memcg = memcg; in uncharge_folio()
7163 css_get(&memcg->css); in uncharge_folio()
7176 if (!mem_cgroup_is_root(memcg)) in uncharge_folio()
7183 css_put(&memcg->css); in uncharge_folio()
7214 if (ug.memcg) in __mem_cgroup_uncharge_list()
7230 struct mem_cgroup *memcg; in mem_cgroup_migrate() local
7246 memcg = folio_memcg(old); in mem_cgroup_migrate()
7247 VM_WARN_ON_ONCE_FOLIO(!memcg, old); in mem_cgroup_migrate()
7248 if (!memcg) in mem_cgroup_migrate()
7252 if (!mem_cgroup_is_root(memcg)) { in mem_cgroup_migrate()
7253 page_counter_charge(&memcg->memory, nr_pages); in mem_cgroup_migrate()
7255 page_counter_charge(&memcg->memsw, nr_pages); in mem_cgroup_migrate()
7258 css_get(&memcg->css); in mem_cgroup_migrate()
7259 commit_charge(new, memcg); in mem_cgroup_migrate()
7262 mem_cgroup_charge_statistics(memcg, nr_pages); in mem_cgroup_migrate()
7263 memcg_check_events(memcg, folio_nid(new)); in mem_cgroup_migrate()
7272 struct mem_cgroup *memcg; in mem_cgroup_sk_alloc() local
7282 memcg = mem_cgroup_from_task(current); in mem_cgroup_sk_alloc()
7283 if (mem_cgroup_is_root(memcg)) in mem_cgroup_sk_alloc()
7285 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) in mem_cgroup_sk_alloc()
7287 if (css_tryget(&memcg->css)) in mem_cgroup_sk_alloc()
7288 sk->sk_memcg = memcg; in mem_cgroup_sk_alloc()
7308 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, in mem_cgroup_charge_skmem() argument
7314 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { in mem_cgroup_charge_skmem()
7315 memcg->tcpmem_pressure = 0; in mem_cgroup_charge_skmem()
7318 memcg->tcpmem_pressure = 1; in mem_cgroup_charge_skmem()
7320 page_counter_charge(&memcg->tcpmem, nr_pages); in mem_cgroup_charge_skmem()
7326 if (try_charge(memcg, gfp_mask, nr_pages) == 0) { in mem_cgroup_charge_skmem()
7327 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages); in mem_cgroup_charge_skmem()
7339 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) in mem_cgroup_uncharge_skmem() argument
7342 page_counter_uncharge(&memcg->tcpmem, nr_pages); in mem_cgroup_uncharge_skmem()
7346 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); in mem_cgroup_uncharge_skmem()
7348 refill_stock(memcg, nr_pages); in mem_cgroup_uncharge_skmem()
7412 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) in mem_cgroup_id_get_online() argument
7414 while (!refcount_inc_not_zero(&memcg->id.ref)) { in mem_cgroup_id_get_online()
7419 if (WARN_ON_ONCE(mem_cgroup_is_root(memcg))) { in mem_cgroup_id_get_online()
7423 memcg = parent_mem_cgroup(memcg); in mem_cgroup_id_get_online()
7424 if (!memcg) in mem_cgroup_id_get_online()
7425 memcg = root_mem_cgroup; in mem_cgroup_id_get_online()
7427 return memcg; in mem_cgroup_id_get_online()
7439 struct mem_cgroup *memcg, *swap_memcg; in mem_cgroup_swapout() local
7452 memcg = folio_memcg(folio); in mem_cgroup_swapout()
7454 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); in mem_cgroup_swapout()
7455 if (!memcg) in mem_cgroup_swapout()
7463 swap_memcg = mem_cgroup_id_get_online(memcg); in mem_cgroup_swapout()
7475 if (!mem_cgroup_is_root(memcg)) in mem_cgroup_swapout()
7476 page_counter_uncharge(&memcg->memory, nr_entries); in mem_cgroup_swapout()
7478 if (memcg != swap_memcg) { in mem_cgroup_swapout()
7481 page_counter_uncharge(&memcg->memsw, nr_entries); in mem_cgroup_swapout()
7491 mem_cgroup_charge_statistics(memcg, -nr_entries); in mem_cgroup_swapout()
7493 memcg_check_events(memcg, folio_nid(folio)); in mem_cgroup_swapout()
7495 css_put(&memcg->css); in mem_cgroup_swapout()
7511 struct mem_cgroup *memcg; in __mem_cgroup_try_charge_swap() local
7517 memcg = folio_memcg(folio); in __mem_cgroup_try_charge_swap()
7519 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); in __mem_cgroup_try_charge_swap()
7520 if (!memcg) in __mem_cgroup_try_charge_swap()
7524 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); in __mem_cgroup_try_charge_swap()
7528 memcg = mem_cgroup_id_get_online(memcg); in __mem_cgroup_try_charge_swap()
7530 if (!mem_cgroup_is_root(memcg) && in __mem_cgroup_try_charge_swap()
7531 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { in __mem_cgroup_try_charge_swap()
7532 memcg_memory_event(memcg, MEMCG_SWAP_MAX); in __mem_cgroup_try_charge_swap()
7533 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); in __mem_cgroup_try_charge_swap()
7534 mem_cgroup_id_put(memcg); in __mem_cgroup_try_charge_swap()
7540 mem_cgroup_id_get_many(memcg, nr_pages - 1); in __mem_cgroup_try_charge_swap()
7541 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); in __mem_cgroup_try_charge_swap()
7543 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages); in __mem_cgroup_try_charge_swap()
7555 struct mem_cgroup *memcg; in __mem_cgroup_uncharge_swap() local
7560 memcg = mem_cgroup_from_id(id); in __mem_cgroup_uncharge_swap()
7561 if (memcg) { in __mem_cgroup_uncharge_swap()
7562 if (!mem_cgroup_is_root(memcg)) { in __mem_cgroup_uncharge_swap()
7564 page_counter_uncharge(&memcg->memsw, nr_pages); in __mem_cgroup_uncharge_swap()
7566 page_counter_uncharge(&memcg->swap, nr_pages); in __mem_cgroup_uncharge_swap()
7568 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); in __mem_cgroup_uncharge_swap()
7569 mem_cgroup_id_put_many(memcg, nr_pages); in __mem_cgroup_uncharge_swap()
7574 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) in mem_cgroup_get_nr_swap_pages() argument
7580 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) in mem_cgroup_get_nr_swap_pages()
7582 READ_ONCE(memcg->swap.max) - in mem_cgroup_get_nr_swap_pages()
7583 page_counter_read(&memcg->swap)); in mem_cgroup_get_nr_swap_pages()
7589 struct mem_cgroup *memcg; in mem_cgroup_swap_full() local
7598 memcg = folio_memcg(folio); in mem_cgroup_swap_full()
7599 if (!memcg) in mem_cgroup_swap_full()
7602 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_swap_full()
7603 unsigned long usage = page_counter_read(&memcg->swap); in mem_cgroup_swap_full()
7605 if (usage * 2 >= READ_ONCE(memcg->swap.high) || in mem_cgroup_swap_full()
7606 usage * 2 >= READ_ONCE(memcg->swap.max)) in mem_cgroup_swap_full()
7625 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in swap_current_read() local
7627 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; in swap_current_read()
7633 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in swap_peak_read() local
7635 return (u64)memcg->swap.watermark * PAGE_SIZE; in swap_peak_read()
7647 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in swap_high_write() local
7656 page_counter_set_high(&memcg->swap, high); in swap_high_write()
7670 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in swap_max_write() local
7679 xchg(&memcg->swap.max, max); in swap_max_write()
7686 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in swap_events_show() local
7689 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH])); in swap_events_show()
7691 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX])); in swap_events_show()
7693 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL])); in swap_events_show()
7772 struct mem_cgroup *memcg, *original_memcg; in obj_cgroup_may_zswap() local
7779 for (memcg = original_memcg; !mem_cgroup_is_root(memcg); in obj_cgroup_may_zswap()
7780 memcg = parent_mem_cgroup(memcg)) { in obj_cgroup_may_zswap()
7781 unsigned long max = READ_ONCE(memcg->zswap_max); in obj_cgroup_may_zswap()
7791 cgroup_rstat_flush(memcg->css.cgroup); in obj_cgroup_may_zswap()
7792 pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE; in obj_cgroup_may_zswap()
7812 struct mem_cgroup *memcg; in obj_cgroup_charge_zswap() local
7824 memcg = obj_cgroup_memcg(objcg); in obj_cgroup_charge_zswap()
7825 mod_memcg_state(memcg, MEMCG_ZSWAP_B, size); in obj_cgroup_charge_zswap()
7826 mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1); in obj_cgroup_charge_zswap()
7839 struct mem_cgroup *memcg; in obj_cgroup_uncharge_zswap() local
7847 memcg = obj_cgroup_memcg(objcg); in obj_cgroup_uncharge_zswap()
7848 mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size); in obj_cgroup_uncharge_zswap()
7849 mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1); in obj_cgroup_uncharge_zswap()
7869 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in zswap_max_write() local
7878 xchg(&memcg->zswap_max, max); in zswap_max_write()