Lines Matching refs:memcg
134 struct mem_cgroup *memcg; member
148 int (*register_event)(struct mem_cgroup *memcg,
155 void (*unregister_event)(struct mem_cgroup *memcg,
167 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
168 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
236 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) in memcg_to_vmpressure() argument
238 if (!memcg) in memcg_to_vmpressure()
239 memcg = root_mem_cgroup; in memcg_to_vmpressure()
240 return &memcg->vmpressure; in memcg_to_vmpressure()
320 static void memcg_reparent_objcgs(struct mem_cgroup *memcg, in memcg_reparent_objcgs() argument
325 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); in memcg_reparent_objcgs()
330 list_add(&objcg->list, &memcg->objcg_list); in memcg_reparent_objcgs()
332 list_for_each_entry(iter, &memcg->objcg_list, list) in memcg_reparent_objcgs()
333 WRITE_ONCE(iter->memcg, parent); in memcg_reparent_objcgs()
335 list_splice(&memcg->objcg_list, &parent->objcg_list); in memcg_reparent_objcgs()
365 struct mem_cgroup *memcg; in mem_cgroup_css_from_page() local
367 memcg = page_memcg(page); in mem_cgroup_css_from_page()
369 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) in mem_cgroup_css_from_page()
370 memcg = root_mem_cgroup; in mem_cgroup_css_from_page()
372 return &memcg->css; in mem_cgroup_css_from_page()
390 struct mem_cgroup *memcg; in page_cgroup_ino() local
394 memcg = page_memcg_check(page); in page_cgroup_ino()
396 while (memcg && !(memcg->css.flags & CSS_ONLINE)) in page_cgroup_ino()
397 memcg = parent_mem_cgroup(memcg); in page_cgroup_ino()
398 if (memcg) in page_cgroup_ino()
399 ino = cgroup_ino(memcg->css.cgroup); in page_cgroup_ino()
462 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) in soft_limit_excess() argument
464 unsigned long nr_pages = page_counter_read(&memcg->memory); in soft_limit_excess()
465 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); in soft_limit_excess()
474 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid) in mem_cgroup_update_tree() argument
487 for (; memcg; memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_update_tree()
488 mz = memcg->nodeinfo[nid]; in mem_cgroup_update_tree()
489 excess = soft_limit_excess(memcg); in mem_cgroup_update_tree()
511 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) in mem_cgroup_remove_from_trees() argument
518 mz = memcg->nodeinfo[nid]; in mem_cgroup_remove_from_trees()
543 if (!soft_limit_excess(mz->memcg) || in __mem_cgroup_largest_soft_limit_node()
544 !css_tryget(&mz->memcg->css)) in __mem_cgroup_largest_soft_limit_node()
607 static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) in memcg_rstat_updated() argument
611 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); in memcg_rstat_updated()
723 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) in memcg_page_state() argument
725 long x = READ_ONCE(memcg->vmstats->state[idx]); in memcg_page_state()
739 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) in __mod_memcg_state() argument
744 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); in __mod_memcg_state()
745 memcg_rstat_updated(memcg, val); in __mod_memcg_state()
749 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) in memcg_page_state_local() argument
755 x += per_cpu(memcg->vmstats_percpu->state[idx], cpu); in memcg_page_state_local()
767 struct mem_cgroup *memcg; in __mod_memcg_lruvec_state() local
770 memcg = pn->memcg; in __mod_memcg_lruvec_state()
794 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); in __mod_memcg_lruvec_state()
799 memcg_rstat_updated(memcg, val); in __mod_memcg_lruvec_state()
828 struct mem_cgroup *memcg; in __mod_lruvec_page_state() local
833 memcg = page_memcg(head); in __mod_lruvec_page_state()
835 if (!memcg) { in __mod_lruvec_page_state()
841 lruvec = mem_cgroup_lruvec(memcg, pgdat); in __mod_lruvec_page_state()
850 struct mem_cgroup *memcg; in __mod_lruvec_kmem_state() local
854 memcg = mem_cgroup_from_slab_obj(p); in __mod_lruvec_kmem_state()
862 if (!memcg) { in __mod_lruvec_kmem_state()
865 lruvec = mem_cgroup_lruvec(memcg, pgdat); in __mod_lruvec_kmem_state()
877 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, in __count_memcg_events() argument
886 __this_cpu_add(memcg->vmstats_percpu->events[index], count); in __count_memcg_events()
887 memcg_rstat_updated(memcg, count); in __count_memcg_events()
891 static unsigned long memcg_events(struct mem_cgroup *memcg, int event) in memcg_events() argument
897 return READ_ONCE(memcg->vmstats->events[index]); in memcg_events()
900 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) in memcg_events_local() argument
910 x += per_cpu(memcg->vmstats_percpu->events[index], cpu); in memcg_events_local()
914 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, in mem_cgroup_charge_statistics() argument
919 __count_memcg_events(memcg, PGPGIN, 1); in mem_cgroup_charge_statistics()
921 __count_memcg_events(memcg, PGPGOUT, 1); in mem_cgroup_charge_statistics()
925 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages); in mem_cgroup_charge_statistics()
928 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, in mem_cgroup_event_ratelimit() argument
933 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events); in mem_cgroup_event_ratelimit()
934 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]); in mem_cgroup_event_ratelimit()
947 __this_cpu_write(memcg->vmstats_percpu->targets[target], next); in mem_cgroup_event_ratelimit()
957 static void memcg_check_events(struct mem_cgroup *memcg, int nid) in memcg_check_events() argument
963 if (unlikely(mem_cgroup_event_ratelimit(memcg, in memcg_check_events()
967 do_softlimit = mem_cgroup_event_ratelimit(memcg, in memcg_check_events()
969 mem_cgroup_threshold(memcg); in memcg_check_events()
971 mem_cgroup_update_tree(memcg, nid); in memcg_check_events()
1010 struct mem_cgroup *memcg; in get_mem_cgroup_from_mm() local
1025 memcg = active_memcg(); in get_mem_cgroup_from_mm()
1026 if (unlikely(memcg)) { in get_mem_cgroup_from_mm()
1028 css_get(&memcg->css); in get_mem_cgroup_from_mm()
1029 return memcg; in get_mem_cgroup_from_mm()
1038 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); in get_mem_cgroup_from_mm()
1039 if (unlikely(!memcg)) in get_mem_cgroup_from_mm()
1040 memcg = root_mem_cgroup; in get_mem_cgroup_from_mm()
1041 } while (!css_tryget(&memcg->css)); in get_mem_cgroup_from_mm()
1043 return memcg; in get_mem_cgroup_from_mm()
1083 struct mem_cgroup *memcg = NULL; in mem_cgroup_iter() local
1150 memcg = mem_cgroup_from_css(css); in mem_cgroup_iter()
1161 (void)cmpxchg(&iter->position, pos, memcg); in mem_cgroup_iter()
1166 if (!memcg) in mem_cgroup_iter()
1175 return memcg; in mem_cgroup_iter()
1208 struct mem_cgroup *memcg = dead_memcg; in invalidate_reclaim_iterators() local
1212 __invalidate_reclaim_iterators(memcg, dead_memcg); in invalidate_reclaim_iterators()
1213 last = memcg; in invalidate_reclaim_iterators()
1214 } while ((memcg = parent_mem_cgroup(memcg))); in invalidate_reclaim_iterators()
1240 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, in mem_cgroup_scan_tasks() argument
1246 BUG_ON(memcg == root_mem_cgroup); in mem_cgroup_scan_tasks()
1248 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_scan_tasks()
1257 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_scan_tasks()
1267 struct mem_cgroup *memcg; in lruvec_memcg_debug() local
1272 memcg = folio_memcg(folio); in lruvec_memcg_debug()
1274 if (!memcg) in lruvec_memcg_debug()
1277 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio); in lruvec_memcg_debug()
1396 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) in mem_cgroup_margin() argument
1402 count = page_counter_read(&memcg->memory); in mem_cgroup_margin()
1403 limit = READ_ONCE(memcg->memory.max); in mem_cgroup_margin()
1408 count = page_counter_read(&memcg->memsw); in mem_cgroup_margin()
1409 limit = READ_ONCE(memcg->memsw.max); in mem_cgroup_margin()
1426 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) in mem_cgroup_under_move() argument
1441 ret = mem_cgroup_is_descendant(from, memcg) || in mem_cgroup_under_move()
1442 mem_cgroup_is_descendant(to, memcg); in mem_cgroup_under_move()
1448 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) in mem_cgroup_wait_acct_move() argument
1451 if (mem_cgroup_under_move(memcg)) { in mem_cgroup_wait_acct_move()
1536 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, in memcg_page_state_output() argument
1539 return memcg_page_state(memcg, item) * memcg_page_state_unit(item); in memcg_page_state_output()
1542 static void memory_stat_format(struct mem_cgroup *memcg, char *buf, int bufsize) in memory_stat_format() argument
1564 size = memcg_page_state_output(memcg, memory_stats[i].idx); in memory_stat_format()
1568 size += memcg_page_state_output(memcg, in memory_stat_format()
1576 memcg_events(memcg, PGSCAN_KSWAPD) + in memory_stat_format()
1577 memcg_events(memcg, PGSCAN_DIRECT)); in memory_stat_format()
1579 memcg_events(memcg, PGSTEAL_KSWAPD) + in memory_stat_format()
1580 memcg_events(memcg, PGSTEAL_DIRECT)); in memory_stat_format()
1589 memcg_events(memcg, memcg_vm_event_stat[i])); in memory_stat_format()
1606 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) in mem_cgroup_print_oom_context() argument
1610 if (memcg) { in mem_cgroup_print_oom_context()
1612 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_context()
1627 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) in mem_cgroup_print_oom_meminfo() argument
1635 K((u64)page_counter_read(&memcg->memory)), in mem_cgroup_print_oom_meminfo()
1636 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt); in mem_cgroup_print_oom_meminfo()
1639 K((u64)page_counter_read(&memcg->swap)), in mem_cgroup_print_oom_meminfo()
1640 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt); in mem_cgroup_print_oom_meminfo()
1643 K((u64)page_counter_read(&memcg->memsw)), in mem_cgroup_print_oom_meminfo()
1644 K((u64)memcg->memsw.max), memcg->memsw.failcnt); in mem_cgroup_print_oom_meminfo()
1646 K((u64)page_counter_read(&memcg->kmem)), in mem_cgroup_print_oom_meminfo()
1647 K((u64)memcg->kmem.max), memcg->kmem.failcnt); in mem_cgroup_print_oom_meminfo()
1651 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_meminfo()
1653 memory_stat_format(memcg, buf, sizeof(buf)); in mem_cgroup_print_oom_meminfo()
1660 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) in mem_cgroup_get_max() argument
1662 unsigned long max = READ_ONCE(memcg->memory.max); in mem_cgroup_get_max()
1665 if (mem_cgroup_swappiness(memcg)) { in mem_cgroup_get_max()
1667 unsigned long swap = READ_ONCE(memcg->memsw.max) - max; in mem_cgroup_get_max()
1672 if (mem_cgroup_swappiness(memcg)) in mem_cgroup_get_max()
1673 max += min(READ_ONCE(memcg->swap.max), in mem_cgroup_get_max()
1679 unsigned long mem_cgroup_size(struct mem_cgroup *memcg) in mem_cgroup_size() argument
1681 return page_counter_read(&memcg->memory); in mem_cgroup_size()
1684 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, in mem_cgroup_out_of_memory() argument
1690 .memcg = memcg, in mem_cgroup_out_of_memory()
1699 if (mem_cgroup_margin(memcg) >= (1 << order)) in mem_cgroup_out_of_memory()
1775 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) in mem_cgroup_oom_trylock() argument
1781 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_oom_trylock()
1788 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_oom_trylock()
1799 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_oom_trylock()
1801 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_oom_trylock()
1814 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) in mem_cgroup_oom_unlock() argument
1820 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_oom_unlock()
1825 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) in mem_cgroup_mark_under_oom() argument
1830 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_mark_under_oom()
1835 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) in mem_cgroup_unmark_under_oom() argument
1844 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_unmark_under_oom()
1853 struct mem_cgroup *memcg; member
1865 oom_wait_memcg = oom_wait_info->memcg; in memcg_oom_wake_function()
1873 static void memcg_oom_recover(struct mem_cgroup *memcg) in memcg_oom_recover() argument
1883 if (memcg && memcg->under_oom) in memcg_oom_recover()
1884 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); in memcg_oom_recover()
1891 static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) in mem_cgroup_oom() argument
1898 memcg_memory_event(memcg, MEMCG_OOM); in mem_cgroup_oom()
1918 if (memcg->oom_kill_disable) { in mem_cgroup_oom()
1920 css_get(&memcg->css); in mem_cgroup_oom()
1921 current->memcg_in_oom = memcg; in mem_cgroup_oom()
1928 mem_cgroup_mark_under_oom(memcg); in mem_cgroup_oom()
1930 locked = mem_cgroup_oom_trylock(memcg); in mem_cgroup_oom()
1933 mem_cgroup_oom_notify(memcg); in mem_cgroup_oom()
1935 mem_cgroup_unmark_under_oom(memcg); in mem_cgroup_oom()
1936 ret = mem_cgroup_out_of_memory(memcg, mask, order); in mem_cgroup_oom()
1939 mem_cgroup_oom_unlock(memcg); in mem_cgroup_oom()
1963 struct mem_cgroup *memcg = current->memcg_in_oom; in mem_cgroup_oom_synchronize() local
1968 if (!memcg) in mem_cgroup_oom_synchronize()
1974 owait.memcg = memcg; in mem_cgroup_oom_synchronize()
1981 mem_cgroup_mark_under_oom(memcg); in mem_cgroup_oom_synchronize()
1983 locked = mem_cgroup_oom_trylock(memcg); in mem_cgroup_oom_synchronize()
1986 mem_cgroup_oom_notify(memcg); in mem_cgroup_oom_synchronize()
1988 if (locked && !memcg->oom_kill_disable) { in mem_cgroup_oom_synchronize()
1989 mem_cgroup_unmark_under_oom(memcg); in mem_cgroup_oom_synchronize()
1991 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, in mem_cgroup_oom_synchronize()
1995 mem_cgroup_unmark_under_oom(memcg); in mem_cgroup_oom_synchronize()
2000 mem_cgroup_oom_unlock(memcg); in mem_cgroup_oom_synchronize()
2006 memcg_oom_recover(memcg); in mem_cgroup_oom_synchronize()
2010 css_put(&memcg->css); in mem_cgroup_oom_synchronize()
2028 struct mem_cgroup *memcg; in mem_cgroup_get_oom_group() local
2038 memcg = mem_cgroup_from_task(victim); in mem_cgroup_get_oom_group()
2039 if (memcg == root_mem_cgroup) in mem_cgroup_get_oom_group()
2047 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain))) in mem_cgroup_get_oom_group()
2055 for (; memcg; memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_get_oom_group()
2056 if (memcg->oom_group) in mem_cgroup_get_oom_group()
2057 oom_group = memcg; in mem_cgroup_get_oom_group()
2059 if (memcg == oom_domain) in mem_cgroup_get_oom_group()
2071 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) in mem_cgroup_print_oom_group() argument
2074 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_group()
2090 struct mem_cgroup *memcg; in folio_memcg_lock() local
2103 memcg = folio_memcg(folio); in folio_memcg_lock()
2104 if (unlikely(!memcg)) in folio_memcg_lock()
2109 might_lock(&memcg->move_lock); in folio_memcg_lock()
2113 if (atomic_read(&memcg->moving_account) <= 0) in folio_memcg_lock()
2116 spin_lock_irqsave(&memcg->move_lock, flags); in folio_memcg_lock()
2117 if (memcg != folio_memcg(folio)) { in folio_memcg_lock()
2118 spin_unlock_irqrestore(&memcg->move_lock, flags); in folio_memcg_lock()
2128 memcg->move_lock_task = current; in folio_memcg_lock()
2129 memcg->move_lock_flags = flags; in folio_memcg_lock()
2137 static void __folio_memcg_unlock(struct mem_cgroup *memcg) in __folio_memcg_unlock() argument
2139 if (memcg && memcg->move_lock_task == current) { in __folio_memcg_unlock()
2140 unsigned long flags = memcg->move_lock_flags; in __folio_memcg_unlock()
2142 memcg->move_lock_task = NULL; in __folio_memcg_unlock()
2143 memcg->move_lock_flags = 0; in __folio_memcg_unlock()
2145 spin_unlock_irqrestore(&memcg->move_lock, flags); in __folio_memcg_unlock()
2195 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages);
2207 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) in memcg_account_kmem() argument
2223 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in consume_stock() argument
2235 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { in consume_stock()
2293 static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in __refill_stock() argument
2298 if (stock->cached != memcg) { /* reset if necessary */ in __refill_stock()
2300 css_get(&memcg->css); in __refill_stock()
2301 stock->cached = memcg; in __refill_stock()
2309 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in refill_stock() argument
2314 __refill_stock(memcg, nr_pages); in refill_stock()
2339 struct mem_cgroup *memcg; in drain_all_stock() local
2343 memcg = stock->cached; in drain_all_stock()
2344 if (memcg && stock->nr_pages && in drain_all_stock()
2345 mem_cgroup_is_descendant(memcg, root_memcg)) in drain_all_stock()
2373 static unsigned long reclaim_high(struct mem_cgroup *memcg, in reclaim_high() argument
2382 if (page_counter_read(&memcg->memory) <= in reclaim_high()
2383 READ_ONCE(memcg->memory.high)) in reclaim_high()
2386 memcg_memory_event(memcg, MEMCG_HIGH); in reclaim_high()
2389 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages, in reclaim_high()
2393 } while ((memcg = parent_mem_cgroup(memcg)) && in reclaim_high()
2394 !mem_cgroup_is_root(memcg)); in reclaim_high()
2401 struct mem_cgroup *memcg; in high_work_func() local
2403 memcg = container_of(work, struct mem_cgroup, high_work); in high_work_func()
2404 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); in high_work_func()
2478 static u64 mem_find_max_overage(struct mem_cgroup *memcg) in mem_find_max_overage() argument
2483 overage = calculate_overage(page_counter_read(&memcg->memory), in mem_find_max_overage()
2484 READ_ONCE(memcg->memory.high)); in mem_find_max_overage()
2486 } while ((memcg = parent_mem_cgroup(memcg)) && in mem_find_max_overage()
2487 !mem_cgroup_is_root(memcg)); in mem_find_max_overage()
2492 static u64 swap_find_max_overage(struct mem_cgroup *memcg) in swap_find_max_overage() argument
2497 overage = calculate_overage(page_counter_read(&memcg->swap), in swap_find_max_overage()
2498 READ_ONCE(memcg->swap.high)); in swap_find_max_overage()
2500 memcg_memory_event(memcg, MEMCG_SWAP_HIGH); in swap_find_max_overage()
2502 } while ((memcg = parent_mem_cgroup(memcg)) && in swap_find_max_overage()
2503 !mem_cgroup_is_root(memcg)); in swap_find_max_overage()
2512 static unsigned long calculate_high_delay(struct mem_cgroup *memcg, in calculate_high_delay() argument
2555 struct mem_cgroup *memcg; in mem_cgroup_handle_over_high() local
2561 memcg = get_mem_cgroup_from_mm(current->mm); in mem_cgroup_handle_over_high()
2574 nr_reclaimed = reclaim_high(memcg, in mem_cgroup_handle_over_high()
2582 penalty_jiffies = calculate_high_delay(memcg, nr_pages, in mem_cgroup_handle_over_high()
2583 mem_find_max_overage(memcg)); in mem_cgroup_handle_over_high()
2585 penalty_jiffies += calculate_high_delay(memcg, nr_pages, in mem_cgroup_handle_over_high()
2586 swap_find_max_overage(memcg)); in mem_cgroup_handle_over_high()
2624 css_put(&memcg->css); in mem_cgroup_handle_over_high()
2627 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, in try_charge_memcg() argument
2642 if (consume_stock(memcg, nr_pages)) in try_charge_memcg()
2646 page_counter_try_charge(&memcg->memsw, batch, &counter)) { in try_charge_memcg()
2647 if (page_counter_try_charge(&memcg->memory, batch, &counter)) in try_charge_memcg()
2650 page_counter_uncharge(&memcg->memsw, batch); in try_charge_memcg()
2757 page_counter_charge(&memcg->memory, nr_pages); in try_charge_memcg()
2759 page_counter_charge(&memcg->memsw, nr_pages); in try_charge_memcg()
2765 refill_stock(memcg, batch - nr_pages); in try_charge_memcg()
2779 mem_high = page_counter_read(&memcg->memory) > in try_charge_memcg()
2780 READ_ONCE(memcg->memory.high); in try_charge_memcg()
2781 swap_high = page_counter_read(&memcg->swap) > in try_charge_memcg()
2782 READ_ONCE(memcg->swap.high); in try_charge_memcg()
2787 schedule_work(&memcg->high_work); in try_charge_memcg()
2807 } while ((memcg = parent_mem_cgroup(memcg))); in try_charge_memcg()
2817 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, in try_charge() argument
2820 if (mem_cgroup_is_root(memcg)) in try_charge()
2823 return try_charge_memcg(memcg, gfp_mask, nr_pages); in try_charge()
2826 static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) in cancel_charge() argument
2828 if (mem_cgroup_is_root(memcg)) in cancel_charge()
2831 page_counter_uncharge(&memcg->memory, nr_pages); in cancel_charge()
2833 page_counter_uncharge(&memcg->memsw, nr_pages); in cancel_charge()
2836 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) in commit_charge() argument
2848 folio->memcg_data = (unsigned long)memcg; in commit_charge()
2867 struct mem_cgroup *memcg; in mod_objcg_mlstate() local
2871 memcg = obj_cgroup_memcg(objcg); in mod_objcg_mlstate()
2872 lruvec = mem_cgroup_lruvec(memcg, pgdat); in mod_objcg_mlstate()
2994 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) in __get_obj_cgroup_from_memcg() argument
2998 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) { in __get_obj_cgroup_from_memcg()
2999 objcg = rcu_dereference(memcg->objcg); in __get_obj_cgroup_from_memcg()
3010 struct mem_cgroup *memcg; in get_obj_cgroup_from_current() local
3017 memcg = active_memcg(); in get_obj_cgroup_from_current()
3019 memcg = mem_cgroup_from_task(current); in get_obj_cgroup_from_current()
3020 objcg = __get_obj_cgroup_from_memcg(memcg); in get_obj_cgroup_from_current()
3036 struct mem_cgroup *memcg; in get_obj_cgroup_from_page() local
3039 memcg = __folio_memcg(page_folio(page)); in get_obj_cgroup_from_page()
3040 if (memcg) in get_obj_cgroup_from_page()
3041 objcg = __get_obj_cgroup_from_memcg(memcg); in get_obj_cgroup_from_page()
3049 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) in memcg_account_kmem() argument
3051 mod_memcg_state(memcg, MEMCG_KMEM, nr_pages); in memcg_account_kmem()
3054 page_counter_charge(&memcg->kmem, nr_pages); in memcg_account_kmem()
3056 page_counter_uncharge(&memcg->kmem, -nr_pages); in memcg_account_kmem()
3069 struct mem_cgroup *memcg; in obj_cgroup_uncharge_pages() local
3071 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_uncharge_pages()
3073 memcg_account_kmem(memcg, -nr_pages); in obj_cgroup_uncharge_pages()
3074 refill_stock(memcg, nr_pages); in obj_cgroup_uncharge_pages()
3076 css_put(&memcg->css); in obj_cgroup_uncharge_pages()
3090 struct mem_cgroup *memcg; in obj_cgroup_charge_pages() local
3093 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_charge_pages()
3095 ret = try_charge_memcg(memcg, gfp, nr_pages); in obj_cgroup_charge_pages()
3099 memcg_account_kmem(memcg, nr_pages); in obj_cgroup_charge_pages()
3101 css_put(&memcg->css); in obj_cgroup_charge_pages()
3249 struct mem_cgroup *memcg; in drain_obj_stock() local
3251 memcg = get_mem_cgroup_from_objcg(old); in drain_obj_stock()
3253 memcg_account_kmem(memcg, -nr_pages); in drain_obj_stock()
3254 __refill_stock(memcg, nr_pages); in drain_obj_stock()
3256 css_put(&memcg->css); in drain_obj_stock()
3303 struct mem_cgroup *memcg; in obj_stock_flush_required() local
3306 memcg = obj_cgroup_memcg(stock->cached_objcg); in obj_stock_flush_required()
3307 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) in obj_stock_flush_required()
3405 struct mem_cgroup *memcg = folio_memcg(folio); in split_page_memcg() local
3408 if (mem_cgroup_disabled() || !memcg) in split_page_memcg()
3417 css_get_many(&memcg->css, nr - 1); in split_page_memcg()
3460 static int mem_cgroup_resize_max(struct mem_cgroup *memcg, in mem_cgroup_resize_max() argument
3467 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory; in mem_cgroup_resize_max()
3480 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) : in mem_cgroup_resize_max()
3481 max <= memcg->memsw.max; in mem_cgroup_resize_max()
3496 drain_all_stock(memcg); in mem_cgroup_resize_max()
3501 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, in mem_cgroup_resize_max()
3509 memcg_oom_recover(memcg); in mem_cgroup_resize_max()
3551 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, in mem_cgroup_soft_limit_reclaim()
3564 excess = soft_limit_excess(mz->memcg); in mem_cgroup_soft_limit_reclaim()
3576 css_put(&mz->memcg->css); in mem_cgroup_soft_limit_reclaim()
3589 css_put(&next_mz->memcg->css); in mem_cgroup_soft_limit_reclaim()
3598 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) in mem_cgroup_force_empty() argument
3605 drain_all_stock(memcg); in mem_cgroup_force_empty()
3608 while (nr_retries && page_counter_read(&memcg->memory)) { in mem_cgroup_force_empty()
3612 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, in mem_cgroup_force_empty()
3624 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_force_empty_write() local
3626 if (mem_cgroup_is_root(memcg)) in mem_cgroup_force_empty_write()
3628 return mem_cgroup_force_empty(memcg) ?: nbytes; in mem_cgroup_force_empty_write()
3650 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) in mem_cgroup_usage() argument
3654 if (mem_cgroup_is_root(memcg)) { in mem_cgroup_usage()
3656 val = memcg_page_state(memcg, NR_FILE_PAGES) + in mem_cgroup_usage()
3657 memcg_page_state(memcg, NR_ANON_MAPPED); in mem_cgroup_usage()
3659 val += memcg_page_state(memcg, MEMCG_SWAP); in mem_cgroup_usage()
3662 val = page_counter_read(&memcg->memory); in mem_cgroup_usage()
3664 val = page_counter_read(&memcg->memsw); in mem_cgroup_usage()
3680 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_read_u64() local
3685 counter = &memcg->memory; in mem_cgroup_read_u64()
3688 counter = &memcg->memsw; in mem_cgroup_read_u64()
3691 counter = &memcg->kmem; in mem_cgroup_read_u64()
3694 counter = &memcg->tcpmem; in mem_cgroup_read_u64()
3702 if (counter == &memcg->memory) in mem_cgroup_read_u64()
3703 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; in mem_cgroup_read_u64()
3704 if (counter == &memcg->memsw) in mem_cgroup_read_u64()
3705 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; in mem_cgroup_read_u64()
3714 return (u64)memcg->soft_limit * PAGE_SIZE; in mem_cgroup_read_u64()
3721 static int memcg_online_kmem(struct mem_cgroup *memcg) in memcg_online_kmem() argument
3728 if (unlikely(mem_cgroup_is_root(memcg))) in memcg_online_kmem()
3735 objcg->memcg = memcg; in memcg_online_kmem()
3736 rcu_assign_pointer(memcg->objcg, objcg); in memcg_online_kmem()
3740 memcg->kmemcg_id = memcg->id.id; in memcg_online_kmem()
3745 static void memcg_offline_kmem(struct mem_cgroup *memcg) in memcg_offline_kmem() argument
3752 if (unlikely(mem_cgroup_is_root(memcg))) in memcg_offline_kmem()
3755 parent = parent_mem_cgroup(memcg); in memcg_offline_kmem()
3759 memcg_reparent_objcgs(memcg, parent); in memcg_offline_kmem()
3767 memcg_reparent_list_lrus(memcg, parent); in memcg_offline_kmem()
3770 static int memcg_online_kmem(struct mem_cgroup *memcg) in memcg_online_kmem() argument
3774 static void memcg_offline_kmem(struct mem_cgroup *memcg) in memcg_offline_kmem() argument
3779 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max) in memcg_update_tcp_max() argument
3785 ret = page_counter_set_max(&memcg->tcpmem, max); in memcg_update_tcp_max()
3789 if (!memcg->tcpmem_active) { in memcg_update_tcp_max()
3807 memcg->tcpmem_active = true; in memcg_update_tcp_max()
3821 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_write() local
3832 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ in mem_cgroup_write()
3838 ret = mem_cgroup_resize_max(memcg, nr_pages, false); in mem_cgroup_write()
3841 ret = mem_cgroup_resize_max(memcg, nr_pages, true); in mem_cgroup_write()
3848 ret = memcg_update_tcp_max(memcg, nr_pages); in mem_cgroup_write()
3856 memcg->soft_limit = nr_pages; in mem_cgroup_write()
3867 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_reset() local
3872 counter = &memcg->memory; in mem_cgroup_reset()
3875 counter = &memcg->memsw; in mem_cgroup_reset()
3878 counter = &memcg->kmem; in mem_cgroup_reset()
3881 counter = &memcg->tcpmem; in mem_cgroup_reset()
3911 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_move_charge_write() local
3922 memcg->move_charge_at_immigrate = val; in mem_cgroup_move_charge_write()
3939 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, in mem_cgroup_node_nr_lru_pages() argument
3942 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); in mem_cgroup_node_nr_lru_pages()
3959 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, in mem_cgroup_nr_lru_pages() argument
3970 nr += memcg_page_state(memcg, NR_LRU_BASE + lru); in mem_cgroup_nr_lru_pages()
3972 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru); in mem_cgroup_nr_lru_pages()
3992 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memcg_numa_stat_show() local
3998 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, in memcg_numa_stat_show()
4002 mem_cgroup_node_nr_lru_pages(memcg, nid, in memcg_numa_stat_show()
4010 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, in memcg_numa_stat_show()
4014 mem_cgroup_node_nr_lru_pages(memcg, nid, in memcg_numa_stat_show()
4063 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memcg_stat_show() local
4077 nr = memcg_page_state_local(memcg, memcg1_stats[i]); in memcg_stat_show()
4084 memcg_events_local(memcg, memcg1_events[i])); in memcg_stat_show()
4088 memcg_page_state_local(memcg, NR_LRU_BASE + i) * in memcg_stat_show()
4093 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { in memcg_stat_show()
4108 nr = memcg_page_state(memcg, memcg1_stats[i]); in memcg_stat_show()
4116 (u64)memcg_events(memcg, memcg1_events[i])); in memcg_stat_show()
4120 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) * in memcg_stat_show()
4131 mz = memcg->nodeinfo[pgdat->node_id]; in memcg_stat_show()
4147 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_swappiness_read() local
4149 return mem_cgroup_swappiness(memcg); in mem_cgroup_swappiness_read()
4155 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_swappiness_write() local
4160 if (!mem_cgroup_is_root(memcg)) in mem_cgroup_swappiness_write()
4161 memcg->swappiness = val; in mem_cgroup_swappiness_write()
4168 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) in __mem_cgroup_threshold() argument
4176 t = rcu_dereference(memcg->thresholds.primary); in __mem_cgroup_threshold()
4178 t = rcu_dereference(memcg->memsw_thresholds.primary); in __mem_cgroup_threshold()
4183 usage = mem_cgroup_usage(memcg, swap); in __mem_cgroup_threshold()
4219 static void mem_cgroup_threshold(struct mem_cgroup *memcg) in mem_cgroup_threshold() argument
4221 while (memcg) { in mem_cgroup_threshold()
4222 __mem_cgroup_threshold(memcg, false); in mem_cgroup_threshold()
4224 __mem_cgroup_threshold(memcg, true); in mem_cgroup_threshold()
4226 memcg = parent_mem_cgroup(memcg); in mem_cgroup_threshold()
4244 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) in mem_cgroup_oom_notify_cb() argument
4250 list_for_each_entry(ev, &memcg->oom_notify, list) in mem_cgroup_oom_notify_cb()
4257 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) in mem_cgroup_oom_notify() argument
4261 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_oom_notify()
4265 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, in __mem_cgroup_usage_register_event() argument
4278 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
4281 thresholds = &memcg->thresholds; in __mem_cgroup_usage_register_event()
4282 usage = mem_cgroup_usage(memcg, false); in __mem_cgroup_usage_register_event()
4284 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_register_event()
4285 usage = mem_cgroup_usage(memcg, true); in __mem_cgroup_usage_register_event()
4291 __mem_cgroup_threshold(memcg, type == _MEMSWAP); in __mem_cgroup_usage_register_event()
4340 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
4345 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, in mem_cgroup_usage_register_event() argument
4348 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); in mem_cgroup_usage_register_event()
4351 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, in memsw_cgroup_usage_register_event() argument
4354 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); in memsw_cgroup_usage_register_event()
4357 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in __mem_cgroup_usage_unregister_event() argument
4365 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
4368 thresholds = &memcg->thresholds; in __mem_cgroup_usage_unregister_event()
4369 usage = mem_cgroup_usage(memcg, false); in __mem_cgroup_usage_unregister_event()
4371 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_unregister_event()
4372 usage = mem_cgroup_usage(memcg, true); in __mem_cgroup_usage_unregister_event()
4380 __mem_cgroup_threshold(memcg, type == _MEMSWAP); in __mem_cgroup_usage_unregister_event()
4439 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
4442 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in mem_cgroup_usage_unregister_event() argument
4445 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); in mem_cgroup_usage_unregister_event()
4448 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in memsw_cgroup_usage_unregister_event() argument
4451 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); in memsw_cgroup_usage_unregister_event()
4454 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, in mem_cgroup_oom_register_event() argument
4466 list_add(&event->list, &memcg->oom_notify); in mem_cgroup_oom_register_event()
4469 if (memcg->under_oom) in mem_cgroup_oom_register_event()
4476 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, in mem_cgroup_oom_unregister_event() argument
4483 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { in mem_cgroup_oom_unregister_event()
4495 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf); in mem_cgroup_oom_control_read() local
4497 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); in mem_cgroup_oom_control_read()
4498 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); in mem_cgroup_oom_control_read()
4500 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); in mem_cgroup_oom_control_read()
4507 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_oom_control_write() local
4510 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1))) in mem_cgroup_oom_control_write()
4513 memcg->oom_kill_disable = val; in mem_cgroup_oom_control_write()
4515 memcg_oom_recover(memcg); in mem_cgroup_oom_control_write()
4524 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) in memcg_wb_domain_init() argument
4526 return wb_domain_init(&memcg->cgwb_domain, gfp); in memcg_wb_domain_init()
4529 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) in memcg_wb_domain_exit() argument
4531 wb_domain_exit(&memcg->cgwb_domain); in memcg_wb_domain_exit()
4534 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) in memcg_wb_domain_size_changed() argument
4536 wb_domain_size_changed(&memcg->cgwb_domain); in memcg_wb_domain_size_changed()
4541 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_wb_domain() local
4543 if (!memcg->css.parent) in mem_cgroup_wb_domain()
4546 return &memcg->cgwb_domain; in mem_cgroup_wb_domain()
4571 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_wb_stats() local
4576 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); in mem_cgroup_wb_stats()
4577 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); in mem_cgroup_wb_stats()
4578 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) + in mem_cgroup_wb_stats()
4579 memcg_page_state(memcg, NR_ACTIVE_FILE); in mem_cgroup_wb_stats()
4582 while ((parent = parent_mem_cgroup(memcg))) { in mem_cgroup_wb_stats()
4583 unsigned long ceiling = min(READ_ONCE(memcg->memory.max), in mem_cgroup_wb_stats()
4584 READ_ONCE(memcg->memory.high)); in mem_cgroup_wb_stats()
4585 unsigned long used = page_counter_read(&memcg->memory); in mem_cgroup_wb_stats()
4588 memcg = parent; in mem_cgroup_wb_stats()
4639 struct mem_cgroup *memcg = folio_memcg(folio); in mem_cgroup_track_foreign_dirty_slowpath() local
4654 frn = &memcg->cgwb_frn[i]; in mem_cgroup_track_foreign_dirty_slowpath()
4681 frn = &memcg->cgwb_frn[oldest]; in mem_cgroup_track_foreign_dirty_slowpath()
4691 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_flush_foreign() local
4697 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i]; in mem_cgroup_flush_foreign()
4718 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) in memcg_wb_domain_init() argument
4723 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) in memcg_wb_domain_exit() argument
4727 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) in memcg_wb_domain_size_changed() argument
4755 struct mem_cgroup *memcg = event->memcg; in memcg_event_remove() local
4759 event->unregister_event(memcg, event->eventfd); in memcg_event_remove()
4766 css_put(&memcg->css); in memcg_event_remove()
4779 struct mem_cgroup *memcg = event->memcg; in memcg_event_wake() local
4792 spin_lock(&memcg->event_list_lock); in memcg_event_wake()
4801 spin_unlock(&memcg->event_list_lock); in memcg_event_wake()
4829 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memcg_write_event_control() local
4859 event->memcg = memcg; in memcg_write_event_control()
4941 ret = event->register_event(memcg, event->eventfd, buf); in memcg_write_event_control()
4947 spin_lock_irq(&memcg->event_list_lock); in memcg_write_event_control()
4948 list_add(&event->list, &memcg->event_list); in memcg_write_event_control()
4949 spin_unlock_irq(&memcg->event_list_lock); in memcg_write_event_control()
5135 static void mem_cgroup_id_remove(struct mem_cgroup *memcg) in mem_cgroup_id_remove() argument
5137 if (memcg->id.id > 0) { in mem_cgroup_id_remove()
5138 idr_remove(&mem_cgroup_idr, memcg->id.id); in mem_cgroup_id_remove()
5139 memcg->id.id = 0; in mem_cgroup_id_remove()
5143 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg, in mem_cgroup_id_get_many() argument
5146 refcount_add(n, &memcg->id.ref); in mem_cgroup_id_get_many()
5149 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) in mem_cgroup_id_put_many() argument
5151 if (refcount_sub_and_test(n, &memcg->id.ref)) { in mem_cgroup_id_put_many()
5152 mem_cgroup_id_remove(memcg); in mem_cgroup_id_put_many()
5155 css_put(&memcg->css); in mem_cgroup_id_put_many()
5159 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) in mem_cgroup_id_put() argument
5161 mem_cgroup_id_put_many(memcg, 1); in mem_cgroup_id_put()
5181 struct mem_cgroup *memcg; in mem_cgroup_get_from_ino() local
5189 memcg = container_of(css, struct mem_cgroup, css); in mem_cgroup_get_from_ino()
5191 memcg = ERR_PTR(-ENOENT); in mem_cgroup_get_from_ino()
5195 return memcg; in mem_cgroup_get_from_ino()
5199 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) in alloc_mem_cgroup_per_node_info() argument
5215 pn->memcg = memcg; in alloc_mem_cgroup_per_node_info()
5217 memcg->nodeinfo[node] = pn; in alloc_mem_cgroup_per_node_info()
5221 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) in free_mem_cgroup_per_node_info() argument
5223 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; in free_mem_cgroup_per_node_info()
5232 static void __mem_cgroup_free(struct mem_cgroup *memcg) in __mem_cgroup_free() argument
5237 free_mem_cgroup_per_node_info(memcg, node); in __mem_cgroup_free()
5238 kfree(memcg->vmstats); in __mem_cgroup_free()
5239 free_percpu(memcg->vmstats_percpu); in __mem_cgroup_free()
5240 kfree(memcg); in __mem_cgroup_free()
5243 static void mem_cgroup_free(struct mem_cgroup *memcg) in mem_cgroup_free() argument
5245 lru_gen_exit_memcg(memcg); in mem_cgroup_free()
5246 memcg_wb_domain_exit(memcg); in mem_cgroup_free()
5247 __mem_cgroup_free(memcg); in mem_cgroup_free()
5252 struct mem_cgroup *memcg; in mem_cgroup_alloc() local
5257 memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL); in mem_cgroup_alloc()
5258 if (!memcg) in mem_cgroup_alloc()
5261 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, in mem_cgroup_alloc()
5263 if (memcg->id.id < 0) { in mem_cgroup_alloc()
5264 error = memcg->id.id; in mem_cgroup_alloc()
5268 memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL); in mem_cgroup_alloc()
5269 if (!memcg->vmstats) in mem_cgroup_alloc()
5272 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu, in mem_cgroup_alloc()
5274 if (!memcg->vmstats_percpu) in mem_cgroup_alloc()
5278 if (alloc_mem_cgroup_per_node_info(memcg, node)) in mem_cgroup_alloc()
5281 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) in mem_cgroup_alloc()
5284 INIT_WORK(&memcg->high_work, high_work_func); in mem_cgroup_alloc()
5285 INIT_LIST_HEAD(&memcg->oom_notify); in mem_cgroup_alloc()
5286 mutex_init(&memcg->thresholds_lock); in mem_cgroup_alloc()
5287 spin_lock_init(&memcg->move_lock); in mem_cgroup_alloc()
5288 vmpressure_init(&memcg->vmpressure); in mem_cgroup_alloc()
5289 INIT_LIST_HEAD(&memcg->event_list); in mem_cgroup_alloc()
5290 spin_lock_init(&memcg->event_list_lock); in mem_cgroup_alloc()
5291 memcg->socket_pressure = jiffies; in mem_cgroup_alloc()
5293 memcg->kmemcg_id = -1; in mem_cgroup_alloc()
5294 INIT_LIST_HEAD(&memcg->objcg_list); in mem_cgroup_alloc()
5297 INIT_LIST_HEAD(&memcg->cgwb_list); in mem_cgroup_alloc()
5299 memcg->cgwb_frn[i].done = in mem_cgroup_alloc()
5303 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock); in mem_cgroup_alloc()
5304 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue); in mem_cgroup_alloc()
5305 memcg->deferred_split_queue.split_queue_len = 0; in mem_cgroup_alloc()
5307 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); in mem_cgroup_alloc()
5308 lru_gen_init_memcg(memcg); in mem_cgroup_alloc()
5309 return memcg; in mem_cgroup_alloc()
5311 mem_cgroup_id_remove(memcg); in mem_cgroup_alloc()
5312 __mem_cgroup_free(memcg); in mem_cgroup_alloc()
5320 struct mem_cgroup *memcg, *old_memcg; in mem_cgroup_css_alloc() local
5323 memcg = mem_cgroup_alloc(); in mem_cgroup_css_alloc()
5325 if (IS_ERR(memcg)) in mem_cgroup_css_alloc()
5326 return ERR_CAST(memcg); in mem_cgroup_css_alloc()
5328 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); in mem_cgroup_css_alloc()
5329 memcg->soft_limit = PAGE_COUNTER_MAX; in mem_cgroup_css_alloc()
5331 memcg->zswap_max = PAGE_COUNTER_MAX; in mem_cgroup_css_alloc()
5333 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); in mem_cgroup_css_alloc()
5335 memcg->swappiness = mem_cgroup_swappiness(parent); in mem_cgroup_css_alloc()
5336 memcg->oom_kill_disable = parent->oom_kill_disable; in mem_cgroup_css_alloc()
5338 page_counter_init(&memcg->memory, &parent->memory); in mem_cgroup_css_alloc()
5339 page_counter_init(&memcg->swap, &parent->swap); in mem_cgroup_css_alloc()
5340 page_counter_init(&memcg->kmem, &parent->kmem); in mem_cgroup_css_alloc()
5341 page_counter_init(&memcg->tcpmem, &parent->tcpmem); in mem_cgroup_css_alloc()
5344 page_counter_init(&memcg->memory, NULL); in mem_cgroup_css_alloc()
5345 page_counter_init(&memcg->swap, NULL); in mem_cgroup_css_alloc()
5346 page_counter_init(&memcg->kmem, NULL); in mem_cgroup_css_alloc()
5347 page_counter_init(&memcg->tcpmem, NULL); in mem_cgroup_css_alloc()
5349 root_mem_cgroup = memcg; in mem_cgroup_css_alloc()
5350 return &memcg->css; in mem_cgroup_css_alloc()
5356 return &memcg->css; in mem_cgroup_css_alloc()
5361 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_online() local
5363 if (memcg_online_kmem(memcg)) in mem_cgroup_css_online()
5371 if (alloc_shrinker_info(memcg)) in mem_cgroup_css_online()
5375 refcount_set(&memcg->id.ref, 1); in mem_cgroup_css_online()
5378 if (unlikely(mem_cgroup_is_root(memcg))) in mem_cgroup_css_online()
5383 memcg_offline_kmem(memcg); in mem_cgroup_css_online()
5385 mem_cgroup_id_remove(memcg); in mem_cgroup_css_online()
5391 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_offline() local
5399 spin_lock_irq(&memcg->event_list_lock); in mem_cgroup_css_offline()
5400 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { in mem_cgroup_css_offline()
5404 spin_unlock_irq(&memcg->event_list_lock); in mem_cgroup_css_offline()
5406 page_counter_set_min(&memcg->memory, 0); in mem_cgroup_css_offline()
5407 page_counter_set_low(&memcg->memory, 0); in mem_cgroup_css_offline()
5409 memcg_offline_kmem(memcg); in mem_cgroup_css_offline()
5410 reparent_shrinker_deferred(memcg); in mem_cgroup_css_offline()
5411 wb_memcg_offline(memcg); in mem_cgroup_css_offline()
5413 drain_all_stock(memcg); in mem_cgroup_css_offline()
5415 mem_cgroup_id_put(memcg); in mem_cgroup_css_offline()
5420 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_released() local
5422 invalidate_reclaim_iterators(memcg); in mem_cgroup_css_released()
5427 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_free() local
5432 wb_wait_for_completion(&memcg->cgwb_frn[i].done); in mem_cgroup_css_free()
5437 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) in mem_cgroup_css_free()
5440 vmpressure_cleanup(&memcg->vmpressure); in mem_cgroup_css_free()
5441 cancel_work_sync(&memcg->high_work); in mem_cgroup_css_free()
5442 mem_cgroup_remove_from_trees(memcg); in mem_cgroup_css_free()
5443 free_shrinker_info(memcg); in mem_cgroup_css_free()
5444 mem_cgroup_free(memcg); in mem_cgroup_css_free()
5462 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_reset() local
5464 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5465 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5466 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5467 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5468 page_counter_set_min(&memcg->memory, 0); in mem_cgroup_css_reset()
5469 page_counter_set_low(&memcg->memory, 0); in mem_cgroup_css_reset()
5470 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5471 memcg->soft_limit = PAGE_COUNTER_MAX; in mem_cgroup_css_reset()
5472 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
5473 memcg_wb_domain_size_changed(memcg); in mem_cgroup_css_reset()
5478 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_rstat_flush() local
5479 struct mem_cgroup *parent = parent_mem_cgroup(memcg); in mem_cgroup_css_rstat_flush()
5484 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); in mem_cgroup_css_rstat_flush()
5492 delta = memcg->vmstats->state_pending[i]; in mem_cgroup_css_rstat_flush()
5494 memcg->vmstats->state_pending[i] = 0; in mem_cgroup_css_rstat_flush()
5507 memcg->vmstats->state[i] += delta; in mem_cgroup_css_rstat_flush()
5513 delta = memcg->vmstats->events_pending[i]; in mem_cgroup_css_rstat_flush()
5515 memcg->vmstats->events_pending[i] = 0; in mem_cgroup_css_rstat_flush()
5526 memcg->vmstats->events[i] += delta; in mem_cgroup_css_rstat_flush()
5532 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; in mem_cgroup_css_rstat_flush()
6034 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ in mem_cgroup_can_attach() local
6055 memcg = mem_cgroup_from_css(css); in mem_cgroup_can_attach()
6065 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); in mem_cgroup_can_attach()
6071 VM_BUG_ON(from == memcg); in mem_cgroup_can_attach()
6087 mc.to = memcg; in mem_cgroup_can_attach()
6313 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memory_current_read() local
6315 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; in memory_current_read()
6321 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memory_peak_read() local
6323 return (u64)memcg->memory.watermark * PAGE_SIZE; in memory_peak_read()
6335 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_min_write() local
6344 page_counter_set_min(&memcg->memory, min); in memory_min_write()
6358 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_low_write() local
6367 page_counter_set_low(&memcg->memory, low); in memory_low_write()
6381 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_high_write() local
6392 page_counter_set_high(&memcg->memory, high); in memory_high_write()
6395 unsigned long nr_pages = page_counter_read(&memcg->memory); in memory_high_write()
6405 drain_all_stock(memcg); in memory_high_write()
6410 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high, in memory_high_write()
6417 memcg_wb_domain_size_changed(memcg); in memory_high_write()
6430 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_max_write() local
6441 xchg(&memcg->memory.max, max); in memory_max_write()
6444 unsigned long nr_pages = page_counter_read(&memcg->memory); in memory_max_write()
6453 drain_all_stock(memcg); in memory_max_write()
6459 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, in memory_max_write()
6465 memcg_memory_event(memcg, MEMCG_OOM); in memory_max_write()
6466 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) in memory_max_write()
6470 memcg_wb_domain_size_changed(memcg); in memory_max_write()
6488 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_events_show() local
6490 __memory_events_show(m, memcg->memory_events); in memory_events_show()
6496 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_events_local_show() local
6498 __memory_events_show(m, memcg->memory_events_local); in memory_events_local_show()
6504 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_stat_show() local
6509 memory_stat_format(memcg, buf, PAGE_SIZE); in memory_stat_show()
6525 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_numa_stat_show() local
6540 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); in memory_numa_stat_show()
6554 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in memory_oom_group_show() local
6556 seq_printf(m, "%d\n", memcg->oom_group); in memory_oom_group_show()
6564 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_oom_group_write() local
6578 memcg->oom_group = oom_group; in memory_oom_group_write()
6586 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_reclaim() local
6612 reclaimed = try_to_free_mem_cgroup_pages(memcg, in memory_reclaim()
6838 struct mem_cgroup *memcg) in mem_cgroup_calculate_protection() argument
6856 if (memcg == root) in mem_cgroup_calculate_protection()
6859 usage = page_counter_read(&memcg->memory); in mem_cgroup_calculate_protection()
6863 parent = parent_mem_cgroup(memcg); in mem_cgroup_calculate_protection()
6866 memcg->memory.emin = READ_ONCE(memcg->memory.min); in mem_cgroup_calculate_protection()
6867 memcg->memory.elow = READ_ONCE(memcg->memory.low); in mem_cgroup_calculate_protection()
6873 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage, in mem_cgroup_calculate_protection()
6874 READ_ONCE(memcg->memory.min), in mem_cgroup_calculate_protection()
6878 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage, in mem_cgroup_calculate_protection()
6879 READ_ONCE(memcg->memory.low), in mem_cgroup_calculate_protection()
6884 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, in charge_memcg() argument
6890 ret = try_charge(memcg, gfp, nr_pages); in charge_memcg()
6894 css_get(&memcg->css); in charge_memcg()
6895 commit_charge(folio, memcg); in charge_memcg()
6898 mem_cgroup_charge_statistics(memcg, nr_pages); in charge_memcg()
6899 memcg_check_events(memcg, folio_nid(folio)); in charge_memcg()
6907 struct mem_cgroup *memcg; in __mem_cgroup_charge() local
6910 memcg = get_mem_cgroup_from_mm(mm); in __mem_cgroup_charge()
6911 ret = charge_memcg(folio, memcg, gfp); in __mem_cgroup_charge()
6912 css_put(&memcg->css); in __mem_cgroup_charge()
6932 struct mem_cgroup *memcg; in mem_cgroup_swapin_charge_folio() local
6941 memcg = mem_cgroup_from_id(id); in mem_cgroup_swapin_charge_folio()
6942 if (!memcg || !css_tryget_online(&memcg->css)) in mem_cgroup_swapin_charge_folio()
6943 memcg = get_mem_cgroup_from_mm(mm); in mem_cgroup_swapin_charge_folio()
6946 ret = charge_memcg(folio, memcg, gfp); in mem_cgroup_swapin_charge_folio()
6948 css_put(&memcg->css); in mem_cgroup_swapin_charge_folio()
6986 struct mem_cgroup *memcg; member
7003 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory); in uncharge_batch()
7005 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory); in uncharge_batch()
7007 memcg_account_kmem(ug->memcg, -ug->nr_kmem); in uncharge_batch()
7008 memcg_oom_recover(ug->memcg); in uncharge_batch()
7012 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); in uncharge_batch()
7013 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory); in uncharge_batch()
7014 memcg_check_events(ug->memcg, ug->nid); in uncharge_batch()
7018 css_put(&ug->memcg->css); in uncharge_batch()
7024 struct mem_cgroup *memcg; in uncharge_folio() local
7040 memcg = get_mem_cgroup_from_objcg(objcg); in uncharge_folio()
7042 memcg = __folio_memcg(folio); in uncharge_folio()
7045 if (!memcg) in uncharge_folio()
7048 if (ug->memcg != memcg) { in uncharge_folio()
7049 if (ug->memcg) { in uncharge_folio()
7053 ug->memcg = memcg; in uncharge_folio()
7057 css_get(&memcg->css); in uncharge_folio()
7070 if (!mem_cgroup_is_root(memcg)) in uncharge_folio()
7077 css_put(&memcg->css); in uncharge_folio()
7108 if (ug.memcg) in __mem_cgroup_uncharge_list()
7124 struct mem_cgroup *memcg; in mem_cgroup_migrate() local
7140 memcg = folio_memcg(old); in mem_cgroup_migrate()
7141 VM_WARN_ON_ONCE_FOLIO(!memcg, old); in mem_cgroup_migrate()
7142 if (!memcg) in mem_cgroup_migrate()
7146 if (!mem_cgroup_is_root(memcg)) { in mem_cgroup_migrate()
7147 page_counter_charge(&memcg->memory, nr_pages); in mem_cgroup_migrate()
7149 page_counter_charge(&memcg->memsw, nr_pages); in mem_cgroup_migrate()
7152 css_get(&memcg->css); in mem_cgroup_migrate()
7153 commit_charge(new, memcg); in mem_cgroup_migrate()
7156 mem_cgroup_charge_statistics(memcg, nr_pages); in mem_cgroup_migrate()
7157 memcg_check_events(memcg, folio_nid(new)); in mem_cgroup_migrate()
7166 struct mem_cgroup *memcg; in mem_cgroup_sk_alloc() local
7176 memcg = mem_cgroup_from_task(current); in mem_cgroup_sk_alloc()
7177 if (memcg == root_mem_cgroup) in mem_cgroup_sk_alloc()
7179 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) in mem_cgroup_sk_alloc()
7181 if (css_tryget(&memcg->css)) in mem_cgroup_sk_alloc()
7182 sk->sk_memcg = memcg; in mem_cgroup_sk_alloc()
7202 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, in mem_cgroup_charge_skmem() argument
7208 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { in mem_cgroup_charge_skmem()
7209 memcg->tcpmem_pressure = 0; in mem_cgroup_charge_skmem()
7212 memcg->tcpmem_pressure = 1; in mem_cgroup_charge_skmem()
7214 page_counter_charge(&memcg->tcpmem, nr_pages); in mem_cgroup_charge_skmem()
7220 if (try_charge(memcg, gfp_mask, nr_pages) == 0) { in mem_cgroup_charge_skmem()
7221 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages); in mem_cgroup_charge_skmem()
7233 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) in mem_cgroup_uncharge_skmem() argument
7236 page_counter_uncharge(&memcg->tcpmem, nr_pages); in mem_cgroup_uncharge_skmem()
7240 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); in mem_cgroup_uncharge_skmem()
7242 refill_stock(memcg, nr_pages); in mem_cgroup_uncharge_skmem()
7305 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) in mem_cgroup_id_get_online() argument
7307 while (!refcount_inc_not_zero(&memcg->id.ref)) { in mem_cgroup_id_get_online()
7312 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) { in mem_cgroup_id_get_online()
7316 memcg = parent_mem_cgroup(memcg); in mem_cgroup_id_get_online()
7317 if (!memcg) in mem_cgroup_id_get_online()
7318 memcg = root_mem_cgroup; in mem_cgroup_id_get_online()
7320 return memcg; in mem_cgroup_id_get_online()
7332 struct mem_cgroup *memcg, *swap_memcg; in mem_cgroup_swapout() local
7345 memcg = folio_memcg(folio); in mem_cgroup_swapout()
7347 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); in mem_cgroup_swapout()
7348 if (!memcg) in mem_cgroup_swapout()
7356 swap_memcg = mem_cgroup_id_get_online(memcg); in mem_cgroup_swapout()
7368 if (!mem_cgroup_is_root(memcg)) in mem_cgroup_swapout()
7369 page_counter_uncharge(&memcg->memory, nr_entries); in mem_cgroup_swapout()
7371 if (memcg != swap_memcg) { in mem_cgroup_swapout()
7374 page_counter_uncharge(&memcg->memsw, nr_entries); in mem_cgroup_swapout()
7384 mem_cgroup_charge_statistics(memcg, -nr_entries); in mem_cgroup_swapout()
7386 memcg_check_events(memcg, folio_nid(folio)); in mem_cgroup_swapout()
7388 css_put(&memcg->css); in mem_cgroup_swapout()
7404 struct mem_cgroup *memcg; in __mem_cgroup_try_charge_swap() local
7410 memcg = folio_memcg(folio); in __mem_cgroup_try_charge_swap()
7412 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); in __mem_cgroup_try_charge_swap()
7413 if (!memcg) in __mem_cgroup_try_charge_swap()
7417 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); in __mem_cgroup_try_charge_swap()
7421 memcg = mem_cgroup_id_get_online(memcg); in __mem_cgroup_try_charge_swap()
7423 if (!mem_cgroup_is_root(memcg) && in __mem_cgroup_try_charge_swap()
7424 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { in __mem_cgroup_try_charge_swap()
7425 memcg_memory_event(memcg, MEMCG_SWAP_MAX); in __mem_cgroup_try_charge_swap()
7426 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); in __mem_cgroup_try_charge_swap()
7427 mem_cgroup_id_put(memcg); in __mem_cgroup_try_charge_swap()
7433 mem_cgroup_id_get_many(memcg, nr_pages - 1); in __mem_cgroup_try_charge_swap()
7434 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); in __mem_cgroup_try_charge_swap()
7436 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages); in __mem_cgroup_try_charge_swap()
7448 struct mem_cgroup *memcg; in __mem_cgroup_uncharge_swap() local
7456 memcg = mem_cgroup_from_id(id); in __mem_cgroup_uncharge_swap()
7457 if (memcg) { in __mem_cgroup_uncharge_swap()
7458 if (!mem_cgroup_is_root(memcg)) { in __mem_cgroup_uncharge_swap()
7460 page_counter_uncharge(&memcg->memsw, nr_pages); in __mem_cgroup_uncharge_swap()
7462 page_counter_uncharge(&memcg->swap, nr_pages); in __mem_cgroup_uncharge_swap()
7464 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); in __mem_cgroup_uncharge_swap()
7465 mem_cgroup_id_put_many(memcg, nr_pages); in __mem_cgroup_uncharge_swap()
7470 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) in mem_cgroup_get_nr_swap_pages() argument
7476 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) in mem_cgroup_get_nr_swap_pages()
7478 READ_ONCE(memcg->swap.max) - in mem_cgroup_get_nr_swap_pages()
7479 page_counter_read(&memcg->swap)); in mem_cgroup_get_nr_swap_pages()
7485 struct mem_cgroup *memcg; in mem_cgroup_swap_full() local
7494 memcg = folio_memcg(folio); in mem_cgroup_swap_full()
7495 if (!memcg) in mem_cgroup_swap_full()
7498 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_swap_full()
7499 unsigned long usage = page_counter_read(&memcg->swap); in mem_cgroup_swap_full()
7501 if (usage * 2 >= READ_ONCE(memcg->swap.high) || in mem_cgroup_swap_full()
7502 usage * 2 >= READ_ONCE(memcg->swap.max)) in mem_cgroup_swap_full()
7521 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in swap_current_read() local
7523 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; in swap_current_read()
7535 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in swap_high_write() local
7544 page_counter_set_high(&memcg->swap, high); in swap_high_write()
7558 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in swap_max_write() local
7567 xchg(&memcg->swap.max, max); in swap_max_write()
7574 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); in swap_events_show() local
7577 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH])); in swap_events_show()
7579 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX])); in swap_events_show()
7581 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL])); in swap_events_show()
7655 struct mem_cgroup *memcg, *original_memcg; in obj_cgroup_may_zswap() local
7662 for (memcg = original_memcg; memcg != root_mem_cgroup; in obj_cgroup_may_zswap()
7663 memcg = parent_mem_cgroup(memcg)) { in obj_cgroup_may_zswap()
7664 unsigned long max = READ_ONCE(memcg->zswap_max); in obj_cgroup_may_zswap()
7674 cgroup_rstat_flush(memcg->css.cgroup); in obj_cgroup_may_zswap()
7675 pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE; in obj_cgroup_may_zswap()
7695 struct mem_cgroup *memcg; in obj_cgroup_charge_zswap() local
7707 memcg = obj_cgroup_memcg(objcg); in obj_cgroup_charge_zswap()
7708 mod_memcg_state(memcg, MEMCG_ZSWAP_B, size); in obj_cgroup_charge_zswap()
7709 mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1); in obj_cgroup_charge_zswap()
7722 struct mem_cgroup *memcg; in obj_cgroup_uncharge_zswap() local
7730 memcg = obj_cgroup_memcg(objcg); in obj_cgroup_uncharge_zswap()
7731 mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size); in obj_cgroup_uncharge_zswap()
7732 mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1); in obj_cgroup_uncharge_zswap()
7752 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in zswap_max_write() local
7761 xchg(&memcg->zswap_max, max); in zswap_max_write()