Lines Matching refs:memcg
145 struct mem_cgroup *memcg; member
159 int (*register_event)(struct mem_cgroup *memcg,
166 void (*unregister_event)(struct mem_cgroup *memcg,
178 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
179 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
252 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) in memcg_to_vmpressure() argument
254 if (!memcg) in memcg_to_vmpressure()
255 memcg = root_mem_cgroup; in memcg_to_vmpressure()
256 return &memcg->vmpressure; in memcg_to_vmpressure()
326 static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg, in memcg_expand_one_shrinker_map() argument
336 mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true); in memcg_expand_one_shrinker_map()
349 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new); in memcg_expand_one_shrinker_map()
356 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) in memcg_free_shrinker_maps() argument
362 if (mem_cgroup_is_root(memcg)) in memcg_free_shrinker_maps()
366 pn = mem_cgroup_nodeinfo(memcg, nid); in memcg_free_shrinker_maps()
374 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg) in memcg_alloc_shrinker_maps() argument
379 if (mem_cgroup_is_root(memcg)) in memcg_alloc_shrinker_maps()
387 memcg_free_shrinker_maps(memcg); in memcg_alloc_shrinker_maps()
391 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map); in memcg_alloc_shrinker_maps()
401 struct mem_cgroup *memcg; in memcg_expand_shrinker_maps() local
412 for_each_mem_cgroup(memcg) { in memcg_expand_shrinker_maps()
413 if (mem_cgroup_is_root(memcg)) in memcg_expand_shrinker_maps()
415 ret = memcg_expand_one_shrinker_map(memcg, size, old_size); in memcg_expand_shrinker_maps()
426 void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) in memcg_set_shrinker_bit() argument
428 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) { in memcg_set_shrinker_bit()
432 map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map); in memcg_set_shrinker_bit()
441 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg) in memcg_alloc_shrinker_maps() argument
445 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) { } in memcg_free_shrinker_maps() argument
461 struct mem_cgroup *memcg; in mem_cgroup_css_from_page() local
463 memcg = page->mem_cgroup; in mem_cgroup_css_from_page()
465 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) in mem_cgroup_css_from_page()
466 memcg = root_mem_cgroup; in mem_cgroup_css_from_page()
468 return &memcg->css; in mem_cgroup_css_from_page()
486 struct mem_cgroup *memcg; in page_cgroup_ino() local
490 memcg = READ_ONCE(page->mem_cgroup); in page_cgroup_ino()
491 while (memcg && !(memcg->css.flags & CSS_ONLINE)) in page_cgroup_ino()
492 memcg = parent_mem_cgroup(memcg); in page_cgroup_ino()
493 if (memcg) in page_cgroup_ino()
494 ino = cgroup_ino(memcg->css.cgroup); in page_cgroup_ino()
500 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page) in mem_cgroup_page_nodeinfo() argument
504 return memcg->nodeinfo[nid]; in mem_cgroup_page_nodeinfo()
584 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) in soft_limit_excess() argument
586 unsigned long nr_pages = page_counter_read(&memcg->memory); in soft_limit_excess()
587 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); in soft_limit_excess()
596 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) in mem_cgroup_update_tree() argument
609 for (; memcg; memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_update_tree()
610 mz = mem_cgroup_page_nodeinfo(memcg, page); in mem_cgroup_update_tree()
611 excess = soft_limit_excess(memcg); in mem_cgroup_update_tree()
633 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) in mem_cgroup_remove_from_trees() argument
640 mz = mem_cgroup_nodeinfo(memcg, nid); in mem_cgroup_remove_from_trees()
665 if (!soft_limit_excess(mz->memcg) || in __mem_cgroup_largest_soft_limit_node()
666 !css_tryget_online(&mz->memcg->css)) in __mem_cgroup_largest_soft_limit_node()
683 static unsigned long memcg_sum_events(struct mem_cgroup *memcg, in memcg_sum_events() argument
686 return atomic_long_read(&memcg->events[event]); in memcg_sum_events()
689 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, in mem_cgroup_charge_statistics() argument
698 __mod_memcg_state(memcg, MEMCG_RSS, nr_pages); in mem_cgroup_charge_statistics()
700 __mod_memcg_state(memcg, MEMCG_CACHE, nr_pages); in mem_cgroup_charge_statistics()
702 __mod_memcg_state(memcg, NR_SHMEM, nr_pages); in mem_cgroup_charge_statistics()
707 __mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages); in mem_cgroup_charge_statistics()
712 __count_memcg_events(memcg, PGPGIN, 1); in mem_cgroup_charge_statistics()
714 __count_memcg_events(memcg, PGPGOUT, 1); in mem_cgroup_charge_statistics()
718 __this_cpu_add(memcg->stat_cpu->nr_page_events, nr_pages); in mem_cgroup_charge_statistics()
721 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, in mem_cgroup_node_nr_lru_pages() argument
724 struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg); in mem_cgroup_node_nr_lru_pages()
738 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, in mem_cgroup_nr_lru_pages() argument
745 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask); in mem_cgroup_nr_lru_pages()
749 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, in mem_cgroup_event_ratelimit() argument
754 val = __this_cpu_read(memcg->stat_cpu->nr_page_events); in mem_cgroup_event_ratelimit()
755 next = __this_cpu_read(memcg->stat_cpu->targets[target]); in mem_cgroup_event_ratelimit()
771 __this_cpu_write(memcg->stat_cpu->targets[target], next); in mem_cgroup_event_ratelimit()
781 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) in memcg_check_events() argument
784 if (unlikely(mem_cgroup_event_ratelimit(memcg, in memcg_check_events()
789 do_softlimit = mem_cgroup_event_ratelimit(memcg, in memcg_check_events()
792 do_numainfo = mem_cgroup_event_ratelimit(memcg, in memcg_check_events()
795 mem_cgroup_threshold(memcg); in memcg_check_events()
797 mem_cgroup_update_tree(memcg, page); in memcg_check_events()
800 atomic_inc(&memcg->numainfo_events); in memcg_check_events()
829 struct mem_cgroup *memcg; in get_mem_cgroup_from_mm() local
842 memcg = root_mem_cgroup; in get_mem_cgroup_from_mm()
844 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); in get_mem_cgroup_from_mm()
845 if (unlikely(!memcg)) in get_mem_cgroup_from_mm()
846 memcg = root_mem_cgroup; in get_mem_cgroup_from_mm()
848 } while (!css_tryget_online(&memcg->css)); in get_mem_cgroup_from_mm()
850 return memcg; in get_mem_cgroup_from_mm()
863 struct mem_cgroup *memcg = page->mem_cgroup; in get_mem_cgroup_from_page() local
869 if (!memcg || !css_tryget_online(&memcg->css)) in get_mem_cgroup_from_page()
870 memcg = root_mem_cgroup; in get_mem_cgroup_from_page()
872 return memcg; in get_mem_cgroup_from_page()
882 struct mem_cgroup *memcg = root_mem_cgroup; in get_mem_cgroup_from_current() local
886 memcg = current->active_memcg; in get_mem_cgroup_from_current()
888 return memcg; in get_mem_cgroup_from_current()
916 struct mem_cgroup *memcg = NULL; in mem_cgroup_iter() local
983 memcg = mem_cgroup_from_css(css); in mem_cgroup_iter()
991 memcg = NULL; in mem_cgroup_iter()
1000 (void)cmpxchg(&iter->position, pos, memcg); in mem_cgroup_iter()
1005 if (!memcg) in mem_cgroup_iter()
1017 return memcg; in mem_cgroup_iter()
1036 struct mem_cgroup *memcg = dead_memcg; in invalidate_reclaim_iterators() local
1042 for (; memcg; memcg = parent_mem_cgroup(memcg)) { in invalidate_reclaim_iterators()
1044 mz = mem_cgroup_nodeinfo(memcg, nid); in invalidate_reclaim_iterators()
1067 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, in mem_cgroup_scan_tasks() argument
1073 BUG_ON(memcg == root_mem_cgroup); in mem_cgroup_scan_tasks()
1075 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_scan_tasks()
1084 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_scan_tasks()
1103 struct mem_cgroup *memcg; in mem_cgroup_page_lruvec() local
1111 memcg = page->mem_cgroup; in mem_cgroup_page_lruvec()
1116 if (!memcg) in mem_cgroup_page_lruvec()
1117 memcg = root_mem_cgroup; in mem_cgroup_page_lruvec()
1119 mz = mem_cgroup_page_nodeinfo(memcg, page); in mem_cgroup_page_lruvec()
1171 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg) in task_in_mem_cgroup() argument
1192 ret = mem_cgroup_is_descendant(task_memcg, memcg); in task_in_mem_cgroup()
1204 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) in mem_cgroup_margin() argument
1210 count = page_counter_read(&memcg->memory); in mem_cgroup_margin()
1211 limit = READ_ONCE(memcg->memory.max); in mem_cgroup_margin()
1216 count = page_counter_read(&memcg->memsw); in mem_cgroup_margin()
1217 limit = READ_ONCE(memcg->memsw.max); in mem_cgroup_margin()
1234 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) in mem_cgroup_under_move() argument
1249 ret = mem_cgroup_is_descendant(from, memcg) || in mem_cgroup_under_move()
1250 mem_cgroup_is_descendant(to, memcg); in mem_cgroup_under_move()
1256 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) in mem_cgroup_wait_acct_move() argument
1259 if (mem_cgroup_under_move(memcg)) { in mem_cgroup_wait_acct_move()
1303 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) in mem_cgroup_print_oom_info() argument
1318 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_info()
1324 K((u64)page_counter_read(&memcg->memory)), in mem_cgroup_print_oom_info()
1325 K((u64)memcg->memory.max), memcg->memory.failcnt); in mem_cgroup_print_oom_info()
1327 K((u64)page_counter_read(&memcg->memsw)), in mem_cgroup_print_oom_info()
1328 K((u64)memcg->memsw.max), memcg->memsw.failcnt); in mem_cgroup_print_oom_info()
1330 K((u64)page_counter_read(&memcg->kmem)), in mem_cgroup_print_oom_info()
1331 K((u64)memcg->kmem.max), memcg->kmem.failcnt); in mem_cgroup_print_oom_info()
1333 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_print_oom_info()
1356 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) in mem_cgroup_get_max() argument
1360 max = memcg->memory.max; in mem_cgroup_get_max()
1361 if (mem_cgroup_swappiness(memcg)) { in mem_cgroup_get_max()
1365 memsw_max = memcg->memsw.max; in mem_cgroup_get_max()
1366 swap_max = memcg->swap.max; in mem_cgroup_get_max()
1373 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, in mem_cgroup_out_of_memory() argument
1379 .memcg = memcg, in mem_cgroup_out_of_memory()
1403 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, in test_mem_cgroup_node_reclaimable() argument
1406 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE)) in test_mem_cgroup_node_reclaimable()
1410 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON)) in test_mem_cgroup_node_reclaimable()
1422 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg) in mem_cgroup_may_update_nodemask() argument
1429 if (!atomic_read(&memcg->numainfo_events)) in mem_cgroup_may_update_nodemask()
1431 if (atomic_inc_return(&memcg->numainfo_updating) > 1) in mem_cgroup_may_update_nodemask()
1435 memcg->scan_nodes = node_states[N_MEMORY]; in mem_cgroup_may_update_nodemask()
1439 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false)) in mem_cgroup_may_update_nodemask()
1440 node_clear(nid, memcg->scan_nodes); in mem_cgroup_may_update_nodemask()
1443 atomic_set(&memcg->numainfo_events, 0); in mem_cgroup_may_update_nodemask()
1444 atomic_set(&memcg->numainfo_updating, 0); in mem_cgroup_may_update_nodemask()
1459 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) in mem_cgroup_select_victim_node() argument
1463 mem_cgroup_may_update_nodemask(memcg); in mem_cgroup_select_victim_node()
1464 node = memcg->last_scanned_node; in mem_cgroup_select_victim_node()
1466 node = next_node_in(node, memcg->scan_nodes); in mem_cgroup_select_victim_node()
1475 memcg->last_scanned_node = node; in mem_cgroup_select_victim_node()
1479 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) in mem_cgroup_select_victim_node() argument
1548 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) in mem_cgroup_oom_trylock() argument
1554 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_oom_trylock()
1561 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_oom_trylock()
1572 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_oom_trylock()
1574 mem_cgroup_iter_break(memcg, iter); in mem_cgroup_oom_trylock()
1587 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) in mem_cgroup_oom_unlock() argument
1593 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_oom_unlock()
1598 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) in mem_cgroup_mark_under_oom() argument
1603 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_mark_under_oom()
1608 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) in mem_cgroup_unmark_under_oom() argument
1617 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_unmark_under_oom()
1626 struct mem_cgroup *memcg; member
1638 oom_wait_memcg = oom_wait_info->memcg; in memcg_oom_wake_function()
1646 static void memcg_oom_recover(struct mem_cgroup *memcg) in memcg_oom_recover() argument
1656 if (memcg && memcg->under_oom) in memcg_oom_recover()
1657 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); in memcg_oom_recover()
1667 static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) in mem_cgroup_oom() argument
1690 if (memcg->oom_kill_disable) { in mem_cgroup_oom()
1693 css_get(&memcg->css); in mem_cgroup_oom()
1694 current->memcg_in_oom = memcg; in mem_cgroup_oom()
1701 if (mem_cgroup_out_of_memory(memcg, mask, order)) in mem_cgroup_oom()
1726 struct mem_cgroup *memcg = current->memcg_in_oom; in mem_cgroup_oom_synchronize() local
1731 if (!memcg) in mem_cgroup_oom_synchronize()
1737 owait.memcg = memcg; in mem_cgroup_oom_synchronize()
1744 mem_cgroup_mark_under_oom(memcg); in mem_cgroup_oom_synchronize()
1746 locked = mem_cgroup_oom_trylock(memcg); in mem_cgroup_oom_synchronize()
1749 mem_cgroup_oom_notify(memcg); in mem_cgroup_oom_synchronize()
1751 if (locked && !memcg->oom_kill_disable) { in mem_cgroup_oom_synchronize()
1752 mem_cgroup_unmark_under_oom(memcg); in mem_cgroup_oom_synchronize()
1754 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, in mem_cgroup_oom_synchronize()
1758 mem_cgroup_unmark_under_oom(memcg); in mem_cgroup_oom_synchronize()
1763 mem_cgroup_oom_unlock(memcg); in mem_cgroup_oom_synchronize()
1769 memcg_oom_recover(memcg); in mem_cgroup_oom_synchronize()
1773 css_put(&memcg->css); in mem_cgroup_oom_synchronize()
1791 struct mem_cgroup *memcg; in mem_cgroup_get_oom_group() local
1801 memcg = mem_cgroup_from_task(victim); in mem_cgroup_get_oom_group()
1802 if (memcg == root_mem_cgroup) in mem_cgroup_get_oom_group()
1810 for (; memcg; memcg = parent_mem_cgroup(memcg)) { in mem_cgroup_get_oom_group()
1811 if (memcg->oom_group) in mem_cgroup_get_oom_group()
1812 oom_group = memcg; in mem_cgroup_get_oom_group()
1814 if (memcg == oom_domain) in mem_cgroup_get_oom_group()
1826 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) in mem_cgroup_print_oom_group() argument
1829 pr_cont_cgroup_path(memcg->css.cgroup); in mem_cgroup_print_oom_group()
1846 struct mem_cgroup *memcg; in lock_page_memcg() local
1865 memcg = page->mem_cgroup; in lock_page_memcg()
1866 if (unlikely(!memcg)) in lock_page_memcg()
1869 if (atomic_read(&memcg->moving_account) <= 0) in lock_page_memcg()
1870 return memcg; in lock_page_memcg()
1872 spin_lock_irqsave(&memcg->move_lock, flags); in lock_page_memcg()
1873 if (memcg != page->mem_cgroup) { in lock_page_memcg()
1874 spin_unlock_irqrestore(&memcg->move_lock, flags); in lock_page_memcg()
1883 memcg->move_lock_task = current; in lock_page_memcg()
1884 memcg->move_lock_flags = flags; in lock_page_memcg()
1886 return memcg; in lock_page_memcg()
1896 void __unlock_page_memcg(struct mem_cgroup *memcg) in __unlock_page_memcg() argument
1898 if (memcg && memcg->move_lock_task == current) { in __unlock_page_memcg()
1899 unsigned long flags = memcg->move_lock_flags; in __unlock_page_memcg()
1901 memcg->move_lock_task = NULL; in __unlock_page_memcg()
1902 memcg->move_lock_flags = 0; in __unlock_page_memcg()
1904 spin_unlock_irqrestore(&memcg->move_lock, flags); in __unlock_page_memcg()
1941 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in consume_stock() argument
1953 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { in consume_stock()
2002 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) in refill_stock() argument
2010 if (stock->cached != memcg) { /* reset if necessary */ in refill_stock()
2012 stock->cached = memcg; in refill_stock()
2042 struct mem_cgroup *memcg; in drain_all_stock() local
2044 memcg = stock->cached; in drain_all_stock()
2045 if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css)) in drain_all_stock()
2047 if (!mem_cgroup_is_descendant(memcg, root_memcg)) { in drain_all_stock()
2048 css_put(&memcg->css); in drain_all_stock()
2057 css_put(&memcg->css); in drain_all_stock()
2066 struct mem_cgroup *memcg; in memcg_hotplug_cpu_dead() local
2071 for_each_mem_cgroup(memcg) { in memcg_hotplug_cpu_dead()
2078 x = this_cpu_xchg(memcg->stat_cpu->count[i], 0); in memcg_hotplug_cpu_dead()
2080 atomic_long_add(x, &memcg->stat[i]); in memcg_hotplug_cpu_dead()
2088 pn = mem_cgroup_nodeinfo(memcg, nid); in memcg_hotplug_cpu_dead()
2098 x = this_cpu_xchg(memcg->stat_cpu->events[i], 0); in memcg_hotplug_cpu_dead()
2100 atomic_long_add(x, &memcg->events[i]); in memcg_hotplug_cpu_dead()
2107 static void reclaim_high(struct mem_cgroup *memcg, in reclaim_high() argument
2112 if (page_counter_read(&memcg->memory) <= memcg->high) in reclaim_high()
2114 memcg_memory_event(memcg, MEMCG_HIGH); in reclaim_high()
2115 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true); in reclaim_high()
2116 } while ((memcg = parent_mem_cgroup(memcg))); in reclaim_high()
2121 struct mem_cgroup *memcg; in high_work_func() local
2123 memcg = container_of(work, struct mem_cgroup, high_work); in high_work_func()
2124 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); in high_work_func()
2134 struct mem_cgroup *memcg; in mem_cgroup_handle_over_high() local
2139 memcg = get_mem_cgroup_from_mm(current->mm); in mem_cgroup_handle_over_high()
2140 reclaim_high(memcg, nr_pages, GFP_KERNEL); in mem_cgroup_handle_over_high()
2141 css_put(&memcg->css); in mem_cgroup_handle_over_high()
2145 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, in try_charge() argument
2158 if (mem_cgroup_is_root(memcg)) in try_charge()
2161 if (consume_stock(memcg, nr_pages)) in try_charge()
2165 page_counter_try_charge(&memcg->memsw, batch, &counter)) { in try_charge()
2166 if (page_counter_try_charge(&memcg->memory, batch, &counter)) in try_charge()
2169 page_counter_uncharge(&memcg->memsw, batch); in try_charge()
2281 page_counter_charge(&memcg->memory, nr_pages); in try_charge()
2283 page_counter_charge(&memcg->memsw, nr_pages); in try_charge()
2284 css_get_many(&memcg->css, nr_pages); in try_charge()
2289 css_get_many(&memcg->css, batch); in try_charge()
2291 refill_stock(memcg, batch - nr_pages); in try_charge()
2303 if (page_counter_read(&memcg->memory) > memcg->high) { in try_charge()
2306 schedule_work(&memcg->high_work); in try_charge()
2313 } while ((memcg = parent_mem_cgroup(memcg))); in try_charge()
2318 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) in cancel_charge() argument
2320 if (mem_cgroup_is_root(memcg)) in cancel_charge()
2323 page_counter_uncharge(&memcg->memory, nr_pages); in cancel_charge()
2325 page_counter_uncharge(&memcg->memsw, nr_pages); in cancel_charge()
2327 css_put_many(&memcg->css, nr_pages); in cancel_charge()
2361 static void commit_charge(struct page *page, struct mem_cgroup *memcg, in commit_charge() argument
2389 page->mem_cgroup = memcg; in commit_charge()
2442 struct mem_cgroup *memcg; member
2451 struct mem_cgroup *memcg = cw->memcg; in memcg_kmem_cache_create_func() local
2454 memcg_create_kmem_cache(memcg, cachep); in memcg_kmem_cache_create_func()
2456 css_put(&memcg->css); in memcg_kmem_cache_create_func()
2463 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, in __memcg_schedule_kmem_cache_create() argument
2472 css_get(&memcg->css); in __memcg_schedule_kmem_cache_create()
2474 cw->memcg = memcg; in __memcg_schedule_kmem_cache_create()
2481 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, in memcg_schedule_kmem_cache_create() argument
2496 __memcg_schedule_kmem_cache_create(memcg, cachep); in memcg_schedule_kmem_cache_create()
2525 struct mem_cgroup *memcg; in memcg_kmem_get_cache() local
2537 memcg = get_mem_cgroup_from_current(); in memcg_kmem_get_cache()
2538 kmemcg_id = READ_ONCE(memcg->kmemcg_id); in memcg_kmem_get_cache()
2558 memcg_schedule_kmem_cache_create(memcg, cachep); in memcg_kmem_get_cache()
2560 css_put(&memcg->css); in memcg_kmem_get_cache()
2571 css_put(&cachep->memcg_params.memcg->css); in memcg_kmem_put_cache()
2584 struct mem_cgroup *memcg) in memcg_kmem_charge_memcg() argument
2590 ret = try_charge(memcg, gfp, nr_pages); in memcg_kmem_charge_memcg()
2595 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) { in memcg_kmem_charge_memcg()
2596 cancel_charge(memcg, nr_pages); in memcg_kmem_charge_memcg()
2600 page->mem_cgroup = memcg; in memcg_kmem_charge_memcg()
2615 struct mem_cgroup *memcg; in memcg_kmem_charge() local
2621 memcg = get_mem_cgroup_from_current(); in memcg_kmem_charge()
2622 if (!mem_cgroup_is_root(memcg)) { in memcg_kmem_charge()
2623 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg); in memcg_kmem_charge()
2627 css_put(&memcg->css); in memcg_kmem_charge()
2637 struct mem_cgroup *memcg = page->mem_cgroup; in memcg_kmem_uncharge() local
2640 if (!memcg) in memcg_kmem_uncharge()
2643 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page); in memcg_kmem_uncharge()
2646 page_counter_uncharge(&memcg->kmem, nr_pages); in memcg_kmem_uncharge()
2648 page_counter_uncharge(&memcg->memory, nr_pages); in memcg_kmem_uncharge()
2650 page_counter_uncharge(&memcg->memsw, nr_pages); in memcg_kmem_uncharge()
2658 css_put_many(&memcg->css, nr_pages); in memcg_kmem_uncharge()
2722 static int mem_cgroup_resize_max(struct mem_cgroup *memcg, in mem_cgroup_resize_max() argument
2729 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory; in mem_cgroup_resize_max()
2742 limits_invariant = memsw ? max >= memcg->memory.max : in mem_cgroup_resize_max()
2743 max <= memcg->memsw.max; in mem_cgroup_resize_max()
2758 drain_all_stock(memcg); in mem_cgroup_resize_max()
2763 if (!try_to_free_mem_cgroup_pages(memcg, 1, in mem_cgroup_resize_max()
2771 memcg_oom_recover(memcg); in mem_cgroup_resize_max()
2815 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, in mem_cgroup_soft_limit_reclaim()
2830 excess = soft_limit_excess(mz->memcg); in mem_cgroup_soft_limit_reclaim()
2842 css_put(&mz->memcg->css); in mem_cgroup_soft_limit_reclaim()
2855 css_put(&next_mz->memcg->css); in mem_cgroup_soft_limit_reclaim()
2865 static inline bool memcg_has_children(struct mem_cgroup *memcg) in memcg_has_children() argument
2870 ret = css_next_child(NULL, &memcg->css); in memcg_has_children()
2880 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) in mem_cgroup_force_empty() argument
2887 drain_all_stock(memcg); in mem_cgroup_force_empty()
2890 while (nr_retries && page_counter_read(&memcg->memory)) { in mem_cgroup_force_empty()
2896 progress = try_to_free_mem_cgroup_pages(memcg, 1, in mem_cgroup_force_empty()
2913 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_force_empty_write() local
2915 if (mem_cgroup_is_root(memcg)) in mem_cgroup_force_empty_write()
2917 return mem_cgroup_force_empty(memcg) ?: nbytes; in mem_cgroup_force_empty_write()
2930 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_hierarchy_write() local
2931 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent); in mem_cgroup_hierarchy_write()
2933 if (memcg->use_hierarchy == val) in mem_cgroup_hierarchy_write()
2946 if (!memcg_has_children(memcg)) in mem_cgroup_hierarchy_write()
2947 memcg->use_hierarchy = val; in mem_cgroup_hierarchy_write()
2966 static void accumulate_memcg_tree(struct mem_cgroup *memcg, in accumulate_memcg_tree() argument
2972 for_each_mem_cgroup_tree(mi, memcg) { in accumulate_memcg_tree()
2987 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) in mem_cgroup_usage() argument
2991 if (mem_cgroup_is_root(memcg)) { in mem_cgroup_usage()
2994 for_each_mem_cgroup_tree(iter, memcg) { in mem_cgroup_usage()
3002 val = page_counter_read(&memcg->memory); in mem_cgroup_usage()
3004 val = page_counter_read(&memcg->memsw); in mem_cgroup_usage()
3020 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_read_u64() local
3025 counter = &memcg->memory; in mem_cgroup_read_u64()
3028 counter = &memcg->memsw; in mem_cgroup_read_u64()
3031 counter = &memcg->kmem; in mem_cgroup_read_u64()
3034 counter = &memcg->tcpmem; in mem_cgroup_read_u64()
3042 if (counter == &memcg->memory) in mem_cgroup_read_u64()
3043 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; in mem_cgroup_read_u64()
3044 if (counter == &memcg->memsw) in mem_cgroup_read_u64()
3045 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; in mem_cgroup_read_u64()
3054 return (u64)memcg->soft_limit * PAGE_SIZE; in mem_cgroup_read_u64()
3061 static int memcg_online_kmem(struct mem_cgroup *memcg) in memcg_online_kmem() argument
3068 BUG_ON(memcg->kmemcg_id >= 0); in memcg_online_kmem()
3069 BUG_ON(memcg->kmem_state); in memcg_online_kmem()
3082 memcg->kmemcg_id = memcg_id; in memcg_online_kmem()
3083 memcg->kmem_state = KMEM_ONLINE; in memcg_online_kmem()
3084 INIT_LIST_HEAD(&memcg->kmem_caches); in memcg_online_kmem()
3089 static void memcg_offline_kmem(struct mem_cgroup *memcg) in memcg_offline_kmem() argument
3095 if (memcg->kmem_state != KMEM_ONLINE) in memcg_offline_kmem()
3103 memcg->kmem_state = KMEM_ALLOCATED; in memcg_offline_kmem()
3105 memcg_deactivate_kmem_caches(memcg); in memcg_offline_kmem()
3107 kmemcg_id = memcg->kmemcg_id; in memcg_offline_kmem()
3110 parent = parent_mem_cgroup(memcg); in memcg_offline_kmem()
3123 css_for_each_descendant_pre(css, &memcg->css) { in memcg_offline_kmem()
3127 if (!memcg->use_hierarchy) in memcg_offline_kmem()
3137 static void memcg_free_kmem(struct mem_cgroup *memcg) in memcg_free_kmem() argument
3140 if (unlikely(memcg->kmem_state == KMEM_ONLINE)) in memcg_free_kmem()
3141 memcg_offline_kmem(memcg); in memcg_free_kmem()
3143 if (memcg->kmem_state == KMEM_ALLOCATED) { in memcg_free_kmem()
3144 memcg_destroy_kmem_caches(memcg); in memcg_free_kmem()
3146 WARN_ON(page_counter_read(&memcg->kmem)); in memcg_free_kmem()
3150 static int memcg_online_kmem(struct mem_cgroup *memcg) in memcg_online_kmem() argument
3154 static void memcg_offline_kmem(struct mem_cgroup *memcg) in memcg_offline_kmem() argument
3157 static void memcg_free_kmem(struct mem_cgroup *memcg) in memcg_free_kmem() argument
3162 static int memcg_update_kmem_max(struct mem_cgroup *memcg, in memcg_update_kmem_max() argument
3168 ret = page_counter_set_max(&memcg->kmem, max); in memcg_update_kmem_max()
3173 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max) in memcg_update_tcp_max() argument
3179 ret = page_counter_set_max(&memcg->tcpmem, max); in memcg_update_tcp_max()
3183 if (!memcg->tcpmem_active) { in memcg_update_tcp_max()
3201 memcg->tcpmem_active = true; in memcg_update_tcp_max()
3215 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_write() local
3226 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ in mem_cgroup_write()
3232 ret = mem_cgroup_resize_max(memcg, nr_pages, false); in mem_cgroup_write()
3235 ret = mem_cgroup_resize_max(memcg, nr_pages, true); in mem_cgroup_write()
3238 ret = memcg_update_kmem_max(memcg, nr_pages); in mem_cgroup_write()
3241 ret = memcg_update_tcp_max(memcg, nr_pages); in mem_cgroup_write()
3246 memcg->soft_limit = nr_pages; in mem_cgroup_write()
3256 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in mem_cgroup_reset() local
3261 counter = &memcg->memory; in mem_cgroup_reset()
3264 counter = &memcg->memsw; in mem_cgroup_reset()
3267 counter = &memcg->kmem; in mem_cgroup_reset()
3270 counter = &memcg->tcpmem; in mem_cgroup_reset()
3300 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_move_charge_write() local
3311 memcg->move_charge_at_immigrate = val; in mem_cgroup_move_charge_write()
3339 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memcg_numa_stat_show() local
3342 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask); in memcg_numa_stat_show()
3345 nr = mem_cgroup_node_nr_lru_pages(memcg, nid, in memcg_numa_stat_show()
3356 for_each_mem_cgroup_tree(iter, memcg) in memcg_numa_stat_show()
3361 for_each_mem_cgroup_tree(iter, memcg) in memcg_numa_stat_show()
3390 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memcg_stat_show() local
3403 memcg_page_state(memcg, memcg1_stats[i]) * in memcg_stat_show()
3409 memcg_sum_events(memcg, memcg1_events[i])); in memcg_stat_show()
3413 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE); in memcg_stat_show()
3417 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { in memcg_stat_show()
3432 accumulate_memcg_tree(memcg, &acc); in memcg_stat_show()
3458 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id); in memcg_stat_show()
3479 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_swappiness_read() local
3481 return mem_cgroup_swappiness(memcg); in mem_cgroup_swappiness_read()
3487 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_swappiness_write() local
3493 memcg->swappiness = val; in mem_cgroup_swappiness_write()
3500 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) in __mem_cgroup_threshold() argument
3508 t = rcu_dereference(memcg->thresholds.primary); in __mem_cgroup_threshold()
3510 t = rcu_dereference(memcg->memsw_thresholds.primary); in __mem_cgroup_threshold()
3515 usage = mem_cgroup_usage(memcg, swap); in __mem_cgroup_threshold()
3551 static void mem_cgroup_threshold(struct mem_cgroup *memcg) in mem_cgroup_threshold() argument
3553 while (memcg) { in mem_cgroup_threshold()
3554 __mem_cgroup_threshold(memcg, false); in mem_cgroup_threshold()
3556 __mem_cgroup_threshold(memcg, true); in mem_cgroup_threshold()
3558 memcg = parent_mem_cgroup(memcg); in mem_cgroup_threshold()
3576 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) in mem_cgroup_oom_notify_cb() argument
3582 list_for_each_entry(ev, &memcg->oom_notify, list) in mem_cgroup_oom_notify_cb()
3589 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) in mem_cgroup_oom_notify() argument
3593 for_each_mem_cgroup_tree(iter, memcg) in mem_cgroup_oom_notify()
3597 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, in __mem_cgroup_usage_register_event() argument
3610 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
3613 thresholds = &memcg->thresholds; in __mem_cgroup_usage_register_event()
3614 usage = mem_cgroup_usage(memcg, false); in __mem_cgroup_usage_register_event()
3616 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_register_event()
3617 usage = mem_cgroup_usage(memcg, true); in __mem_cgroup_usage_register_event()
3623 __mem_cgroup_threshold(memcg, type == _MEMSWAP); in __mem_cgroup_usage_register_event()
3674 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
3679 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, in mem_cgroup_usage_register_event() argument
3682 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); in mem_cgroup_usage_register_event()
3685 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, in memsw_cgroup_usage_register_event() argument
3688 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); in memsw_cgroup_usage_register_event()
3691 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in __mem_cgroup_usage_unregister_event() argument
3699 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
3702 thresholds = &memcg->thresholds; in __mem_cgroup_usage_unregister_event()
3703 usage = mem_cgroup_usage(memcg, false); in __mem_cgroup_usage_unregister_event()
3705 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_unregister_event()
3706 usage = mem_cgroup_usage(memcg, true); in __mem_cgroup_usage_unregister_event()
3714 __mem_cgroup_threshold(memcg, type == _MEMSWAP); in __mem_cgroup_usage_unregister_event()
3767 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
3770 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in mem_cgroup_usage_unregister_event() argument
3773 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); in mem_cgroup_usage_unregister_event()
3776 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, in memsw_cgroup_usage_unregister_event() argument
3779 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); in memsw_cgroup_usage_unregister_event()
3782 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, in mem_cgroup_oom_register_event() argument
3794 list_add(&event->list, &memcg->oom_notify); in mem_cgroup_oom_register_event()
3797 if (memcg->under_oom) in mem_cgroup_oom_register_event()
3804 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, in mem_cgroup_oom_unregister_event() argument
3811 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { in mem_cgroup_oom_unregister_event()
3823 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); in mem_cgroup_oom_control_read() local
3825 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); in mem_cgroup_oom_control_read()
3826 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); in mem_cgroup_oom_control_read()
3828 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); in mem_cgroup_oom_control_read()
3835 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_oom_control_write() local
3841 memcg->oom_kill_disable = val; in mem_cgroup_oom_control_write()
3843 memcg_oom_recover(memcg); in mem_cgroup_oom_control_write()
3850 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) in memcg_wb_domain_init() argument
3852 return wb_domain_init(&memcg->cgwb_domain, gfp); in memcg_wb_domain_init()
3855 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) in memcg_wb_domain_exit() argument
3857 wb_domain_exit(&memcg->cgwb_domain); in memcg_wb_domain_exit()
3860 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) in memcg_wb_domain_size_changed() argument
3862 wb_domain_size_changed(&memcg->cgwb_domain); in memcg_wb_domain_size_changed()
3867 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_wb_domain() local
3869 if (!memcg->css.parent) in mem_cgroup_wb_domain()
3872 return &memcg->cgwb_domain; in mem_cgroup_wb_domain()
3897 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); in mem_cgroup_wb_stats() local
3900 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); in mem_cgroup_wb_stats()
3903 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); in mem_cgroup_wb_stats()
3904 *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) | in mem_cgroup_wb_stats()
3908 while ((parent = parent_mem_cgroup(memcg))) { in mem_cgroup_wb_stats()
3909 unsigned long ceiling = min(memcg->memory.max, memcg->high); in mem_cgroup_wb_stats()
3910 unsigned long used = page_counter_read(&memcg->memory); in mem_cgroup_wb_stats()
3913 memcg = parent; in mem_cgroup_wb_stats()
3919 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) in memcg_wb_domain_init() argument
3924 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) in memcg_wb_domain_exit() argument
3928 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) in memcg_wb_domain_size_changed() argument
3956 struct mem_cgroup *memcg = event->memcg; in memcg_event_remove() local
3960 event->unregister_event(memcg, event->eventfd); in memcg_event_remove()
3967 css_put(&memcg->css); in memcg_event_remove()
3980 struct mem_cgroup *memcg = event->memcg; in memcg_event_wake() local
3993 spin_lock(&memcg->event_list_lock); in memcg_event_wake()
4002 spin_unlock(&memcg->event_list_lock); in memcg_event_wake()
4030 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memcg_write_event_control() local
4056 event->memcg = memcg; in memcg_write_event_control()
4128 ret = event->register_event(memcg, event->eventfd, buf); in memcg_write_event_control()
4134 spin_lock(&memcg->event_list_lock); in memcg_write_event_control()
4135 list_add(&event->list, &memcg->event_list); in memcg_write_event_control()
4136 spin_unlock(&memcg->event_list_lock); in memcg_write_event_control()
4314 static void mem_cgroup_id_remove(struct mem_cgroup *memcg) in mem_cgroup_id_remove() argument
4316 if (memcg->id.id > 0) { in mem_cgroup_id_remove()
4317 idr_remove(&mem_cgroup_idr, memcg->id.id); in mem_cgroup_id_remove()
4318 memcg->id.id = 0; in mem_cgroup_id_remove()
4322 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n) in mem_cgroup_id_get_many() argument
4324 VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0); in mem_cgroup_id_get_many()
4325 atomic_add(n, &memcg->id.ref); in mem_cgroup_id_get_many()
4328 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) in mem_cgroup_id_put_many() argument
4330 VM_BUG_ON(atomic_read(&memcg->id.ref) < n); in mem_cgroup_id_put_many()
4331 if (atomic_sub_and_test(n, &memcg->id.ref)) { in mem_cgroup_id_put_many()
4332 mem_cgroup_id_remove(memcg); in mem_cgroup_id_put_many()
4335 css_put(&memcg->css); in mem_cgroup_id_put_many()
4339 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg) in mem_cgroup_id_get() argument
4341 mem_cgroup_id_get_many(memcg, 1); in mem_cgroup_id_get()
4344 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) in mem_cgroup_id_put() argument
4346 mem_cgroup_id_put_many(memcg, 1); in mem_cgroup_id_put()
4361 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) in alloc_mem_cgroup_per_node_info() argument
4388 pn->memcg = memcg; in alloc_mem_cgroup_per_node_info()
4390 memcg->nodeinfo[node] = pn; in alloc_mem_cgroup_per_node_info()
4394 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) in free_mem_cgroup_per_node_info() argument
4396 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; in free_mem_cgroup_per_node_info()
4405 static void __mem_cgroup_free(struct mem_cgroup *memcg) in __mem_cgroup_free() argument
4410 free_mem_cgroup_per_node_info(memcg, node); in __mem_cgroup_free()
4411 free_percpu(memcg->stat_cpu); in __mem_cgroup_free()
4412 kfree(memcg); in __mem_cgroup_free()
4415 static void mem_cgroup_free(struct mem_cgroup *memcg) in mem_cgroup_free() argument
4417 memcg_wb_domain_exit(memcg); in mem_cgroup_free()
4418 __mem_cgroup_free(memcg); in mem_cgroup_free()
4423 struct mem_cgroup *memcg; in mem_cgroup_alloc() local
4430 memcg = kzalloc(size, GFP_KERNEL); in mem_cgroup_alloc()
4431 if (!memcg) in mem_cgroup_alloc()
4434 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, in mem_cgroup_alloc()
4437 if (memcg->id.id < 0) in mem_cgroup_alloc()
4440 memcg->stat_cpu = alloc_percpu(struct mem_cgroup_stat_cpu); in mem_cgroup_alloc()
4441 if (!memcg->stat_cpu) in mem_cgroup_alloc()
4445 if (alloc_mem_cgroup_per_node_info(memcg, node)) in mem_cgroup_alloc()
4448 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) in mem_cgroup_alloc()
4451 INIT_WORK(&memcg->high_work, high_work_func); in mem_cgroup_alloc()
4452 memcg->last_scanned_node = MAX_NUMNODES; in mem_cgroup_alloc()
4453 INIT_LIST_HEAD(&memcg->oom_notify); in mem_cgroup_alloc()
4454 mutex_init(&memcg->thresholds_lock); in mem_cgroup_alloc()
4455 spin_lock_init(&memcg->move_lock); in mem_cgroup_alloc()
4456 vmpressure_init(&memcg->vmpressure); in mem_cgroup_alloc()
4457 INIT_LIST_HEAD(&memcg->event_list); in mem_cgroup_alloc()
4458 spin_lock_init(&memcg->event_list_lock); in mem_cgroup_alloc()
4459 memcg->socket_pressure = jiffies; in mem_cgroup_alloc()
4461 memcg->kmemcg_id = -1; in mem_cgroup_alloc()
4464 INIT_LIST_HEAD(&memcg->cgwb_list); in mem_cgroup_alloc()
4466 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); in mem_cgroup_alloc()
4467 return memcg; in mem_cgroup_alloc()
4469 mem_cgroup_id_remove(memcg); in mem_cgroup_alloc()
4470 __mem_cgroup_free(memcg); in mem_cgroup_alloc()
4478 struct mem_cgroup *memcg; in mem_cgroup_css_alloc() local
4481 memcg = mem_cgroup_alloc(); in mem_cgroup_css_alloc()
4482 if (!memcg) in mem_cgroup_css_alloc()
4485 memcg->high = PAGE_COUNTER_MAX; in mem_cgroup_css_alloc()
4486 memcg->soft_limit = PAGE_COUNTER_MAX; in mem_cgroup_css_alloc()
4488 memcg->swappiness = mem_cgroup_swappiness(parent); in mem_cgroup_css_alloc()
4489 memcg->oom_kill_disable = parent->oom_kill_disable; in mem_cgroup_css_alloc()
4492 memcg->use_hierarchy = true; in mem_cgroup_css_alloc()
4493 page_counter_init(&memcg->memory, &parent->memory); in mem_cgroup_css_alloc()
4494 page_counter_init(&memcg->swap, &parent->swap); in mem_cgroup_css_alloc()
4495 page_counter_init(&memcg->memsw, &parent->memsw); in mem_cgroup_css_alloc()
4496 page_counter_init(&memcg->kmem, &parent->kmem); in mem_cgroup_css_alloc()
4497 page_counter_init(&memcg->tcpmem, &parent->tcpmem); in mem_cgroup_css_alloc()
4499 page_counter_init(&memcg->memory, NULL); in mem_cgroup_css_alloc()
4500 page_counter_init(&memcg->swap, NULL); in mem_cgroup_css_alloc()
4501 page_counter_init(&memcg->memsw, NULL); in mem_cgroup_css_alloc()
4502 page_counter_init(&memcg->kmem, NULL); in mem_cgroup_css_alloc()
4503 page_counter_init(&memcg->tcpmem, NULL); in mem_cgroup_css_alloc()
4515 root_mem_cgroup = memcg; in mem_cgroup_css_alloc()
4516 return &memcg->css; in mem_cgroup_css_alloc()
4519 error = memcg_online_kmem(memcg); in mem_cgroup_css_alloc()
4526 return &memcg->css; in mem_cgroup_css_alloc()
4528 mem_cgroup_id_remove(memcg); in mem_cgroup_css_alloc()
4529 mem_cgroup_free(memcg); in mem_cgroup_css_alloc()
4535 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_online() local
4542 if (memcg_alloc_shrinker_maps(memcg)) { in mem_cgroup_css_online()
4543 mem_cgroup_id_remove(memcg); in mem_cgroup_css_online()
4548 atomic_set(&memcg->id.ref, 1); in mem_cgroup_css_online()
4555 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_offline() local
4563 spin_lock(&memcg->event_list_lock); in mem_cgroup_css_offline()
4564 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { in mem_cgroup_css_offline()
4568 spin_unlock(&memcg->event_list_lock); in mem_cgroup_css_offline()
4570 page_counter_set_min(&memcg->memory, 0); in mem_cgroup_css_offline()
4571 page_counter_set_low(&memcg->memory, 0); in mem_cgroup_css_offline()
4573 memcg_offline_kmem(memcg); in mem_cgroup_css_offline()
4574 wb_memcg_offline(memcg); in mem_cgroup_css_offline()
4576 mem_cgroup_id_put(memcg); in mem_cgroup_css_offline()
4581 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_released() local
4583 invalidate_reclaim_iterators(memcg); in mem_cgroup_css_released()
4588 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_free() local
4593 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) in mem_cgroup_css_free()
4596 vmpressure_cleanup(&memcg->vmpressure); in mem_cgroup_css_free()
4597 cancel_work_sync(&memcg->high_work); in mem_cgroup_css_free()
4598 mem_cgroup_remove_from_trees(memcg); in mem_cgroup_css_free()
4599 memcg_free_shrinker_maps(memcg); in mem_cgroup_css_free()
4600 memcg_free_kmem(memcg); in mem_cgroup_css_free()
4601 mem_cgroup_free(memcg); in mem_cgroup_css_free()
4619 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in mem_cgroup_css_reset() local
4621 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
4622 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
4623 page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
4624 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
4625 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); in mem_cgroup_css_reset()
4626 page_counter_set_min(&memcg->memory, 0); in mem_cgroup_css_reset()
4627 page_counter_set_low(&memcg->memory, 0); in mem_cgroup_css_reset()
4628 memcg->high = PAGE_COUNTER_MAX; in mem_cgroup_css_reset()
4629 memcg->soft_limit = PAGE_COUNTER_MAX; in mem_cgroup_css_reset()
4630 memcg_wb_domain_size_changed(memcg); in mem_cgroup_css_reset()
5095 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ in mem_cgroup_can_attach() local
5116 memcg = mem_cgroup_from_css(css); in mem_cgroup_can_attach()
5126 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); in mem_cgroup_can_attach()
5132 VM_BUG_ON(from == memcg); in mem_cgroup_can_attach()
5148 mc.to = memcg; in mem_cgroup_can_attach()
5359 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in memory_current_read() local
5361 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; in memory_current_read()
5366 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memory_min_show() local
5367 unsigned long min = READ_ONCE(memcg->memory.min); in memory_min_show()
5380 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_min_write() local
5389 page_counter_set_min(&memcg->memory, min); in memory_min_write()
5396 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memory_low_show() local
5397 unsigned long low = READ_ONCE(memcg->memory.low); in memory_low_show()
5410 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_low_write() local
5419 page_counter_set_low(&memcg->memory, low); in memory_low_write()
5426 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memory_high_show() local
5427 unsigned long high = READ_ONCE(memcg->high); in memory_high_show()
5440 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_high_write() local
5450 memcg->high = high; in memory_high_write()
5452 nr_pages = page_counter_read(&memcg->memory); in memory_high_write()
5454 try_to_free_mem_cgroup_pages(memcg, nr_pages - high, in memory_high_write()
5457 memcg_wb_domain_size_changed(memcg); in memory_high_write()
5463 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memory_max_show() local
5464 unsigned long max = READ_ONCE(memcg->memory.max); in memory_max_show()
5477 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_max_write() local
5488 xchg(&memcg->memory.max, max); in memory_max_write()
5491 unsigned long nr_pages = page_counter_read(&memcg->memory); in memory_max_write()
5502 drain_all_stock(memcg); in memory_max_write()
5508 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, in memory_max_write()
5514 memcg_memory_event(memcg, MEMCG_OOM); in memory_max_write()
5515 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) in memory_max_write()
5519 memcg_wb_domain_size_changed(memcg); in memory_max_write()
5525 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memory_events_show() local
5528 atomic_long_read(&memcg->memory_events[MEMCG_LOW])); in memory_events_show()
5530 atomic_long_read(&memcg->memory_events[MEMCG_HIGH])); in memory_events_show()
5532 atomic_long_read(&memcg->memory_events[MEMCG_MAX])); in memory_events_show()
5534 atomic_long_read(&memcg->memory_events[MEMCG_OOM])); in memory_events_show()
5536 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); in memory_events_show()
5543 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memory_stat_show() local
5561 accumulate_memcg_tree(memcg, &acc); in memory_stat_show()
5620 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in memory_oom_group_show() local
5622 seq_printf(m, "%d\n", memcg->oom_group); in memory_oom_group_show()
5630 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in memory_oom_group_write() local
5644 memcg->oom_group = oom_group; in memory_oom_group_write()
5786 struct mem_cgroup *memcg) in mem_cgroup_protected() argument
5798 if (memcg == root) in mem_cgroup_protected()
5801 usage = page_counter_read(&memcg->memory); in mem_cgroup_protected()
5805 emin = memcg->memory.min; in mem_cgroup_protected()
5806 elow = memcg->memory.low; in mem_cgroup_protected()
5808 parent = parent_mem_cgroup(memcg); in mem_cgroup_protected()
5821 min_usage = min(usage, memcg->memory.min); in mem_cgroup_protected()
5835 low_usage = min(usage, memcg->memory.low); in mem_cgroup_protected()
5845 memcg->memory.emin = emin; in mem_cgroup_protected()
5846 memcg->memory.elow = elow; in mem_cgroup_protected()
5878 struct mem_cgroup *memcg = NULL; in mem_cgroup_try_charge() local
5902 memcg = mem_cgroup_from_id(id); in mem_cgroup_try_charge()
5903 if (memcg && !css_tryget_online(&memcg->css)) in mem_cgroup_try_charge()
5904 memcg = NULL; in mem_cgroup_try_charge()
5909 if (!memcg) in mem_cgroup_try_charge()
5910 memcg = get_mem_cgroup_from_mm(mm); in mem_cgroup_try_charge()
5912 ret = try_charge(memcg, gfp_mask, nr_pages); in mem_cgroup_try_charge()
5914 css_put(&memcg->css); in mem_cgroup_try_charge()
5916 *memcgp = memcg; in mem_cgroup_try_charge()
5924 struct mem_cgroup *memcg; in mem_cgroup_try_charge_delay() local
5928 memcg = *memcgp; in mem_cgroup_try_charge_delay()
5929 mem_cgroup_throttle_swaprate(memcg, page_to_nid(page), gfp_mask); in mem_cgroup_try_charge_delay()
5950 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, in mem_cgroup_commit_charge() argument
5965 if (!memcg) in mem_cgroup_commit_charge()
5968 commit_charge(page, memcg, lrucare); in mem_cgroup_commit_charge()
5971 mem_cgroup_charge_statistics(memcg, page, compound, nr_pages); in mem_cgroup_commit_charge()
5972 memcg_check_events(memcg, page); in mem_cgroup_commit_charge()
5994 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg, in mem_cgroup_cancel_charge() argument
6006 if (!memcg) in mem_cgroup_cancel_charge()
6009 cancel_charge(memcg, nr_pages); in mem_cgroup_cancel_charge()
6013 struct mem_cgroup *memcg; member
6033 if (!mem_cgroup_is_root(ug->memcg)) { in uncharge_batch()
6034 page_counter_uncharge(&ug->memcg->memory, nr_pages); in uncharge_batch()
6036 page_counter_uncharge(&ug->memcg->memsw, nr_pages); in uncharge_batch()
6038 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem); in uncharge_batch()
6039 memcg_oom_recover(ug->memcg); in uncharge_batch()
6043 __mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon); in uncharge_batch()
6044 __mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file); in uncharge_batch()
6045 __mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge); in uncharge_batch()
6046 __mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem); in uncharge_batch()
6047 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); in uncharge_batch()
6048 __this_cpu_add(ug->memcg->stat_cpu->nr_page_events, nr_pages); in uncharge_batch()
6049 memcg_check_events(ug->memcg, ug->dummy_page); in uncharge_batch()
6052 if (!mem_cgroup_is_root(ug->memcg)) in uncharge_batch()
6053 css_put_many(&ug->memcg->css, nr_pages); in uncharge_batch()
6071 if (ug->memcg != page->mem_cgroup) { in uncharge_page()
6072 if (ug->memcg) { in uncharge_page()
6076 ug->memcg = page->mem_cgroup; in uncharge_page()
6124 if (ug.memcg) in uncharge_list()
6179 struct mem_cgroup *memcg; in mem_cgroup_migrate() local
6198 memcg = oldpage->mem_cgroup; in mem_cgroup_migrate()
6199 if (!memcg) in mem_cgroup_migrate()
6206 page_counter_charge(&memcg->memory, nr_pages); in mem_cgroup_migrate()
6208 page_counter_charge(&memcg->memsw, nr_pages); in mem_cgroup_migrate()
6209 css_get_many(&memcg->css, nr_pages); in mem_cgroup_migrate()
6211 commit_charge(newpage, memcg, false); in mem_cgroup_migrate()
6214 mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages); in mem_cgroup_migrate()
6215 memcg_check_events(memcg, newpage); in mem_cgroup_migrate()
6224 struct mem_cgroup *memcg; in mem_cgroup_sk_alloc() local
6244 memcg = mem_cgroup_from_task(current); in mem_cgroup_sk_alloc()
6245 if (memcg == root_mem_cgroup) in mem_cgroup_sk_alloc()
6247 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) in mem_cgroup_sk_alloc()
6249 if (css_tryget_online(&memcg->css)) in mem_cgroup_sk_alloc()
6250 sk->sk_memcg = memcg; in mem_cgroup_sk_alloc()
6269 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) in mem_cgroup_charge_skmem() argument
6276 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { in mem_cgroup_charge_skmem()
6277 memcg->tcpmem_pressure = 0; in mem_cgroup_charge_skmem()
6280 page_counter_charge(&memcg->tcpmem, nr_pages); in mem_cgroup_charge_skmem()
6281 memcg->tcpmem_pressure = 1; in mem_cgroup_charge_skmem()
6289 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages); in mem_cgroup_charge_skmem()
6291 if (try_charge(memcg, gfp_mask, nr_pages) == 0) in mem_cgroup_charge_skmem()
6294 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages); in mem_cgroup_charge_skmem()
6303 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) in mem_cgroup_uncharge_skmem() argument
6306 page_counter_uncharge(&memcg->tcpmem, nr_pages); in mem_cgroup_uncharge_skmem()
6310 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); in mem_cgroup_uncharge_skmem()
6312 refill_stock(memcg, nr_pages); in mem_cgroup_uncharge_skmem()
6378 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) in mem_cgroup_id_get_online() argument
6380 while (!atomic_inc_not_zero(&memcg->id.ref)) { in mem_cgroup_id_get_online()
6385 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) { in mem_cgroup_id_get_online()
6389 memcg = parent_mem_cgroup(memcg); in mem_cgroup_id_get_online()
6390 if (!memcg) in mem_cgroup_id_get_online()
6391 memcg = root_mem_cgroup; in mem_cgroup_id_get_online()
6393 return memcg; in mem_cgroup_id_get_online()
6405 struct mem_cgroup *memcg, *swap_memcg; in mem_cgroup_swapout() local
6415 memcg = page->mem_cgroup; in mem_cgroup_swapout()
6418 if (!memcg) in mem_cgroup_swapout()
6426 swap_memcg = mem_cgroup_id_get_online(memcg); in mem_cgroup_swapout()
6438 if (!mem_cgroup_is_root(memcg)) in mem_cgroup_swapout()
6439 page_counter_uncharge(&memcg->memory, nr_entries); in mem_cgroup_swapout()
6441 if (memcg != swap_memcg) { in mem_cgroup_swapout()
6444 page_counter_uncharge(&memcg->memsw, nr_entries); in mem_cgroup_swapout()
6454 mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page), in mem_cgroup_swapout()
6456 memcg_check_events(memcg, page); in mem_cgroup_swapout()
6458 if (!mem_cgroup_is_root(memcg)) in mem_cgroup_swapout()
6459 css_put_many(&memcg->css, nr_entries); in mem_cgroup_swapout()
6475 struct mem_cgroup *memcg; in mem_cgroup_try_charge_swap() local
6481 memcg = page->mem_cgroup; in mem_cgroup_try_charge_swap()
6484 if (!memcg) in mem_cgroup_try_charge_swap()
6488 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); in mem_cgroup_try_charge_swap()
6492 memcg = mem_cgroup_id_get_online(memcg); in mem_cgroup_try_charge_swap()
6494 if (!mem_cgroup_is_root(memcg) && in mem_cgroup_try_charge_swap()
6495 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { in mem_cgroup_try_charge_swap()
6496 memcg_memory_event(memcg, MEMCG_SWAP_MAX); in mem_cgroup_try_charge_swap()
6497 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); in mem_cgroup_try_charge_swap()
6498 mem_cgroup_id_put(memcg); in mem_cgroup_try_charge_swap()
6504 mem_cgroup_id_get_many(memcg, nr_pages - 1); in mem_cgroup_try_charge_swap()
6505 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); in mem_cgroup_try_charge_swap()
6507 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages); in mem_cgroup_try_charge_swap()
6519 struct mem_cgroup *memcg; in mem_cgroup_uncharge_swap() local
6527 memcg = mem_cgroup_from_id(id); in mem_cgroup_uncharge_swap()
6528 if (memcg) { in mem_cgroup_uncharge_swap()
6529 if (!mem_cgroup_is_root(memcg)) { in mem_cgroup_uncharge_swap()
6531 page_counter_uncharge(&memcg->swap, nr_pages); in mem_cgroup_uncharge_swap()
6533 page_counter_uncharge(&memcg->memsw, nr_pages); in mem_cgroup_uncharge_swap()
6535 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); in mem_cgroup_uncharge_swap()
6536 mem_cgroup_id_put_many(memcg, nr_pages); in mem_cgroup_uncharge_swap()
6541 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) in mem_cgroup_get_nr_swap_pages() argument
6547 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) in mem_cgroup_get_nr_swap_pages()
6549 READ_ONCE(memcg->swap.max) - in mem_cgroup_get_nr_swap_pages()
6550 page_counter_read(&memcg->swap)); in mem_cgroup_get_nr_swap_pages()
6556 struct mem_cgroup *memcg; in mem_cgroup_swap_full() local
6565 memcg = page->mem_cgroup; in mem_cgroup_swap_full()
6566 if (!memcg) in mem_cgroup_swap_full()
6569 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) in mem_cgroup_swap_full()
6570 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.max) in mem_cgroup_swap_full()
6596 struct mem_cgroup *memcg = mem_cgroup_from_css(css); in swap_current_read() local
6598 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; in swap_current_read()
6603 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in swap_max_show() local
6604 unsigned long max = READ_ONCE(memcg->swap.max); in swap_max_show()
6617 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); in swap_max_write() local
6626 xchg(&memcg->swap.max, max); in swap_max_write()
6633 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); in swap_events_show() local
6636 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX])); in swap_events_show()
6638 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL])); in swap_events_show()