1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* memcontrol.c - Memory Controller
19  * Charge lifetime sanitation
35 #include <linux/page-flags.h>
36 #include <linux/backing-dev.h>
106  * Cgroups above their limits are maintained in a RB-Tree, independent of
198  * limit reclaim to prevent infinite loops, if they ever occur.
203 /* for encoding cft->private value on file */
235 	return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||  in should_force_charge()
236 		(current->flags & PF_EXITING);  in should_force_charge()
244 	return &memcg->vmpressure;  in memcg_to_vmpressure()
249 	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;  in vmpressure_to_css()
265 	 * objcg->nr_charged_bytes can't have an arbitrary byte value.  in obj_cgroup_release()
269 	 * 1) CPU0: objcg == stock->cached_objcg  in obj_cgroup_release()
274 	 *          objcg->nr_charged_bytes = PAGE_SIZE - 92  in obj_cgroup_release()
276 	 *          92 bytes are added to stock->nr_bytes  in obj_cgroup_release()
278 	 *          92 bytes are added to objcg->nr_charged_bytes  in obj_cgroup_release()
283 	nr_bytes = atomic_read(&objcg->nr_charged_bytes);  in obj_cgroup_release()
284 	WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));  in obj_cgroup_release()
291 	list_del(&objcg->list);  in obj_cgroup_release()
308 	ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,  in obj_cgroup_alloc()
314 	INIT_LIST_HEAD(&objcg->list);  in obj_cgroup_alloc()
323 	objcg = rcu_replace_pointer(memcg->objcg, NULL, true);  in memcg_reparent_objcgs()
328 	xchg(&objcg->memcg, parent);  in memcg_reparent_objcgs()
329 	css_get(&parent->css);  in memcg_reparent_objcgs()
330 	list_add(&objcg->list, &parent->objcg_list);  in memcg_reparent_objcgs()
333 	list_for_each_entry(iter, &memcg->objcg_list, list) {  in memcg_reparent_objcgs()
334 		css_get(&parent->css);  in memcg_reparent_objcgs()
335 		xchg(&iter->memcg, parent);  in memcg_reparent_objcgs()
336 		css_put(&memcg->css);  in memcg_reparent_objcgs()
338 	list_splice(&memcg->objcg_list, &parent->objcg_list);  in memcg_reparent_objcgs()
342 	percpu_ref_kill(&objcg->refcnt);  in memcg_reparent_objcgs()
349  *  but only a few kmem-limited. Or also, if we have, for instance, 200
350  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
353  * The current size of the caches array is stored in memcg_nr_cache_ids. It
374  * the alloc/free process all the time. In a small machine, 4 kmem-limited
415 			mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);  in memcg_expand_one_shrinker_map()
422 			return -ENOMEM;  in memcg_expand_one_shrinker_map()
425 		memset(new->map, (int)0xff, old_size);  in memcg_expand_one_shrinker_map()
426 		memset((void *)new->map + old_size, 0, size - old_size);  in memcg_expand_one_shrinker_map()
428 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);  in memcg_expand_one_shrinker_map()
429 		call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);  in memcg_expand_one_shrinker_map()
446 		map = rcu_dereference_protected(pn->shrinker_map, true);  in memcg_free_shrinker_maps()
449 		rcu_assign_pointer(pn->shrinker_map, NULL);  in memcg_free_shrinker_maps()
467 			ret = -ENOMEM;  in memcg_alloc_shrinker_maps()
470 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);  in memcg_alloc_shrinker_maps()
513 		map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);  in memcg_set_shrinker_bit()
516 		set_bit(shrinker_id, map->map);  in memcg_set_shrinker_bit()
522  * mem_cgroup_css_from_page - css of the memcg associated with a page
536 	memcg = page->mem_cgroup;  in mem_cgroup_css_from_page()
541 	return &memcg->css;  in mem_cgroup_css_from_page()
545  * page_cgroup_ino - return inode number of the memcg a page is charged to
563 	memcg = page->mem_cgroup;  in page_cgroup_ino()
574 	while (memcg && !(memcg->css.flags & CSS_ONLINE))  in page_cgroup_ino()
577 		ino = cgroup_ino(memcg->css.cgroup);  in page_cgroup_ino()
587 	return memcg->nodeinfo[nid];  in mem_cgroup_page_nodeinfo()
608 	struct rb_node **p = &mctz->rb_root.rb_node;  in __mem_cgroup_insert_exceeded()
613 	if (mz->on_tree)  in __mem_cgroup_insert_exceeded()
616 	mz->usage_in_excess = new_usage_in_excess;  in __mem_cgroup_insert_exceeded()
617 	if (!mz->usage_in_excess)  in __mem_cgroup_insert_exceeded()
623 		if (mz->usage_in_excess < mz_node->usage_in_excess) {  in __mem_cgroup_insert_exceeded()
624 			p = &(*p)->rb_left;  in __mem_cgroup_insert_exceeded()
630 		 * limit by the same amount  in __mem_cgroup_insert_exceeded()
632 		else if (mz->usage_in_excess >= mz_node->usage_in_excess)  in __mem_cgroup_insert_exceeded()
633 			p = &(*p)->rb_right;  in __mem_cgroup_insert_exceeded()
637 		mctz->rb_rightmost = &mz->tree_node;  in __mem_cgroup_insert_exceeded()
639 	rb_link_node(&mz->tree_node, parent, p);  in __mem_cgroup_insert_exceeded()
640 	rb_insert_color(&mz->tree_node, &mctz->rb_root);  in __mem_cgroup_insert_exceeded()
641 	mz->on_tree = true;  in __mem_cgroup_insert_exceeded()
647 	if (!mz->on_tree)  in __mem_cgroup_remove_exceeded()
650 	if (&mz->tree_node == mctz->rb_rightmost)  in __mem_cgroup_remove_exceeded()
651 		mctz->rb_rightmost = rb_prev(&mz->tree_node);  in __mem_cgroup_remove_exceeded()
653 	rb_erase(&mz->tree_node, &mctz->rb_root);  in __mem_cgroup_remove_exceeded()
654 	mz->on_tree = false;  in __mem_cgroup_remove_exceeded()
662 	spin_lock_irqsave(&mctz->lock, flags);  in mem_cgroup_remove_exceeded()
664 	spin_unlock_irqrestore(&mctz->lock, flags);  in mem_cgroup_remove_exceeded()
669 	unsigned long nr_pages = page_counter_read(&memcg->memory);  in soft_limit_excess()
670 	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);  in soft_limit_excess()
674 		excess = nr_pages - soft_limit;  in soft_limit_excess()
696 		 * We have to update the tree if mz is on RB-tree or  in mem_cgroup_update_tree()
699 		if (excess || mz->on_tree) {  in mem_cgroup_update_tree()
702 			spin_lock_irqsave(&mctz->lock, flags);  in mem_cgroup_update_tree()
703 			/* if on-tree, remove it */  in mem_cgroup_update_tree()
704 			if (mz->on_tree)  in mem_cgroup_update_tree()
707 			 * Insert again. mz->usage_in_excess will be updated.  in mem_cgroup_update_tree()
711 			spin_unlock_irqrestore(&mctz->lock, flags);  in mem_cgroup_update_tree()
737 	if (!mctz->rb_rightmost)  in __mem_cgroup_largest_soft_limit_node()
740 	mz = rb_entry(mctz->rb_rightmost,  in __mem_cgroup_largest_soft_limit_node()
748 	if (!soft_limit_excess(mz->memcg) ||  in __mem_cgroup_largest_soft_limit_node()
749 	    !css_tryget(&mz->memcg->css))  in __mem_cgroup_largest_soft_limit_node()
760 	spin_lock_irq(&mctz->lock);  in mem_cgroup_largest_soft_limit_node()
762 	spin_unlock_irq(&mctz->lock);  in mem_cgroup_largest_soft_limit_node()
767  * __mod_memcg_state - update cgroup memory statistics
769  * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
782 	x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);  in __mod_memcg_state()
790 		__this_cpu_add(memcg->vmstats_local->stat[idx], x);  in __mod_memcg_state()
792 			atomic_long_add(x, &mi->vmstats[idx]);  in __mod_memcg_state()
795 	__this_cpu_write(memcg->vmstats_percpu->stat[idx], x);  in __mod_memcg_state()
803 	parent = parent_mem_cgroup(pn->memcg);  in parent_nodeinfo()
817 	memcg = pn->memcg;  in __mod_memcg_lruvec_state()
823 	__this_cpu_add(pn->lruvec_stat_local->count[idx], val);  in __mod_memcg_lruvec_state()
828 	x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);  in __mod_memcg_lruvec_state()
833 		for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))  in __mod_memcg_lruvec_state()
834 			atomic_long_add(x, &pi->lruvec_stat[idx]);  in __mod_memcg_lruvec_state()
837 	__this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);  in __mod_memcg_lruvec_state()
841  * __mod_lruvec_state - update lruvec memory statistics
848  * change of state at this level: per-node, per-cgroup, per-lruvec.
873 	 * when we free the slab object, we need to update the per-memcg  in __mod_lruvec_slab_state()
897  * __count_memcg_events - account VM events in a cgroup
910 	x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);  in __count_memcg_events()
918 		__this_cpu_add(memcg->vmstats_local->events[idx], x);  in __count_memcg_events()
920 			atomic_long_add(x, &mi->vmevents[idx]);  in __count_memcg_events()
923 	__this_cpu_write(memcg->vmstats_percpu->events[idx], x);  in __count_memcg_events()
928 	return atomic_long_read(&memcg->vmevents[event]);  in memcg_events()
937 		x += per_cpu(memcg->vmstats_local->events[event], cpu);  in memcg_events_local()
950 		nr_pages = -nr_pages; /* for event */  in mem_cgroup_charge_statistics()
953 	__this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);  in mem_cgroup_charge_statistics()
961 	val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);  in mem_cgroup_event_ratelimit()
962 	next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);  in mem_cgroup_event_ratelimit()
964 	if ((long)(next - val) < 0) {  in mem_cgroup_event_ratelimit()
975 		__this_cpu_write(memcg->vmstats_percpu->targets[target], next);  in mem_cgroup_event_ratelimit()
987 	/* threshold event is triggered in finer grain than soft limit */  in memcg_check_events()
1003 	 * mm_update_next_owner() may clear mm->owner to NULL  in mem_cgroup_from_task()
1018  * Obtain a reference on mm->memcg and returns it if successful. Otherwise
1039 			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));  in get_mem_cgroup_from_mm()
1043 	} while (!css_tryget(&memcg->css));  in get_mem_cgroup_from_mm()
1053  * Obtain a reference on page->memcg and returns it if successful. Otherwise
1058 	struct mem_cgroup *memcg = page->mem_cgroup;  in get_mem_cgroup_from_page()
1065 	if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css)))  in get_mem_cgroup_from_page()
1077 		return current->active_memcg;  in active_memcg()
1087 		/* current->active_memcg must hold a ref. */  in get_active_memcg()
1088 		if (WARN_ON_ONCE(!css_tryget(&memcg->css)))  in get_active_memcg()
1091 			memcg = current->active_memcg;  in get_active_memcg()
1104 	/* Memcg to charge can't be determined. */  in memcg_kmem_bypass()
1105 	if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))  in memcg_kmem_bypass()
1112  * If active memcg is set, do not fallback to current->mm->memcg.
1122 	return get_mem_cgroup_from_mm(current->mm);  in get_mem_cgroup_from_current()
1126  * mem_cgroup_iter - iterate over memory cgroup hierarchy
1132  * @root itself, or %NULL after a full round-trip.
1136  * to cancel a hierarchy walk before the round-trip is complete.
1160 	if (!root->use_hierarchy && root != root_mem_cgroup) {  in mem_cgroup_iter()
1171 		mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);  in mem_cgroup_iter()
1172 		iter = &mz->iter;  in mem_cgroup_iter()
1174 		if (prev && reclaim->generation != iter->generation)  in mem_cgroup_iter()
1178 			pos = READ_ONCE(iter->position);  in mem_cgroup_iter()
1179 			if (!pos || css_tryget(&pos->css))  in mem_cgroup_iter()
1182 			 * css reference reached zero, so iter->position will  in mem_cgroup_iter()
1183 			 * be cleared by ->css_released. However, we should not  in mem_cgroup_iter()
1184 			 * rely on this happening soon, because ->css_released  in mem_cgroup_iter()
1185 			 * is called from a work queue, and by busy-waiting we  in mem_cgroup_iter()
1186 			 * might block it. So we clear iter->position right  in mem_cgroup_iter()
1189 			(void)cmpxchg(&iter->position, pos, NULL);  in mem_cgroup_iter()
1194 		css = &pos->css;  in mem_cgroup_iter()
1197 		css = css_next_descendant_pre(css, &root->css);  in mem_cgroup_iter()
1202 			 * the hierarchy - make sure they see at least  in mem_cgroup_iter()
1217 		if (css == &root->css)  in mem_cgroup_iter()
1232 		(void)cmpxchg(&iter->position, pos, memcg);  in mem_cgroup_iter()
1235 			css_put(&pos->css);  in mem_cgroup_iter()
1238 			iter->generation++;  in mem_cgroup_iter()
1240 			reclaim->generation = iter->generation;  in mem_cgroup_iter()
1247 		css_put(&prev->css);  in mem_cgroup_iter()
1253  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1263 		css_put(&prev->css);  in mem_cgroup_iter_break()
1275 		iter = &mz->iter;  in __invalidate_reclaim_iterators()
1276 		cmpxchg(&iter->position, dead_memcg, NULL);  in __invalidate_reclaim_iterators()
1291 	 * When cgruop1 non-hierarchy mode is used,  in invalidate_reclaim_iterators()
1302  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1308  * descendants and calls @fn for each task. If @fn returns a non-zero
1326 		css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);  in mem_cgroup_scan_tasks()
1339  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1343  * This function relies on page->mem_cgroup being stable - see the
1353 		lruvec = &pgdat->__lruvec;  in mem_cgroup_page_lruvec()
1357 	memcg = page->mem_cgroup;  in mem_cgroup_page_lruvec()
1359 	 * Swapcache readahead pages are added to the LRU - and  in mem_cgroup_page_lruvec()
1360 	 * possibly migrated - before they are charged.  in mem_cgroup_page_lruvec()
1366 	lruvec = &mz->lruvec;  in mem_cgroup_page_lruvec()
1370 	 * we have to be prepared to initialize lruvec->zone here;  in mem_cgroup_page_lruvec()
1373 	if (unlikely(lruvec->pgdat != pgdat))  in mem_cgroup_page_lruvec()
1374 		lruvec->pgdat = pgdat;  in mem_cgroup_page_lruvec()
1379  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1400 	lru_size = &mz->lru_zone_size[zid][lru];  in mem_cgroup_update_lru_size()
1418  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1428 	unsigned long limit;  in mem_cgroup_margin()  local
1430 	count = page_counter_read(&memcg->memory);  in mem_cgroup_margin()
1431 	limit = READ_ONCE(memcg->memory.max);  in mem_cgroup_margin()
1432 	if (count < limit)  in mem_cgroup_margin()
1433 		margin = limit - count;  in mem_cgroup_margin()
1436 		count = page_counter_read(&memcg->memsw);  in mem_cgroup_margin()
1437 		limit = READ_ONCE(memcg->memsw.max);  in mem_cgroup_margin()
1438 		if (count < limit)  in mem_cgroup_margin()
1439 			margin = min(margin, limit - count);  in mem_cgroup_margin()
1451  * moving cgroups. This is for waiting at high-memory pressure
1478 	if (mc.moving_task && current != mc.moving_task) {  in mem_cgroup_wait_acct_move()
1482 			/* moving charge context might have finished. */  in mem_cgroup_wait_acct_move()
1570 	 * 1) generic big picture -> specifics and details  in memory_stat_format()
1571 	 * 2) reflecting userspace activity -> reflecting kernel heuristics  in memory_stat_format()
1573 	 * Current memory state:  in memory_stat_format()
1626 #define K(x) ((x) << (PAGE_SHIFT-10))
1630  * @memcg: The memory cgroup that went over limit
1642 		pr_cont_cgroup_path(memcg->css.cgroup);  in mem_cgroup_print_oom_context()
1655  * @memcg: The memory cgroup that went over limit
1661 	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",  in mem_cgroup_print_oom_meminfo()
1662 		K((u64)page_counter_read(&memcg->memory)),  in mem_cgroup_print_oom_meminfo()
1663 		K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);  in mem_cgroup_print_oom_meminfo()
1665 		pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",  in mem_cgroup_print_oom_meminfo()
1666 			K((u64)page_counter_read(&memcg->swap)),  in mem_cgroup_print_oom_meminfo()
1667 			K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);  in mem_cgroup_print_oom_meminfo()
1669 		pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",  in mem_cgroup_print_oom_meminfo()
1670 			K((u64)page_counter_read(&memcg->memsw)),  in mem_cgroup_print_oom_meminfo()
1671 			K((u64)memcg->memsw.max), memcg->memsw.failcnt);  in mem_cgroup_print_oom_meminfo()
1672 		pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",  in mem_cgroup_print_oom_meminfo()
1673 			K((u64)page_counter_read(&memcg->kmem)),  in mem_cgroup_print_oom_meminfo()
1674 			K((u64)memcg->kmem.max), memcg->kmem.failcnt);  in mem_cgroup_print_oom_meminfo()
1678 	pr_cont_cgroup_path(memcg->css.cgroup);  in mem_cgroup_print_oom_meminfo()
1688  * Return the memory (and swap, if configured) limit for a memcg.
1692 	unsigned long max = READ_ONCE(memcg->memory.max);  in mem_cgroup_get_max()
1696 			max += min(READ_ONCE(memcg->swap.max),  in mem_cgroup_get_max()
1700 			/* Calculate swap excess capacity from memsw limit */  in mem_cgroup_get_max()
1701 			unsigned long swap = READ_ONCE(memcg->memsw.max) - max;  in mem_cgroup_get_max()
1711 	return page_counter_read(&memcg->memory);  in mem_cgroup_size()
1802  * Check OOM-Killer is already running under our hierarchy.
1812 		if (iter->oom_lock) {  in mem_cgroup_oom_trylock()
1821 			iter->oom_lock = true;  in mem_cgroup_oom_trylock()
1834 			iter->oom_lock = false;  in mem_cgroup_oom_trylock()
1851 		iter->oom_lock = false;  in mem_cgroup_oom_unlock()
1861 		iter->under_oom++;  in mem_cgroup_mark_under_oom()
1875 		if (iter->under_oom > 0)  in mem_cgroup_unmark_under_oom()
1876 			iter->under_oom--;  in mem_cgroup_unmark_under_oom()
1895 	oom_wait_memcg = oom_wait_info->memcg;  in memcg_oom_wake_function()
1906 	 * For the following lockless ->under_oom test, the only required  in memcg_oom_recover()
1913 	if (memcg && memcg->under_oom)  in memcg_oom_recover()
1935 	 * We are in the middle of the charge context here, so we  in mem_cgroup_oom()
1940 	 * handling until the charge can succeed; remember the context and put  in mem_cgroup_oom()
1944 	 * On the other hand, in-kernel OOM killer allows for an async victim  in mem_cgroup_oom()
1950 	 * victim and then we have to bail out from the charge path.  in mem_cgroup_oom()
1952 	if (memcg->oom_kill_disable) {  in mem_cgroup_oom()
1953 		if (!current->in_user_fault)  in mem_cgroup_oom()
1955 		css_get(&memcg->css);  in mem_cgroup_oom()
1956 		current->memcg_in_oom = memcg;  in mem_cgroup_oom()
1957 		current->memcg_oom_gfp_mask = mask;  in mem_cgroup_oom()
1958 		current->memcg_oom_order = order;  in mem_cgroup_oom()
1983  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1991  * situation.  Sleeping directly in the charge context with all kinds
2001 	struct mem_cgroup *memcg = current->memcg_in_oom;  in mem_cgroup_oom_synchronize()
2015 	owait.wait.private = current;  in mem_cgroup_oom_synchronize()
2026 	if (locked && !memcg->oom_kill_disable) {  in mem_cgroup_oom_synchronize()
2029 		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,  in mem_cgroup_oom_synchronize()
2030 					 current->memcg_oom_order);  in mem_cgroup_oom_synchronize()
2040 		 * There is no guarantee that an OOM-lock contender  in mem_cgroup_oom_synchronize()
2047 	current->memcg_in_oom = NULL;  in mem_cgroup_oom_synchronize()
2048 	css_put(&memcg->css);  in mem_cgroup_oom_synchronize()
2053  * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
2055  * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2058  * by killing all belonging OOM-killable tasks.
2060  * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2091 	 * highest-level memory cgroup with oom.group set.  in mem_cgroup_get_oom_group()
2094 		if (memcg->oom_group)  in mem_cgroup_get_oom_group()
2102 		css_get(&oom_group->css);  in mem_cgroup_get_oom_group()
2112 	pr_cont_cgroup_path(memcg->css.cgroup);  in mem_cgroup_print_oom_group()
2117  * lock_page_memcg - lock a page->mem_cgroup binding
2135 	 * path can get away without acquiring the memcg->move_lock  in lock_page_memcg()
2149 	memcg = head->mem_cgroup;  in lock_page_memcg()
2153 	if (atomic_read(&memcg->moving_account) <= 0)  in lock_page_memcg()
2156 	spin_lock_irqsave(&memcg->move_lock, flags);  in lock_page_memcg()
2157 	if (memcg != head->mem_cgroup) {  in lock_page_memcg()
2158 		spin_unlock_irqrestore(&memcg->move_lock, flags);  in lock_page_memcg()
2163 	 * When charge migration first begins, we can have locked and  in lock_page_memcg()
2167 	memcg->move_lock_task = current;  in lock_page_memcg()
2168 	memcg->move_lock_flags = flags;  in lock_page_memcg()
2175  * __unlock_page_memcg - unlock and unpin a memcg
2182 	if (memcg && memcg->move_lock_task == current) {  in __unlock_page_memcg()
2183 		unsigned long flags = memcg->move_lock_flags;  in __unlock_page_memcg()
2185 		memcg->move_lock_task = NULL;  in __unlock_page_memcg()
2186 		memcg->move_lock_flags = 0;  in __unlock_page_memcg()
2188 		spin_unlock_irqrestore(&memcg->move_lock, flags);  in __unlock_page_memcg()
2195  * unlock_page_memcg - unlock a page->mem_cgroup binding
2202 	__unlock_page_memcg(head->mem_cgroup);  in unlock_page_memcg()
2239  * consume_stock: Try to consume stocked charge on this cpu.
2241  * @nr_pages: how many pages to charge.
2243  * The charges will only happen if @memcg matches the current cpu's memcg
2261 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {  in consume_stock()
2262 		stock->nr_pages -= nr_pages;  in consume_stock()
2276 	struct mem_cgroup *old = stock->cached;  in drain_stock()
2281 	if (stock->nr_pages) {  in drain_stock()
2282 		page_counter_uncharge(&old->memory, stock->nr_pages);  in drain_stock()
2284 			page_counter_uncharge(&old->memsw, stock->nr_pages);  in drain_stock()
2285 		stock->nr_pages = 0;  in drain_stock()
2288 	css_put(&old->css);  in drain_stock()
2289 	stock->cached = NULL;  in drain_stock()
2306 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);  in drain_local_stock()
2323 	if (stock->cached != memcg) { /* reset if necessary */  in refill_stock()
2325 		css_get(&memcg->css);  in refill_stock()
2326 		stock->cached = memcg;  in refill_stock()
2328 	stock->nr_pages += nr_pages;  in refill_stock()
2330 	if (stock->nr_pages > MEMCG_CHARGE_BATCH)  in refill_stock()
2337  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2348 	 * Notify other cpus that system-wide "drain" is running  in drain_all_stock()
2351 	 * per-cpu data. CPU up doesn't touch memcg_stock at all.  in drain_all_stock()
2360 		memcg = stock->cached;  in drain_all_stock()
2361 		if (memcg && stock->nr_pages &&  in drain_all_stock()
2369 		    !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {  in drain_all_stock()
2371 				drain_local_stock(&stock->work);  in drain_all_stock()
2373 				schedule_work_on(cpu, &stock->work);  in drain_all_stock()
2395 			x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);  in memcg_hotplug_cpu_dead()
2398 					atomic_long_add(x, &memcg->vmstats[i]);  in memcg_hotplug_cpu_dead()
2407 				x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);  in memcg_hotplug_cpu_dead()
2410 						atomic_long_add(x, &pn->lruvec_stat[i]);  in memcg_hotplug_cpu_dead()
2418 			x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);  in memcg_hotplug_cpu_dead()
2421 					atomic_long_add(x, &memcg->vmevents[i]);  in memcg_hotplug_cpu_dead()
2437 		if (page_counter_read(&memcg->memory) <=  in reclaim_high()
2438 		    READ_ONCE(memcg->memory.high))  in reclaim_high()
2473  * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
2475  * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
2480  * reasonable delay curve compared to precision-adjusted overage, not
2482  * limit penalises misbehaviour cgroups by slowing them down exponentially. For
2485  *  +-------+------------------------+
2487  *  +-------+------------------------+
2509  *  +-------+------------------------+
2527 	overage = usage - high;  in calculate_overage()
2537 		overage = calculate_overage(page_counter_read(&memcg->memory),  in mem_find_max_overage()
2538 					    READ_ONCE(memcg->memory.high));  in mem_find_max_overage()
2551 		overage = calculate_overage(page_counter_read(&memcg->swap),  in swap_find_max_overage()
2552 					    READ_ONCE(memcg->swap.high));  in swap_find_max_overage()
2589 	 * N-sized allocations are throttled approximately the same as one  in calculate_high_delay()
2590 	 * 4N-sized allocation.  in calculate_high_delay()
2593 	 * larger the current charge patch is than that.  in calculate_high_delay()
2600  * and reclaims memory over the high limit.
2607 	unsigned int nr_pages = current->memcg_nr_pages_over_high;  in mem_cgroup_handle_over_high()
2615 	memcg = get_mem_cgroup_from_mm(current->mm);  in mem_cgroup_handle_over_high()
2616 	current->memcg_nr_pages_over_high = 0;  in mem_cgroup_handle_over_high()
2663 	if (nr_reclaimed || nr_retries--) {  in mem_cgroup_handle_over_high()
2671 	 * need to account for any ill-begotten jiffies to pay them off later.  in mem_cgroup_handle_over_high()
2678 	css_put(&memcg->css);  in mem_cgroup_handle_over_high()
2701 	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {  in try_charge()
2702 		if (page_counter_try_charge(&memcg->memory, batch, &counter))  in try_charge()
2705 			page_counter_uncharge(&memcg->memsw, batch);  in try_charge()
2728 	 * memory shortage.  Allow dying and OOM-killed tasks to  in try_charge()
2739 	 * under the limit over triggering OOM kills in these cases.  in try_charge()
2741 	if (unlikely(current->flags & PF_MEMALLOC))  in try_charge()
2744 	if (unlikely(task_in_memcg_oom(current)))  in try_charge()
2769 	 * Even though the limit is exceeded at this point, reclaim  in try_charge()
2770 	 * may have been able to free some pages.  Retry the charge  in try_charge()
2774 	 * unlikely to succeed so close to the limit, and we fall back  in try_charge()
2780 	 * At task move, charge accounts can be doubly counted. So, it's  in try_charge()
2786 	if (nr_retries--)  in try_charge()
2795 	if (fatal_signal_pending(current))  in try_charge()
2800 	 * a forward progress or bypass the charge if the oom killer  in try_charge()
2816 		return -ENOMEM;  in try_charge()
2820 	 * being freed very soon.  Allow memory usage go over the limit  in try_charge()
2823 	page_counter_charge(&memcg->memory, nr_pages);  in try_charge()
2825 		page_counter_charge(&memcg->memsw, nr_pages);  in try_charge()
2831 		refill_stock(memcg, batch - nr_pages);  in try_charge()
2838 	 * not recorded as it most likely matches current's and won't  in try_charge()
2839 	 * change in the meantime.  As high limit is checked again before  in try_charge()
2845 		mem_high = page_counter_read(&memcg->memory) >  in try_charge()
2846 			READ_ONCE(memcg->memory.high);  in try_charge()
2847 		swap_high = page_counter_read(&memcg->swap) >  in try_charge()
2848 			READ_ONCE(memcg->swap.high);  in try_charge()
2853 				schedule_work(&memcg->high_work);  in try_charge()
2865 			 * Target some best-effort fairness between the tasks,  in try_charge()
2869 			current->memcg_nr_pages_over_high += batch;  in try_charge()
2870 			set_notify_resume(current);  in try_charge()
2884 	page_counter_uncharge(&memcg->memory, nr_pages);  in cancel_charge()
2886 		page_counter_uncharge(&memcg->memsw, nr_pages);  in cancel_charge()
2892 	VM_BUG_ON_PAGE(page->mem_cgroup, page);  in commit_charge()
2894 	 * Any of the following ensures page->mem_cgroup stability:  in commit_charge()
2896 	 * - the page lock  in commit_charge()
2897 	 * - LRU isolation  in commit_charge()
2898 	 * - lock_page_memcg()  in commit_charge()
2899 	 * - exclusive reference  in commit_charge()
2901 	page->mem_cgroup = memcg;  in commit_charge()
2914 		return -ENOMEM;  in memcg_alloc_page_obj_cgroups()
2916 	if (cmpxchg(&page->obj_cgroups, NULL,  in memcg_alloc_page_obj_cgroups()
2941 	 * If page->mem_cgroup is set, it's either a simple mem_cgroup pointer  in mem_cgroup_from_obj()
2944 	 * The page->mem_cgroup pointer can be asynchronously changed  in mem_cgroup_from_obj()
2948 	if (!page->mem_cgroup)  in mem_cgroup_from_obj()
2952 	 * Slab objects are accounted individually, not per-page.  in mem_cgroup_from_obj()
2954 	 * the page->obj_cgroups.  in mem_cgroup_from_obj()
2960 		off = obj_to_index(page->slab_cache, page, p);  in mem_cgroup_from_obj()
2968 	/* All other pages use page->mem_cgroup */  in mem_cgroup_from_obj()
2969 	return page->mem_cgroup;  in mem_cgroup_from_obj()
2984 		memcg = mem_cgroup_from_task(current);  in get_obj_cgroup_from_current()
2987 		objcg = rcu_dereference(memcg->objcg);  in get_obj_cgroup_from_current()
3040  * __memcg_kmem_charge: charge a number of kernel pages to a memcg
3041  * @memcg: memory cgroup to charge
3043  * @nr_pages: number of pages to charge
3058 	    !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {  in __memcg_kmem_charge()
3066 			page_counter_charge(&memcg->kmem, nr_pages);  in __memcg_kmem_charge()
3070 		return -ENOMEM;  in __memcg_kmem_charge()
3083 		page_counter_uncharge(&memcg->kmem, nr_pages);  in __memcg_kmem_uncharge()
3085 	page_counter_uncharge(&memcg->memory, nr_pages);  in __memcg_kmem_uncharge()
3087 		page_counter_uncharge(&memcg->memsw, nr_pages);  in __memcg_kmem_uncharge()
3091  * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
3092  * @page: page to charge
3107 			page->mem_cgroup = memcg;  in __memcg_kmem_charge_page()
3111 		css_put(&memcg->css);  in __memcg_kmem_charge_page()
3123 	struct mem_cgroup *memcg = page->mem_cgroup;  in __memcg_kmem_uncharge_page()
3131 	page->mem_cgroup = NULL;  in __memcg_kmem_uncharge_page()
3132 	css_put(&memcg->css);  in __memcg_kmem_uncharge_page()
3148 	if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {  in consume_obj_stock()
3149 		stock->nr_bytes -= nr_bytes;  in consume_obj_stock()
3160 	struct obj_cgroup *old = stock->cached_objcg;  in drain_obj_stock()
3165 	if (stock->nr_bytes) {  in drain_obj_stock()
3166 		unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;  in drain_obj_stock()
3167 		unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);  in drain_obj_stock()
3176 		 * The leftover is flushed to the centralized per-memcg value.  in drain_obj_stock()
3178 		 * to a per-cpu stock (probably, on an other CPU), see  in drain_obj_stock()
3181 		 * How often it's flushed is a trade-off between the memory  in drain_obj_stock()
3182 		 * limit enforcement accuracy and potential CPU contention,  in drain_obj_stock()
3185 		atomic_add(nr_bytes, &old->nr_charged_bytes);  in drain_obj_stock()
3186 		stock->nr_bytes = 0;  in drain_obj_stock()
3190 	stock->cached_objcg = NULL;  in drain_obj_stock()
3198 	if (stock->cached_objcg) {  in obj_stock_flush_required()
3199 		memcg = obj_cgroup_memcg(stock->cached_objcg);  in obj_stock_flush_required()
3215 	if (stock->cached_objcg != objcg) { /* reset if necessary */  in refill_obj_stock()
3218 		stock->cached_objcg = objcg;  in refill_obj_stock()
3219 		stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);  in refill_obj_stock()
3221 	stock->nr_bytes += nr_bytes;  in refill_obj_stock()
3223 	if (stock->nr_bytes > PAGE_SIZE)  in refill_obj_stock()
3239 	 * In theory, memcg->nr_charged_bytes can have enough  in obj_cgroup_charge()
3240 	 * pre-charged bytes to satisfy the allocation. However,  in obj_cgroup_charge()
3241 	 * flushing memcg->nr_charged_bytes requires two atomic  in obj_cgroup_charge()
3242 	 * operations, and memcg->nr_charged_bytes can't be big,  in obj_cgroup_charge()
3244 	 * memcg->nr_charged_bytes will be flushed in  in obj_cgroup_charge()
3250 	css_get(&memcg->css);  in obj_cgroup_charge()
3254 	nr_bytes = size & (PAGE_SIZE - 1);  in obj_cgroup_charge()
3261 		refill_obj_stock(objcg, PAGE_SIZE - nr_bytes);  in obj_cgroup_charge()
3263 	css_put(&memcg->css);  in obj_cgroup_charge()
3278  * pgdat->lru_lock and migration entries setup in all page mappings.
3282 	struct mem_cgroup *memcg = head->mem_cgroup;  in mem_cgroup_split_huge_fixup()
3289 		css_get(&memcg->css);  in mem_cgroup_split_huge_fixup()
3297  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
3305  * Returns 0 on success, -EINVAL on failure.
3319 		mod_memcg_state(from, MEMCG_SWAP, -1);  in mem_cgroup_move_swap_account()
3323 	return -EINVAL;  in mem_cgroup_move_swap_account()
3329 	return -EINVAL;  in mem_cgroup_move_swap_account()
3342 	struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;  in mem_cgroup_resize_max()
3345 		if (signal_pending(current)) {  in mem_cgroup_resize_max()
3346 			ret = -EINTR;  in mem_cgroup_resize_max()
3352 		 * Make sure that the new limit (memsw or memory limit) doesn't  in mem_cgroup_resize_max()
3355 		limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :  in mem_cgroup_resize_max()
3356 					   max <= memcg->memsw.max;  in mem_cgroup_resize_max()
3359 			ret = -EINVAL;  in mem_cgroup_resize_max()
3362 		if (max > counter->max)  in mem_cgroup_resize_max()
3378 			ret = -EBUSY;  in mem_cgroup_resize_max()
3404 	mctz = soft_limit_tree_node(pgdat->node_id);  in mem_cgroup_soft_limit_reclaim()
3409 	 * are acceptable as soft limit is best effort anyway.  in mem_cgroup_soft_limit_reclaim()
3411 	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))  in mem_cgroup_soft_limit_reclaim()
3416 	 * keep exceeding their soft limit and putting the system under  in mem_cgroup_soft_limit_reclaim()
3428 		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,  in mem_cgroup_soft_limit_reclaim()
3432 		spin_lock_irq(&mctz->lock);  in mem_cgroup_soft_limit_reclaim()
3443 		excess = soft_limit_excess(mz->memcg);  in mem_cgroup_soft_limit_reclaim()
3454 		spin_unlock_irq(&mctz->lock);  in mem_cgroup_soft_limit_reclaim()
3455 		css_put(&mz->memcg->css);  in mem_cgroup_soft_limit_reclaim()
3468 		css_put(&next_mz->memcg->css);  in mem_cgroup_soft_limit_reclaim()
3483 	ret = css_next_child(NULL, &memcg->css);  in memcg_has_children()
3497 	/* we call try-to-free pages for make this cgroup empty */  in mem_cgroup_force_empty()
3503 	while (nr_retries && page_counter_read(&memcg->memory)) {  in mem_cgroup_force_empty()
3506 		if (signal_pending(current))  in mem_cgroup_force_empty()
3507 			return -EINTR;  in mem_cgroup_force_empty()
3512 			nr_retries--;  in mem_cgroup_force_empty()
3529 		return -EINVAL;  in mem_cgroup_force_empty_write()
3536 	return mem_cgroup_from_css(css)->use_hierarchy;  in mem_cgroup_hierarchy_read()
3544 	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);  in mem_cgroup_hierarchy_write()
3546 	if (memcg->use_hierarchy == val)  in mem_cgroup_hierarchy_write()
3552 	 * occur, provided the current cgroup has no children.  in mem_cgroup_hierarchy_write()
3557 	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&  in mem_cgroup_hierarchy_write()
3560 			memcg->use_hierarchy = val;  in mem_cgroup_hierarchy_write()
3562 			retval = -EBUSY;  in mem_cgroup_hierarchy_write()
3564 		retval = -EINVAL;  in mem_cgroup_hierarchy_write()
3580 			val = page_counter_read(&memcg->memory);  in mem_cgroup_usage()
3582 			val = page_counter_read(&memcg->memsw);  in mem_cgroup_usage()
3601 	switch (MEMFILE_TYPE(cft->private)) {  in mem_cgroup_read_u64()
3603 		counter = &memcg->memory;  in mem_cgroup_read_u64()
3606 		counter = &memcg->memsw;  in mem_cgroup_read_u64()
3609 		counter = &memcg->kmem;  in mem_cgroup_read_u64()
3612 		counter = &memcg->tcpmem;  in mem_cgroup_read_u64()
3618 	switch (MEMFILE_ATTR(cft->private)) {  in mem_cgroup_read_u64()
3620 		if (counter == &memcg->memory)  in mem_cgroup_read_u64()
3622 		if (counter == &memcg->memsw)  in mem_cgroup_read_u64()
3626 		return (u64)counter->max * PAGE_SIZE;  in mem_cgroup_read_u64()
3628 		return (u64)counter->watermark * PAGE_SIZE;  in mem_cgroup_read_u64()
3630 		return counter->failcnt;  in mem_cgroup_read_u64()
3632 		return (u64)memcg->soft_limit * PAGE_SIZE;  in mem_cgroup_read_u64()
3646 			stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);  in memcg_flush_percpu_vmstats()
3650 			atomic_long_add(stat[i], &mi->vmstats[i]);  in memcg_flush_percpu_vmstats()
3653 		struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];  in memcg_flush_percpu_vmstats()
3662 					pn->lruvec_stat_cpu->count[i], cpu);  in memcg_flush_percpu_vmstats()
3666 				atomic_long_add(stat[i], &pi->lruvec_stat[i]);  in memcg_flush_percpu_vmstats()
3681 			events[i] += per_cpu(memcg->vmstats_percpu->events[i],  in memcg_flush_percpu_vmevents()
3686 			atomic_long_add(events[i], &mi->vmevents[i]);  in memcg_flush_percpu_vmevents()
3698 	BUG_ON(memcg->kmemcg_id >= 0);  in memcg_online_kmem()
3699 	BUG_ON(memcg->kmem_state);  in memcg_online_kmem()
3708 		return -ENOMEM;  in memcg_online_kmem()
3710 	objcg->memcg = memcg;  in memcg_online_kmem()
3711 	rcu_assign_pointer(memcg->objcg, objcg);  in memcg_online_kmem()
3716 	 * A memory cgroup is considered kmem-online as soon as it gets  in memcg_online_kmem()
3721 	memcg->kmemcg_id = memcg_id;  in memcg_online_kmem()
3722 	memcg->kmem_state = KMEM_ONLINE;  in memcg_online_kmem()
3733 	if (memcg->kmem_state != KMEM_ONLINE)  in memcg_offline_kmem()
3736 	memcg->kmem_state = KMEM_ALLOCATED;  in memcg_offline_kmem()
3744 	kmemcg_id = memcg->kmemcg_id;  in memcg_offline_kmem()
3752 	 * ordering is imposed by list_lru_node->lock taken by  in memcg_offline_kmem()
3756 	css_for_each_descendant_pre(css, &memcg->css) {  in memcg_offline_kmem()
3758 		BUG_ON(child->kmemcg_id != kmemcg_id);  in memcg_offline_kmem()
3759 		child->kmemcg_id = parent->kmemcg_id;  in memcg_offline_kmem()
3760 		if (!memcg->use_hierarchy)  in memcg_offline_kmem()
3773 	if (unlikely(memcg->kmem_state == KMEM_ONLINE))  in memcg_free_kmem()
3795 	ret = page_counter_set_max(&memcg->kmem, max);  in memcg_update_kmem_max()
3806 	ret = page_counter_set_max(&memcg->tcpmem, max);  in memcg_update_tcp_max()
3810 	if (!memcg->tcpmem_active) {  in memcg_update_tcp_max()
3828 		memcg->tcpmem_active = true;  in memcg_update_tcp_max()
3847 	ret = page_counter_memparse(buf, "-1", &nr_pages);  in mem_cgroup_write()
3851 	switch (MEMFILE_ATTR(of_cft(of)->private)) {  in mem_cgroup_write()
3853 		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */  in mem_cgroup_write()
3854 			ret = -EINVAL;  in mem_cgroup_write()
3857 		switch (MEMFILE_TYPE(of_cft(of)->private)) {  in mem_cgroup_write()
3866 				     "Please report your usecase to linux-mm@kvack.org if you "  in mem_cgroup_write()
3876 		memcg->soft_limit = nr_pages;  in mem_cgroup_write()
3889 	switch (MEMFILE_TYPE(of_cft(of)->private)) {  in mem_cgroup_reset()
3891 		counter = &memcg->memory;  in mem_cgroup_reset()
3894 		counter = &memcg->memsw;  in mem_cgroup_reset()
3897 		counter = &memcg->kmem;  in mem_cgroup_reset()
3900 		counter = &memcg->tcpmem;  in mem_cgroup_reset()
3906 	switch (MEMFILE_ATTR(of_cft(of)->private)) {  in mem_cgroup_reset()
3911 		counter->failcnt = 0;  in mem_cgroup_reset()
3923 	return mem_cgroup_from_css(css)->move_charge_at_immigrate;  in mem_cgroup_move_charge_read()
3933 		return -EINVAL;  in mem_cgroup_move_charge_write()
3936 	 * No kind of locking is needed in here, because ->can_attach() will  in mem_cgroup_move_charge_write()
3941 	memcg->move_charge_at_immigrate = val;  in mem_cgroup_move_charge_write()
3948 	return -ENOSYS;  in mem_cgroup_move_charge_write()
3956 #define LRU_ALL	     ((1 << NR_LRU_LISTS) - 1)
4014 		seq_printf(m, "%s=%lu", stat->name,  in memcg_numa_stat_show()
4015 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,  in memcg_numa_stat_show()
4020 							stat->lru_mask, false));  in memcg_numa_stat_show()
4026 		seq_printf(m, "hierarchical_%s=%lu", stat->name,  in memcg_numa_stat_show()
4027 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,  in memcg_numa_stat_show()
4032 							stat->lru_mask, true));  in memcg_numa_stat_show()
4108 		memory = min(memory, READ_ONCE(mi->memory.max));  in memcg_stat_show()
4109 		memsw = min(memsw, READ_ONCE(mi->memsw.max));  in memcg_stat_show()
4149 			mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);  in memcg_stat_show()
4151 			anon_cost += mz->lruvec.anon_cost;  in memcg_stat_show()
4152 			file_cost += mz->lruvec.file_cost;  in memcg_stat_show()
4176 		return -EINVAL;  in mem_cgroup_swappiness_write()
4178 	if (css->parent)  in mem_cgroup_swappiness_write()
4179 		memcg->swappiness = val;  in mem_cgroup_swappiness_write()
4194 		t = rcu_dereference(memcg->thresholds.primary);  in __mem_cgroup_threshold()
4196 		t = rcu_dereference(memcg->memsw_thresholds.primary);  in __mem_cgroup_threshold()
4208 	i = t->current_threshold;  in __mem_cgroup_threshold()
4216 	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)  in __mem_cgroup_threshold()
4217 		eventfd_signal(t->entries[i].eventfd, 1);  in __mem_cgroup_threshold()
4228 	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)  in __mem_cgroup_threshold()
4229 		eventfd_signal(t->entries[i].eventfd, 1);  in __mem_cgroup_threshold()
4232 	t->current_threshold = i - 1;  in __mem_cgroup_threshold()
4253 	if (_a->threshold > _b->threshold)  in compare_thresholds()
4256 	if (_a->threshold < _b->threshold)  in compare_thresholds()
4257 		return -1;  in compare_thresholds()
4268 	list_for_each_entry(ev, &memcg->oom_notify, list)  in mem_cgroup_oom_notify_cb()
4269 		eventfd_signal(ev->eventfd, 1);  in mem_cgroup_oom_notify_cb()
4292 	ret = page_counter_memparse(args, "-1", &threshold);  in __mem_cgroup_usage_register_event()
4296 	mutex_lock(&memcg->thresholds_lock);  in __mem_cgroup_usage_register_event()
4299 		thresholds = &memcg->thresholds;  in __mem_cgroup_usage_register_event()
4302 		thresholds = &memcg->memsw_thresholds;  in __mem_cgroup_usage_register_event()
4308 	if (thresholds->primary)  in __mem_cgroup_usage_register_event()
4311 	size = thresholds->primary ? thresholds->primary->size + 1 : 1;  in __mem_cgroup_usage_register_event()
4316 		ret = -ENOMEM;  in __mem_cgroup_usage_register_event()
4319 	new->size = size;  in __mem_cgroup_usage_register_event()
4322 	if (thresholds->primary)  in __mem_cgroup_usage_register_event()
4323 		memcpy(new->entries, thresholds->primary->entries,  in __mem_cgroup_usage_register_event()
4324 		       flex_array_size(new, entries, size - 1));  in __mem_cgroup_usage_register_event()
4327 	new->entries[size - 1].eventfd = eventfd;  in __mem_cgroup_usage_register_event()
4328 	new->entries[size - 1].threshold = threshold;  in __mem_cgroup_usage_register_event()
4330 	/* Sort thresholds. Registering of new threshold isn't time-critical */  in __mem_cgroup_usage_register_event()
4331 	sort(new->entries, size, sizeof(*new->entries),  in __mem_cgroup_usage_register_event()
4334 	/* Find current threshold */  in __mem_cgroup_usage_register_event()
4335 	new->current_threshold = -1;  in __mem_cgroup_usage_register_event()
4337 		if (new->entries[i].threshold <= usage) {  in __mem_cgroup_usage_register_event()
4339 			 * new->current_threshold will not be used until  in __mem_cgroup_usage_register_event()
4343 			++new->current_threshold;  in __mem_cgroup_usage_register_event()
4349 	kfree(thresholds->spare);  in __mem_cgroup_usage_register_event()
4350 	thresholds->spare = thresholds->primary;  in __mem_cgroup_usage_register_event()
4352 	rcu_assign_pointer(thresholds->primary, new);  in __mem_cgroup_usage_register_event()
4358 	mutex_unlock(&memcg->thresholds_lock);  in __mem_cgroup_usage_register_event()
4383 	mutex_lock(&memcg->thresholds_lock);  in __mem_cgroup_usage_unregister_event()
4386 		thresholds = &memcg->thresholds;  in __mem_cgroup_usage_unregister_event()
4389 		thresholds = &memcg->memsw_thresholds;  in __mem_cgroup_usage_unregister_event()
4394 	if (!thresholds->primary)  in __mem_cgroup_usage_unregister_event()
4402 	for (i = 0; i < thresholds->primary->size; i++) {  in __mem_cgroup_usage_unregister_event()
4403 		if (thresholds->primary->entries[i].eventfd != eventfd)  in __mem_cgroup_usage_unregister_event()
4409 	new = thresholds->spare;  in __mem_cgroup_usage_unregister_event()
4422 	new->size = size;  in __mem_cgroup_usage_unregister_event()
4424 	/* Copy thresholds and find current threshold */  in __mem_cgroup_usage_unregister_event()
4425 	new->current_threshold = -1;  in __mem_cgroup_usage_unregister_event()
4426 	for (i = 0, j = 0; i < thresholds->primary->size; i++) {  in __mem_cgroup_usage_unregister_event()
4427 		if (thresholds->primary->entries[i].eventfd == eventfd)  in __mem_cgroup_usage_unregister_event()
4430 		new->entries[j] = thresholds->primary->entries[i];  in __mem_cgroup_usage_unregister_event()
4431 		if (new->entries[j].threshold <= usage) {  in __mem_cgroup_usage_unregister_event()
4433 			 * new->current_threshold will not be used  in __mem_cgroup_usage_unregister_event()
4437 			++new->current_threshold;  in __mem_cgroup_usage_unregister_event()
4444 	thresholds->spare = thresholds->primary;  in __mem_cgroup_usage_unregister_event()
4446 	rcu_assign_pointer(thresholds->primary, new);  in __mem_cgroup_usage_unregister_event()
4453 		kfree(thresholds->spare);  in __mem_cgroup_usage_unregister_event()
4454 		thresholds->spare = NULL;  in __mem_cgroup_usage_unregister_event()
4457 	mutex_unlock(&memcg->thresholds_lock);  in __mem_cgroup_usage_unregister_event()
4479 		return -ENOMEM;  in mem_cgroup_oom_register_event()
4483 	event->eventfd = eventfd;  in mem_cgroup_oom_register_event()
4484 	list_add(&event->list, &memcg->oom_notify);  in mem_cgroup_oom_register_event()
4487 	if (memcg->under_oom)  in mem_cgroup_oom_register_event()
4501 	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {  in mem_cgroup_oom_unregister_event()
4502 		if (ev->eventfd == eventfd) {  in mem_cgroup_oom_unregister_event()
4503 			list_del(&ev->list);  in mem_cgroup_oom_unregister_event()
4515 	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);  in mem_cgroup_oom_control_read()
4516 	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);  in mem_cgroup_oom_control_read()
4518 		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));  in mem_cgroup_oom_control_read()
4528 	if (!css->parent || !((val == 0) || (val == 1)))  in mem_cgroup_oom_control_write()
4529 		return -EINVAL;  in mem_cgroup_oom_control_write()
4531 	memcg->oom_kill_disable = val;  in mem_cgroup_oom_control_write()
4544 	return wb_domain_init(&memcg->cgwb_domain, gfp);  in memcg_wb_domain_init()
4549 	wb_domain_exit(&memcg->cgwb_domain);  in memcg_wb_domain_exit()
4554 	wb_domain_size_changed(&memcg->cgwb_domain);  in memcg_wb_domain_size_changed()
4559 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);  in mem_cgroup_wb_domain()
4561 	if (!memcg->css.parent)  in mem_cgroup_wb_domain()
4564 	return &memcg->cgwb_domain;  in mem_cgroup_wb_domain()
4573 	long x = atomic_long_read(&memcg->vmstats[idx]);  in memcg_exact_page_state()
4577 		x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];  in memcg_exact_page_state()
4584  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4592  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
4595  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
4605 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);  in mem_cgroup_wb_stats()
4616 		unsigned long ceiling = min(READ_ONCE(memcg->memory.max),  in mem_cgroup_wb_stats()
4617 					    READ_ONCE(memcg->memory.high));  in mem_cgroup_wb_stats()
4618 		unsigned long used = page_counter_read(&memcg->memory);  in mem_cgroup_wb_stats()
4620 		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));  in mem_cgroup_wb_stats()
4629  * trackes ownership per-page while the latter per-inode.  This was a
4630  * deliberate design decision because honoring per-page ownership in the
4632  * and deemed unnecessary given that write-sharing an inode across
4633  * different cgroups isn't a common use-case.
4635  * Combined with inode majority-writer ownership switching, this works well
4656  * page - a page whose memcg and writeback ownerships don't match - is
4662  * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
4672 	struct mem_cgroup *memcg = page->mem_cgroup;  in mem_cgroup_track_foreign_dirty_slowpath()
4676 	int oldest = -1;  in mem_cgroup_track_foreign_dirty_slowpath()
4687 		frn = &memcg->cgwb_frn[i];  in mem_cgroup_track_foreign_dirty_slowpath()
4688 		if (frn->bdi_id == wb->bdi->id &&  in mem_cgroup_track_foreign_dirty_slowpath()
4689 		    frn->memcg_id == wb->memcg_css->id)  in mem_cgroup_track_foreign_dirty_slowpath()
4691 		if (time_before64(frn->at, oldest_at) &&  in mem_cgroup_track_foreign_dirty_slowpath()
4692 		    atomic_read(&frn->done.cnt) == 1) {  in mem_cgroup_track_foreign_dirty_slowpath()
4694 			oldest_at = frn->at;  in mem_cgroup_track_foreign_dirty_slowpath()
4700 		 * Re-using an existing one.  Update timestamp lazily to  in mem_cgroup_track_foreign_dirty_slowpath()
4702 		 * reasonably up-to-date and significantly shorter than  in mem_cgroup_track_foreign_dirty_slowpath()
4710 		if (time_before64(frn->at, now - update_intv))  in mem_cgroup_track_foreign_dirty_slowpath()
4711 			frn->at = now;  in mem_cgroup_track_foreign_dirty_slowpath()
4714 		frn = &memcg->cgwb_frn[oldest];  in mem_cgroup_track_foreign_dirty_slowpath()
4715 		frn->bdi_id = wb->bdi->id;  in mem_cgroup_track_foreign_dirty_slowpath()
4716 		frn->memcg_id = wb->memcg_css->id;  in mem_cgroup_track_foreign_dirty_slowpath()
4717 		frn->at = now;  in mem_cgroup_track_foreign_dirty_slowpath()
4724 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);  in mem_cgroup_flush_foreign()
4730 		struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];  in mem_cgroup_flush_foreign()
4738 		if (time_after64(frn->at, now - intv) &&  in mem_cgroup_flush_foreign()
4739 		    atomic_read(&frn->done.cnt) == 1) {  in mem_cgroup_flush_foreign()
4740 			frn->at = 0;  in mem_cgroup_flush_foreign()
4741 			trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);  in mem_cgroup_flush_foreign()
4742 			cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,  in mem_cgroup_flush_foreign()
4744 					       &frn->done);  in mem_cgroup_flush_foreign()
4771  * This is way over-engineered.  It tries to support fully configurable
4788 	struct mem_cgroup *memcg = event->memcg;  in memcg_event_remove()
4790 	remove_wait_queue(event->wqh, &event->wait);  in memcg_event_remove()
4792 	event->unregister_event(memcg, event->eventfd);  in memcg_event_remove()
4795 	eventfd_signal(event->eventfd, 1);  in memcg_event_remove()
4797 	eventfd_ctx_put(event->eventfd);  in memcg_event_remove()
4799 	css_put(&memcg->css);  in memcg_event_remove()
4805  * Called with wqh->lock held and interrupts disabled.
4812 	struct mem_cgroup *memcg = event->memcg;  in memcg_event_wake()
4822 		 * side will require wqh->lock via remove_wait_queue(),  in memcg_event_wake()
4825 		spin_lock(&memcg->event_list_lock);  in memcg_event_wake()
4826 		if (!list_empty(&event->list)) {  in memcg_event_wake()
4827 			list_del_init(&event->list);  in memcg_event_wake()
4832 			schedule_work(&event->remove);  in memcg_event_wake()
4834 		spin_unlock(&memcg->event_list_lock);  in memcg_event_wake()
4846 	event->wqh = wqh;  in memcg_event_ptable_queue_proc()
4847 	add_wait_queue(wqh, &event->wait);  in memcg_event_ptable_queue_proc()
4876 		return -EINVAL;  in memcg_write_event_control()
4881 		return -EINVAL;  in memcg_write_event_control()
4886 		return -ENOMEM;  in memcg_write_event_control()
4888 	event->memcg = memcg;  in memcg_write_event_control()
4889 	INIT_LIST_HEAD(&event->list);  in memcg_write_event_control()
4890 	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);  in memcg_write_event_control()
4891 	init_waitqueue_func_entry(&event->wait, memcg_event_wake);  in memcg_write_event_control()
4892 	INIT_WORK(&event->remove, memcg_event_remove);  in memcg_write_event_control()
4896 		ret = -EBADF;  in memcg_write_event_control()
4900 	event->eventfd = eventfd_ctx_fileget(efile.file);  in memcg_write_event_control()
4901 	if (IS_ERR(event->eventfd)) {  in memcg_write_event_control()
4902 		ret = PTR_ERR(event->eventfd);  in memcg_write_event_control()
4908 		ret = -EBADF;  in memcg_write_event_control()
4926 	name = cfile.file->f_path.dentry->d_name.name;  in memcg_write_event_control()
4929 		event->register_event = mem_cgroup_usage_register_event;  in memcg_write_event_control()
4930 		event->unregister_event = mem_cgroup_usage_unregister_event;  in memcg_write_event_control()
4932 		event->register_event = mem_cgroup_oom_register_event;  in memcg_write_event_control()
4933 		event->unregister_event = mem_cgroup_oom_unregister_event;  in memcg_write_event_control()
4935 		event->register_event = vmpressure_register_event;  in memcg_write_event_control()
4936 		event->unregister_event = vmpressure_unregister_event;  in memcg_write_event_control()
4938 		event->register_event = memsw_cgroup_usage_register_event;  in memcg_write_event_control()
4939 		event->unregister_event = memsw_cgroup_usage_unregister_event;  in memcg_write_event_control()
4941 		ret = -EINVAL;  in memcg_write_event_control()
4950 	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,  in memcg_write_event_control()
4952 	ret = -EINVAL;  in memcg_write_event_control()
4960 	ret = event->register_event(memcg, event->eventfd, buf);  in memcg_write_event_control()
4964 	vfs_poll(efile.file, &event->pt);  in memcg_write_event_control()
4966 	spin_lock(&memcg->event_list_lock);  in memcg_write_event_control()
4967 	list_add(&event->list, &memcg->event_list);  in memcg_write_event_control()
4968 	spin_unlock(&memcg->event_list_lock);  in memcg_write_event_control()
4980 	eventfd_ctx_put(event->eventfd);  in memcg_write_event_control()
5121  * Swap-out records and page cache shadow entries need to store memcg
5124  * memory-controlled cgroups to 64k.
5131  * even when there are much fewer than 64k cgroups - possibly none.
5133  * Maintain a private 16-bit ID space for memcg, and allow the ID to
5146 	if (memcg->id.id > 0) {  in mem_cgroup_id_remove()
5147 		idr_remove(&mem_cgroup_idr, memcg->id.id);  in mem_cgroup_id_remove()
5148 		memcg->id.id = 0;  in mem_cgroup_id_remove()
5155 	refcount_add(n, &memcg->id.ref);  in mem_cgroup_id_get_many()
5160 	if (refcount_sub_and_test(n, &memcg->id.ref)) {  in mem_cgroup_id_put_many()
5164 		css_put(&memcg->css);  in mem_cgroup_id_put_many()
5174  * mem_cgroup_from_id - look up a memcg from a memcg id
5198 		tmp = -1;  in alloc_mem_cgroup_per_node_info()
5203 	pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,  in alloc_mem_cgroup_per_node_info()
5205 	if (!pn->lruvec_stat_local) {  in alloc_mem_cgroup_per_node_info()
5210 	pn->lruvec_stat_cpu = alloc_percpu_gfp(struct lruvec_stat,  in alloc_mem_cgroup_per_node_info()
5212 	if (!pn->lruvec_stat_cpu) {  in alloc_mem_cgroup_per_node_info()
5213 		free_percpu(pn->lruvec_stat_local);  in alloc_mem_cgroup_per_node_info()
5218 	lruvec_init(&pn->lruvec);  in alloc_mem_cgroup_per_node_info()
5219 	pn->usage_in_excess = 0;  in alloc_mem_cgroup_per_node_info()
5220 	pn->on_tree = false;  in alloc_mem_cgroup_per_node_info()
5221 	pn->memcg = memcg;  in alloc_mem_cgroup_per_node_info()
5223 	memcg->nodeinfo[node] = pn;  in alloc_mem_cgroup_per_node_info()
5229 	struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];  in free_mem_cgroup_per_node_info()
5234 	free_percpu(pn->lruvec_stat_cpu);  in free_mem_cgroup_per_node_info()
5235 	free_percpu(pn->lruvec_stat_local);  in free_mem_cgroup_per_node_info()
5245 	free_percpu(memcg->vmstats_percpu);  in __mem_cgroup_free()
5246 	free_percpu(memcg->vmstats_local);  in __mem_cgroup_free()
5268 	long error = -ENOMEM;  in mem_cgroup_alloc()
5277 	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,  in mem_cgroup_alloc()
5280 	if (memcg->id.id < 0) {  in mem_cgroup_alloc()
5281 		error = memcg->id.id;  in mem_cgroup_alloc()
5285 	memcg->vmstats_local = alloc_percpu_gfp(struct memcg_vmstats_percpu,  in mem_cgroup_alloc()
5287 	if (!memcg->vmstats_local)  in mem_cgroup_alloc()
5290 	memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,  in mem_cgroup_alloc()
5292 	if (!memcg->vmstats_percpu)  in mem_cgroup_alloc()
5302 	INIT_WORK(&memcg->high_work, high_work_func);  in mem_cgroup_alloc()
5303 	INIT_LIST_HEAD(&memcg->oom_notify);  in mem_cgroup_alloc()
5304 	mutex_init(&memcg->thresholds_lock);  in mem_cgroup_alloc()
5305 	spin_lock_init(&memcg->move_lock);  in mem_cgroup_alloc()
5306 	vmpressure_init(&memcg->vmpressure);  in mem_cgroup_alloc()
5307 	INIT_LIST_HEAD(&memcg->event_list);  in mem_cgroup_alloc()
5308 	spin_lock_init(&memcg->event_list_lock);  in mem_cgroup_alloc()
5309 	memcg->socket_pressure = jiffies;  in mem_cgroup_alloc()
5311 	memcg->kmemcg_id = -1;  in mem_cgroup_alloc()
5312 	INIT_LIST_HEAD(&memcg->objcg_list);  in mem_cgroup_alloc()
5315 	INIT_LIST_HEAD(&memcg->cgwb_list);  in mem_cgroup_alloc()
5317 		memcg->cgwb_frn[i].done =  in mem_cgroup_alloc()
5321 	spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);  in mem_cgroup_alloc()
5322 	INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);  in mem_cgroup_alloc()
5323 	memcg->deferred_split_queue.split_queue_len = 0;  in mem_cgroup_alloc()
5325 	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);  in mem_cgroup_alloc()
5338 	long error = -ENOMEM;  in mem_cgroup_css_alloc()
5346 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);  in mem_cgroup_css_alloc()
5347 	memcg->soft_limit = PAGE_COUNTER_MAX;  in mem_cgroup_css_alloc()
5348 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);  in mem_cgroup_css_alloc()
5350 		memcg->swappiness = mem_cgroup_swappiness(parent);  in mem_cgroup_css_alloc()
5351 		memcg->oom_kill_disable = parent->oom_kill_disable;  in mem_cgroup_css_alloc()
5354 		page_counter_init(&memcg->memory, NULL);  in mem_cgroup_css_alloc()
5355 		page_counter_init(&memcg->swap, NULL);  in mem_cgroup_css_alloc()
5356 		page_counter_init(&memcg->kmem, NULL);  in mem_cgroup_css_alloc()
5357 		page_counter_init(&memcg->tcpmem, NULL);  in mem_cgroup_css_alloc()
5358 	} else if (parent->use_hierarchy) {  in mem_cgroup_css_alloc()
5359 		memcg->use_hierarchy = true;  in mem_cgroup_css_alloc()
5360 		page_counter_init(&memcg->memory, &parent->memory);  in mem_cgroup_css_alloc()
5361 		page_counter_init(&memcg->swap, &parent->swap);  in mem_cgroup_css_alloc()
5362 		page_counter_init(&memcg->kmem, &parent->kmem);  in mem_cgroup_css_alloc()
5363 		page_counter_init(&memcg->tcpmem, &parent->tcpmem);  in mem_cgroup_css_alloc()
5365 		page_counter_init(&memcg->memory, &root_mem_cgroup->memory);  in mem_cgroup_css_alloc()
5366 		page_counter_init(&memcg->swap, &root_mem_cgroup->swap);  in mem_cgroup_css_alloc()
5367 		page_counter_init(&memcg->kmem, &root_mem_cgroup->kmem);  in mem_cgroup_css_alloc()
5368 		page_counter_init(&memcg->tcpmem, &root_mem_cgroup->tcpmem);  in mem_cgroup_css_alloc()
5381 		return &memcg->css;  in mem_cgroup_css_alloc()
5391 	return &memcg->css;  in mem_cgroup_css_alloc()
5409 		return -ENOMEM;  in mem_cgroup_css_online()
5413 	refcount_set(&memcg->id.ref, 1);  in mem_cgroup_css_online()
5428 	spin_lock(&memcg->event_list_lock);  in mem_cgroup_css_offline()
5429 	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {  in mem_cgroup_css_offline()
5430 		list_del_init(&event->list);  in mem_cgroup_css_offline()
5431 		schedule_work(&event->remove);  in mem_cgroup_css_offline()
5433 	spin_unlock(&memcg->event_list_lock);  in mem_cgroup_css_offline()
5435 	page_counter_set_min(&memcg->memory, 0);  in mem_cgroup_css_offline()
5436 	page_counter_set_low(&memcg->memory, 0);  in mem_cgroup_css_offline()
5460 		wb_wait_for_completion(&memcg->cgwb_frn[i].done);  in mem_cgroup_css_free()
5465 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)  in mem_cgroup_css_free()
5468 	vmpressure_cleanup(&memcg->vmpressure);  in mem_cgroup_css_free()
5469 	cancel_work_sync(&memcg->high_work);  in mem_cgroup_css_free()
5477  * mem_cgroup_css_reset - reset the states of a mem_cgroup
5486  * The current implementation only resets the essential configurations.
5493 	page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);  in mem_cgroup_css_reset()
5494 	page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);  in mem_cgroup_css_reset()
5495 	page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);  in mem_cgroup_css_reset()
5496 	page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);  in mem_cgroup_css_reset()
5497 	page_counter_set_min(&memcg->memory, 0);  in mem_cgroup_css_reset()
5498 	page_counter_set_low(&memcg->memory, 0);  in mem_cgroup_css_reset()
5499 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);  in mem_cgroup_css_reset()
5500 	memcg->soft_limit = PAGE_COUNTER_MAX;  in mem_cgroup_css_reset()
5501 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);  in mem_cgroup_css_reset()
5506 /* Handlers for move charge at task migration. */
5511 	/* Try a single bulk charge without reclaim first, kswapd may wake */  in mem_cgroup_do_precharge()
5519 	while (count--) {  in mem_cgroup_do_precharge()
5595 	entry->val = ent.val;  in mc_handle_swap_pte()
5610 	if (!vma->vm_file) /* anonymous vma */  in mc_handle_file_pte()
5615 	/* page is moved even if it's not RSS of this task(page-faulted). */  in mc_handle_file_pte()
5617 	return find_get_incore_page(vma->vm_file->f_mapping,  in mc_handle_file_pte()
5622  * mem_cgroup_move_account - move account of the page
5624  * @compound: charge the page as compound or small page
5630  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
5649 	 * page->mem_cgroup of its source page while we change it.  in mem_cgroup_move_account()
5651 	ret = -EBUSY;  in mem_cgroup_move_account()
5655 	ret = -EINVAL;  in mem_cgroup_move_account()
5656 	if (page->mem_cgroup != from)  in mem_cgroup_move_account()
5667 			__mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);  in mem_cgroup_move_account()
5671 						   -nr_pages);  in mem_cgroup_move_account()
5678 		__mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);  in mem_cgroup_move_account()
5682 			__mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);  in mem_cgroup_move_account()
5687 			__mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);  in mem_cgroup_move_account()
5692 			struct address_space *mapping = page_mapping(page);  in mem_cgroup_move_account()  local
5694 			if (mapping_can_writeback(mapping)) {  in mem_cgroup_move_account()
5696 						   -nr_pages);  in mem_cgroup_move_account()
5704 		__mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);  in mem_cgroup_move_account()
5711 	 * It is safe to change page->mem_cgroup here because the page  in mem_cgroup_move_account()
5714 	 * that would rely on a stable page->mem_cgroup.  in mem_cgroup_move_account()
5717 	 * to save space. As soon as we switch page->mem_cgroup to a  in mem_cgroup_move_account()
5723 	css_get(&to->css);  in mem_cgroup_move_account()
5724 	css_put(&from->css);  in mem_cgroup_move_account()
5726 	page->mem_cgroup = to;  in mem_cgroup_move_account()
5735 	mem_cgroup_charge_statistics(from, page, -nr_pages);  in mem_cgroup_move_account()
5745  * get_mctgt_type - get target type of moving charge
5752  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
5754  *     move charge. if @target is not NULL, the page is stored in target->page
5757  *     target for charge migration. if @target is not NULL, the entry is stored
5758  *     in target->ent.
5761  *     For now we such page is charge like a regular page would be as for all
5792 		if (page->mem_cgroup == mc.from) {  in get_mctgt_type()
5797 				target->page = page;  in get_mctgt_type()
5804 	 * But we cannot move a tail-page in a THP.  in get_mctgt_type()
5810 			target->ent = ent;  in get_mctgt_type()
5836 	if (page->mem_cgroup == mc.from) {  in get_mctgt_type_thp()
5840 			target->page = page;  in get_mctgt_type_thp()
5857 	struct vm_area_struct *vma = walk->vma;  in mem_cgroup_count_precharge_pte_range()
5876 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);  in mem_cgroup_count_precharge_pte_range()
5880 	pte_unmap_unlock(pte - 1, ptl);  in mem_cgroup_count_precharge_pte_range()
5895 	walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);  in mem_cgroup_count_precharge()
5909 	mc.moving_task = current;  in mem_cgroup_precharge_mc()
5936 			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);  in __mem_cgroup_clear_mc()
5941 		 * we charged both to->memory and to->memsw, so we  in __mem_cgroup_clear_mc()
5942 		 * should uncharge to->memory.  in __mem_cgroup_clear_mc()
5945 			page_counter_uncharge(&mc.to->memory, mc.moved_swap);  in __mem_cgroup_clear_mc()
5983 	/* charge immigration isn't supported on the default hierarchy */  in mem_cgroup_can_attach()
5988 	 * Multi-process migrations only happen on the default hierarchy  in mem_cgroup_can_attach()
5989 	 * where charge immigration is not used.  Perform charge  in mem_cgroup_can_attach()
6004 	 * tunable will only affect upcoming migrations, not the current one.  in mem_cgroup_can_attach()
6007 	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);  in mem_cgroup_can_attach()
6019 	if (mm->owner == p) {  in mem_cgroup_can_attach()
6054 	struct vm_area_struct *vma = walk->vma;  in mem_cgroup_move_charge_pte_range()
6073 					mc.precharge -= HPAGE_PMD_NR;  in mem_cgroup_move_charge_pte_range()
6083 				mc.precharge -= HPAGE_PMD_NR;  in mem_cgroup_move_charge_pte_range()
6095 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);  in mem_cgroup_move_charge_pte_range()
6114 			 * memcg. There should be somebody mapping the head.  in mem_cgroup_move_charge_pte_range()
6122 				mc.precharge--;  in mem_cgroup_move_charge_pte_range()
6134 				mc.precharge--;  in mem_cgroup_move_charge_pte_range()
6144 	pte_unmap_unlock(pte - 1, ptl);  in mem_cgroup_move_charge_pte_range()
6150 		 * We try charge one by one, but don't do any additional  in mem_cgroup_move_charge_pte_range()
6151 		 * charges to mc.to if we have failed in charge once in attach()  in mem_cgroup_move_charge_pte_range()
6172 	 * for already started RCU-only updates to finish.  in mem_cgroup_move_charge()
6174 	atomic_inc(&mc.from->moving_account);  in mem_cgroup_move_charge()
6182 		 * to move enough charges, but moving charge is a best-effort  in mem_cgroup_move_charge()
6191 	 * additional charge, the page walk just aborts.  in mem_cgroup_move_charge()
6193 	walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,  in mem_cgroup_move_charge()
6197 	atomic_dec(&mc.from->moving_account);  in mem_cgroup_move_charge()
6233 		root_mem_cgroup->use_hierarchy = true;  in mem_cgroup_bind()
6235 		root_mem_cgroup->use_hierarchy = false;  in mem_cgroup_bind()
6253 	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;  in memory_current_read()
6259 		READ_ONCE(mem_cgroup_from_seq(m)->memory.min));  in memory_min_show()
6274 	page_counter_set_min(&memcg->memory, min);  in memory_min_write()
6282 		READ_ONCE(mem_cgroup_from_seq(m)->memory.low));  in memory_low_show()
6297 	page_counter_set_low(&memcg->memory, low);  in memory_low_write()
6305 		READ_ONCE(mem_cgroup_from_seq(m)->memory.high));  in memory_high_show()
6323 		unsigned long nr_pages = page_counter_read(&memcg->memory);  in memory_high_write()
6329 		if (signal_pending(current))  in memory_high_write()
6338 		reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,  in memory_high_write()
6341 		if (!reclaimed && !nr_retries--)  in memory_high_write()
6345 	page_counter_set_high(&memcg->memory, high);  in memory_high_write()
6355 		READ_ONCE(mem_cgroup_from_seq(m)->memory.max));  in memory_max_show()
6372 	xchg(&memcg->memory.max, max);  in memory_max_write()
6375 		unsigned long nr_pages = page_counter_read(&memcg->memory);  in memory_max_write()
6380 		if (signal_pending(current))  in memory_max_write()
6390 			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,  in memory_max_write()
6392 				nr_reclaims--;  in memory_max_write()
6419 	__memory_events_show(m, memcg->memory_events);  in memory_events_show()
6427 	__memory_events_show(m, memcg->memory_events_local);  in memory_events_local_show()
6438 		return -ENOMEM;  in memory_stat_show()
6477 	seq_printf(m, "%d\n", memcg->oom_group);  in memory_oom_group_show()
6490 		return -EINVAL;  in memory_oom_group_write()
6497 		return -EINVAL;  in memory_oom_group_write()
6499 	memcg->oom_group = oom_group;  in memory_oom_group_write()
6506 		.name = "current",
6602  *    This makes distribution proportional, but also work-conserving:
6613  *    of the ancestor's claim to protection, any unutilized -
6614  *    "floating" - protection from up the tree is distributed in
6640 	 * claimed protection in order to be work-conserving: claimed  in effective_protection()
6678 	 * aren't read atomically - make sure the division is sane.  in effective_protection()
6687 		unclaimed = parent_effective - siblings_protected;  in effective_protection()
6688 		unclaimed *= usage - protected;  in effective_protection()
6689 		unclaimed /= parent_usage - siblings_protected;  in effective_protection()
6698  * mem_cgroup_protected - check if memory consumption is in the normal range
6699  * @root: the top ancestor of the sub-tree being checked
6703  *          of a top-down tree iteration, not for isolated queries.
6727 	usage = page_counter_read(&memcg->memory);  in mem_cgroup_calculate_protection()
6732 	/* No parent means a non-hierarchical mode on v1 memcg */  in mem_cgroup_calculate_protection()
6737 		memcg->memory.emin = READ_ONCE(memcg->memory.min);  in mem_cgroup_calculate_protection()
6738 		memcg->memory.elow = READ_ONCE(memcg->memory.low);  in mem_cgroup_calculate_protection()
6742 	parent_usage = page_counter_read(&parent->memory);  in mem_cgroup_calculate_protection()
6744 	WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,  in mem_cgroup_calculate_protection()
6745 			READ_ONCE(memcg->memory.min),  in mem_cgroup_calculate_protection()
6746 			READ_ONCE(parent->memory.emin),  in mem_cgroup_calculate_protection()
6747 			atomic_long_read(&parent->memory.children_min_usage)));  in mem_cgroup_calculate_protection()
6749 	WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,  in mem_cgroup_calculate_protection()
6750 			READ_ONCE(memcg->memory.low),  in mem_cgroup_calculate_protection()
6751 			READ_ONCE(parent->memory.elow),  in mem_cgroup_calculate_protection()
6752 			atomic_long_read(&parent->memory.children_low_usage)));  in mem_cgroup_calculate_protection()
6756  * mem_cgroup_charge - charge a newly allocated page to a cgroup
6757  * @page: page to charge
6761  * Try to charge @page to the memcg that @mm belongs to, reclaiming
6780 		 * Every swap fault against a single page tries to charge the  in mem_cgroup_charge()
6782 		 * already charged pages, too.  page->mem_cgroup is protected  in mem_cgroup_charge()
6787 		if (compound_head(page)->mem_cgroup)  in mem_cgroup_charge()
6793 		if (memcg && !css_tryget_online(&memcg->css))  in mem_cgroup_charge()
6805 	css_get(&memcg->css);  in mem_cgroup_charge()
6818 		 * memory+swap charge, drop the swap entry duplicate.  in mem_cgroup_charge()
6824 	css_put(&memcg->css);  in mem_cgroup_charge()
6846 	if (!mem_cgroup_is_root(ug->memcg)) {  in uncharge_batch()
6847 		page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);  in uncharge_batch()
6849 			page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);  in uncharge_batch()
6850 		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)  in uncharge_batch()
6851 			page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);  in uncharge_batch()
6852 		memcg_oom_recover(ug->memcg);  in uncharge_batch()
6856 	__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);  in uncharge_batch()
6857 	__this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);  in uncharge_batch()
6858 	memcg_check_events(ug->memcg, ug->dummy_page);  in uncharge_batch()
6862 	css_put(&ug->memcg->css);  in uncharge_batch()
6871 	if (!page->mem_cgroup)  in uncharge_page()
6876 	 * page->mem_cgroup at this point, we have fully  in uncharge_page()
6880 	if (ug->memcg != page->mem_cgroup) {  in uncharge_page()
6881 		if (ug->memcg) {  in uncharge_page()
6885 		ug->memcg = page->mem_cgroup;  in uncharge_page()
6888 		css_get(&ug->memcg->css);  in uncharge_page()
6892 	ug->nr_pages += nr_pages;  in uncharge_page()
6895 		ug->pgpgout++;  in uncharge_page()
6897 		ug->nr_kmem += nr_pages;  in uncharge_page()
6901 	ug->dummy_page = page;  in uncharge_page()
6902 	page->mem_cgroup = NULL;  in uncharge_page()
6903 	css_put(&ug->memcg->css);  in uncharge_page()
6914 	 * Note that the list can be a single page->lru; hence the  in uncharge_list()
6915 	 * do-while loop instead of a simple list_for_each_entry().  in uncharge_list()
6917 	next = page_list->next;  in uncharge_list()
6922 		next = page->lru.next;  in uncharge_list()
6932  * mem_cgroup_uncharge - uncharge a page
6944 	/* Don't touch page->lru of any random page, pre-check: */  in mem_cgroup_uncharge()
6945 	if (!page->mem_cgroup)  in mem_cgroup_uncharge()
6954  * mem_cgroup_uncharge_list - uncharge a list of page
6970  * mem_cgroup_migrate - charge a page's replacement
6974  * Charge @newpage as a replacement page for @oldpage. @oldpage will
6977  * Both pages must be locked, @newpage->mapping must be set up.
6995 	if (newpage->mem_cgroup)  in mem_cgroup_migrate()
6999 	memcg = oldpage->mem_cgroup;  in mem_cgroup_migrate()
7003 	/* Force-charge the new page. The old one will be freed soon */  in mem_cgroup_migrate()
7006 	page_counter_charge(&memcg->memory, nr_pages);  in mem_cgroup_migrate()
7008 		page_counter_charge(&memcg->memsw, nr_pages);  in mem_cgroup_migrate()
7010 	css_get(&memcg->css);  in mem_cgroup_migrate()
7034 	memcg = mem_cgroup_from_task(current);  in mem_cgroup_sk_alloc()
7037 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)  in mem_cgroup_sk_alloc()
7039 	if (css_tryget(&memcg->css))  in mem_cgroup_sk_alloc()
7040 		sk->sk_memcg = memcg;  in mem_cgroup_sk_alloc()
7047 	if (sk->sk_memcg)  in mem_cgroup_sk_free()
7048 		css_put(&sk->sk_memcg->css);  in mem_cgroup_sk_free()
7052  * mem_cgroup_charge_skmem - charge socket memory
7053  * @memcg: memcg to charge
7054  * @nr_pages: number of pages to charge
7056  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
7057  * @memcg's configured limit, %false if the charge had to be forced.
7066 		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {  in mem_cgroup_charge_skmem()
7067 			memcg->tcpmem_pressure = 0;  in mem_cgroup_charge_skmem()
7070 		page_counter_charge(&memcg->tcpmem, nr_pages);  in mem_cgroup_charge_skmem()
7071 		memcg->tcpmem_pressure = 1;  in mem_cgroup_charge_skmem()
7089  * mem_cgroup_uncharge_skmem - uncharge socket memory
7096 		page_counter_uncharge(&memcg->tcpmem, nr_pages);  in mem_cgroup_uncharge_skmem()
7100 	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);  in mem_cgroup_uncharge_skmem()
7125  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
7137 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,  in mem_cgroup_init()
7146 		rtpn->rb_root = RB_ROOT;  in mem_cgroup_init()
7147 		rtpn->rb_rightmost = NULL;  in mem_cgroup_init()
7148 		spin_lock_init(&rtpn->lock);  in mem_cgroup_init()
7159 	while (!refcount_inc_not_zero(&memcg->id.ref)) {  in mem_cgroup_id_get_online()
7176  * mem_cgroup_swapout - transfer a memsw charge to swap
7177  * @page: page whose memsw charge to transfer
7178  * @entry: swap entry to move the charge to
7180  * Transfer the memsw charge of @page to @entry.
7194 	memcg = page->mem_cgroup;  in mem_cgroup_swapout()
7202 	 * have an ID allocated to it anymore, charge the closest online  in mem_cgroup_swapout()
7203 	 * ancestor for the swap instead and transfer the memory+swap charge.  in mem_cgroup_swapout()
7209 		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);  in mem_cgroup_swapout()
7215 	page->mem_cgroup = NULL;  in mem_cgroup_swapout()
7218 		page_counter_uncharge(&memcg->memory, nr_entries);  in mem_cgroup_swapout()
7222 			page_counter_charge(&swap_memcg->memsw, nr_entries);  in mem_cgroup_swapout()
7223 		page_counter_uncharge(&memcg->memsw, nr_entries);  in mem_cgroup_swapout()
7228 	 * i_pages lock which is taken with interrupts-off. It is  in mem_cgroup_swapout()
7230 	 * only synchronisation we have for updating the per-CPU variables.  in mem_cgroup_swapout()
7233 	mem_cgroup_charge_statistics(memcg, page, -nr_entries);  in mem_cgroup_swapout()
7236 	css_put(&memcg->css);  in mem_cgroup_swapout()
7240  * mem_cgroup_try_charge_swap - try charging swap space for a page
7242  * @entry: swap entry to charge
7244  * Try to charge @page's memcg for the swap space at @entry.
7246  * Returns 0 on success, -ENOMEM on failure.
7258 	memcg = page->mem_cgroup;  in mem_cgroup_try_charge_swap()
7272 	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {  in mem_cgroup_try_charge_swap()
7276 		return -ENOMEM;  in mem_cgroup_try_charge_swap()
7281 		mem_cgroup_id_get_many(memcg, nr_pages - 1);  in mem_cgroup_try_charge_swap()
7290  * mem_cgroup_uncharge_swap - uncharge swap space
7305 				page_counter_uncharge(&memcg->swap, nr_pages);  in mem_cgroup_uncharge_swap()
7307 				page_counter_uncharge(&memcg->memsw, nr_pages);  in mem_cgroup_uncharge_swap()
7309 		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);  in mem_cgroup_uncharge_swap()
7323 				      READ_ONCE(memcg->swap.max) -  in mem_cgroup_get_nr_swap_pages()
7324 				      page_counter_read(&memcg->swap));  in mem_cgroup_get_nr_swap_pages()
7339 	memcg = page->mem_cgroup;  in mem_cgroup_swap_full()
7344 		unsigned long usage = page_counter_read(&memcg->swap);  in mem_cgroup_swap_full()
7346 		if (usage * 2 >= READ_ONCE(memcg->swap.high) ||  in mem_cgroup_swap_full()
7347 		    usage * 2 >= READ_ONCE(memcg->swap.max))  in mem_cgroup_swap_full()
7369 	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;  in swap_current_read()
7375 		READ_ONCE(mem_cgroup_from_seq(m)->swap.high));  in swap_high_show()
7390 	page_counter_set_high(&memcg->swap, high);  in swap_high_write()
7398 		READ_ONCE(mem_cgroup_from_seq(m)->swap.max));  in swap_max_show()
7413 	xchg(&memcg->swap.max, max);  in swap_max_write()
7423 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));  in swap_events_show()
7425 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));  in swap_events_show()
7427 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));  in swap_events_show()
7434 		.name = "swap.current",
7495 	/* No memory control -> no swap control */  in mem_cgroup_swap_init()