269 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
274 	struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);  in obj_cgroup_release()  local
299 	nr_bytes = atomic_read(&objcg->nr_charged_bytes);  in obj_cgroup_release()
304 		obj_cgroup_uncharge_pages(objcg, nr_pages);  in obj_cgroup_release()
307 	list_del(&objcg->list);  in obj_cgroup_release()
311 	kfree_rcu(objcg, rcu);  in obj_cgroup_release()
316 	struct obj_cgroup *objcg;  in obj_cgroup_alloc()  local
319 	objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);  in obj_cgroup_alloc()
320 	if (!objcg)  in obj_cgroup_alloc()
323 	ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,  in obj_cgroup_alloc()
326 		kfree(objcg);  in obj_cgroup_alloc()
329 	INIT_LIST_HEAD(&objcg->list);  in obj_cgroup_alloc()
330 	return objcg;  in obj_cgroup_alloc()
336 	struct obj_cgroup *objcg, *iter;  in memcg_reparent_objcgs()  local
338 	objcg = rcu_replace_pointer(memcg->objcg, NULL, true);  in memcg_reparent_objcgs()
343 	list_add(&objcg->list, &memcg->objcg_list);  in memcg_reparent_objcgs()
352 	percpu_ref_kill(&objcg->refcnt);  in memcg_reparent_objcgs()
756 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,  in mod_objcg_mlstate()  argument
764 	memcg = obj_cgroup_memcg(objcg);  in mod_objcg_mlstate()
2751 static struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)  in get_mem_cgroup_from_objcg()  argument
2757 	memcg = obj_cgroup_memcg(objcg);  in get_mem_cgroup_from_objcg()
2835 		struct obj_cgroup *objcg;  in mem_cgroup_from_obj()  local
2839 		objcg = page_objcgs(page)[off];  in mem_cgroup_from_obj()
2840 		if (objcg)  in mem_cgroup_from_obj()
2841 			return obj_cgroup_memcg(objcg);  in mem_cgroup_from_obj()
2858 	struct obj_cgroup *objcg = NULL;  in get_obj_cgroup_from_current()  local
2871 		objcg = rcu_dereference(memcg->objcg);  in get_obj_cgroup_from_current()
2872 		if (objcg && obj_cgroup_tryget(objcg))  in get_obj_cgroup_from_current()
2874 		objcg = NULL;  in get_obj_cgroup_from_current()
2878 	return objcg;  in get_obj_cgroup_from_current()
2929 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,  in obj_cgroup_uncharge_pages()  argument
2934 	memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_uncharge_pages()
2951 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,  in obj_cgroup_charge_pages()  argument
2958 	memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_charge_pages()
2995 	struct obj_cgroup *objcg;  in __memcg_kmem_charge_page()  local
2998 	objcg = get_obj_cgroup_from_current();  in __memcg_kmem_charge_page()
2999 	if (objcg) {  in __memcg_kmem_charge_page()
3000 		ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);  in __memcg_kmem_charge_page()
3002 			page->memcg_data = (unsigned long)objcg |  in __memcg_kmem_charge_page()
3006 		obj_cgroup_put(objcg);  in __memcg_kmem_charge_page()
3018 	struct obj_cgroup *objcg;  in __memcg_kmem_uncharge_page()  local
3024 	objcg = __page_objcg(page);  in __memcg_kmem_uncharge_page()
3025 	obj_cgroup_uncharge_pages(objcg, nr_pages);  in __memcg_kmem_uncharge_page()
3027 	obj_cgroup_put(objcg);  in __memcg_kmem_uncharge_page()
3030 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,  in mod_objcg_state()  argument
3042 	if (stock->cached_objcg != objcg) {  in mod_objcg_state()
3044 		obj_cgroup_get(objcg);  in mod_objcg_state()
3045 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)  in mod_objcg_state()
3046 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;  in mod_objcg_state()
3047 		stock->cached_objcg = objcg;  in mod_objcg_state()
3054 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B,  in mod_objcg_state()
3059 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B,  in mod_objcg_state()
3085 		mod_objcg_mlstate(objcg, pgdat, idx, nr);  in mod_objcg_state()
3090 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)  in consume_obj_stock()  argument
3096 	if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {  in consume_obj_stock()
3176 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,  in refill_obj_stock()  argument
3183 	if (stock->cached_objcg != objcg) { /* reset if necessary */  in refill_obj_stock()
3185 		obj_cgroup_get(objcg);  in refill_obj_stock()
3186 		stock->cached_objcg = objcg;  in refill_obj_stock()
3187 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)  in refill_obj_stock()
3188 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;  in refill_obj_stock()
3201 		obj_cgroup_uncharge_pages(objcg, nr_pages);  in refill_obj_stock()
3204 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)  in obj_cgroup_charge()  argument
3209 	if (consume_obj_stock(objcg, size))  in obj_cgroup_charge()
3241 	ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);  in obj_cgroup_charge()
3243 		refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);  in obj_cgroup_charge()
3248 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)  in obj_cgroup_uncharge()  argument
3250 	refill_obj_stock(objcg, size, true);  in obj_cgroup_uncharge()
3590 	struct obj_cgroup *objcg;  in memcg_online_kmem()  local
3603 	objcg = obj_cgroup_alloc();  in memcg_online_kmem()
3604 	if (!objcg) {  in memcg_online_kmem()
3608 	objcg->memcg = memcg;  in memcg_online_kmem()
3609 	rcu_assign_pointer(memcg->objcg, objcg);  in memcg_online_kmem()
6838 	struct obj_cgroup *objcg;  in uncharge_page()  local
6849 		objcg = __page_objcg(page);  in uncharge_page()
6854 		memcg = get_mem_cgroup_from_objcg(objcg);  in uncharge_page()
6881 		obj_cgroup_put(objcg);  in uncharge_page()