260 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
265 	struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);  in obj_cgroup_release()  local
290 	nr_bytes = atomic_read(&objcg->nr_charged_bytes);  in obj_cgroup_release()
295 		obj_cgroup_uncharge_pages(objcg, nr_pages);  in obj_cgroup_release()
298 	list_del(&objcg->list);  in obj_cgroup_release()
302 	kfree_rcu(objcg, rcu);  in obj_cgroup_release()
307 	struct obj_cgroup *objcg;  in obj_cgroup_alloc()  local
310 	objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);  in obj_cgroup_alloc()
311 	if (!objcg)  in obj_cgroup_alloc()
314 	ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,  in obj_cgroup_alloc()
317 		kfree(objcg);  in obj_cgroup_alloc()
320 	INIT_LIST_HEAD(&objcg->list);  in obj_cgroup_alloc()
321 	return objcg;  in obj_cgroup_alloc()
327 	struct obj_cgroup *objcg, *iter;  in memcg_reparent_objcgs()  local
329 	objcg = rcu_replace_pointer(memcg->objcg, NULL, true);  in memcg_reparent_objcgs()
334 	list_add(&objcg->list, &memcg->objcg_list);  in memcg_reparent_objcgs()
343 	percpu_ref_kill(&objcg->refcnt);  in memcg_reparent_objcgs()
2873 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,  in mod_objcg_mlstate()  argument
2881 	memcg = obj_cgroup_memcg(objcg);  in mod_objcg_mlstate()
3006 	struct obj_cgroup *objcg = NULL;  in __get_obj_cgroup_from_memcg()  local
3009 		objcg = rcu_dereference(memcg->objcg);  in __get_obj_cgroup_from_memcg()
3010 		if (objcg && obj_cgroup_tryget(objcg))  in __get_obj_cgroup_from_memcg()
3012 		objcg = NULL;  in __get_obj_cgroup_from_memcg()
3014 	return objcg;  in __get_obj_cgroup_from_memcg()
3019 	struct obj_cgroup *objcg = NULL;  in get_obj_cgroup_from_current()  local
3030 	objcg = __get_obj_cgroup_from_memcg(memcg);  in get_obj_cgroup_from_current()
3032 	return objcg;  in get_obj_cgroup_from_current()
3037 	struct obj_cgroup *objcg;  in get_obj_cgroup_from_folio()  local
3043 		objcg = __folio_objcg(folio);  in get_obj_cgroup_from_folio()
3044 		obj_cgroup_get(objcg);  in get_obj_cgroup_from_folio()
3051 			objcg = __get_obj_cgroup_from_memcg(memcg);  in get_obj_cgroup_from_folio()
3053 			objcg = NULL;  in get_obj_cgroup_from_folio()
3056 	return objcg;  in get_obj_cgroup_from_folio()
3076 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,  in obj_cgroup_uncharge_pages()  argument
3081 	memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_uncharge_pages()
3097 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,  in obj_cgroup_charge_pages()  argument
3103 	memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_charge_pages()
3126 	struct obj_cgroup *objcg;  in __memcg_kmem_charge_page()  local
3129 	objcg = get_obj_cgroup_from_current();  in __memcg_kmem_charge_page()
3130 	if (objcg) {  in __memcg_kmem_charge_page()
3131 		ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);  in __memcg_kmem_charge_page()
3133 			page->memcg_data = (unsigned long)objcg |  in __memcg_kmem_charge_page()
3137 		obj_cgroup_put(objcg);  in __memcg_kmem_charge_page()
3150 	struct obj_cgroup *objcg;  in __memcg_kmem_uncharge_page()  local
3156 	objcg = __folio_objcg(folio);  in __memcg_kmem_uncharge_page()
3157 	obj_cgroup_uncharge_pages(objcg, nr_pages);  in __memcg_kmem_uncharge_page()
3159 	obj_cgroup_put(objcg);  in __memcg_kmem_uncharge_page()
3162 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,  in mod_objcg_state()  argument
3178 	if (READ_ONCE(stock->cached_objcg) != objcg) {  in mod_objcg_state()
3180 		obj_cgroup_get(objcg);  in mod_objcg_state()
3181 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)  in mod_objcg_state()
3182 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;  in mod_objcg_state()
3183 		WRITE_ONCE(stock->cached_objcg, objcg);  in mod_objcg_state()
3190 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B,  in mod_objcg_state()
3195 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B,  in mod_objcg_state()
3221 		mod_objcg_mlstate(objcg, pgdat, idx, nr);  in mod_objcg_state()
3228 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)  in consume_obj_stock()  argument
3237 	if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) {  in consume_obj_stock()
3313 	struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg);  in obj_stock_flush_required()  local
3316 	if (objcg) {  in obj_stock_flush_required()
3317 		memcg = obj_cgroup_memcg(objcg);  in obj_stock_flush_required()
3325 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,  in refill_obj_stock()  argument
3336 	if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */  in refill_obj_stock()
3338 		obj_cgroup_get(objcg);  in refill_obj_stock()
3339 		WRITE_ONCE(stock->cached_objcg, objcg);  in refill_obj_stock()
3340 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)  in refill_obj_stock()
3341 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;  in refill_obj_stock()
3356 		obj_cgroup_uncharge_pages(objcg, nr_pages);  in refill_obj_stock()
3359 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)  in obj_cgroup_charge()  argument
3364 	if (consume_obj_stock(objcg, size))  in obj_cgroup_charge()
3396 	ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);  in obj_cgroup_charge()
3398 		refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);  in obj_cgroup_charge()
3403 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)  in obj_cgroup_uncharge()  argument
3405 	refill_obj_stock(objcg, size, true);  in obj_cgroup_uncharge()
3750 	struct obj_cgroup *objcg;  in memcg_online_kmem()  local
3758 	objcg = obj_cgroup_alloc();  in memcg_online_kmem()
3759 	if (!objcg)  in memcg_online_kmem()
3762 	objcg->memcg = memcg;  in memcg_online_kmem()
3763 	rcu_assign_pointer(memcg->objcg, objcg);  in memcg_online_kmem()
7131 	struct obj_cgroup *objcg;  in uncharge_folio()  local
7141 		objcg = __folio_objcg(folio);  in uncharge_folio()
7146 		memcg = get_mem_cgroup_from_objcg(objcg);  in uncharge_folio()
7173 		obj_cgroup_put(objcg);  in uncharge_folio()
7770 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)  in obj_cgroup_may_zswap()  argument
7778 	original_memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_may_zswap()
7810 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)  in obj_cgroup_charge_zswap()  argument
7820 	if (obj_cgroup_charge(objcg, GFP_KERNEL, size))  in obj_cgroup_charge_zswap()
7824 	memcg = obj_cgroup_memcg(objcg);  in obj_cgroup_charge_zswap()
7837 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)  in obj_cgroup_uncharge_zswap()  argument
7844 	obj_cgroup_uncharge(objcg, size);  in obj_cgroup_uncharge_zswap()
7847 	memcg = obj_cgroup_memcg(objcg);  in obj_cgroup_uncharge_zswap()