136 static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags)  in __alloc()  argument
138 	if (c->percpu_size) {  in __alloc()
139 		void **obj = kmalloc_node(c->percpu_size, flags, node);  in __alloc()
140 		void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);  in __alloc()
151 	return kmalloc_node(c->unit_size, flags | __GFP_ZERO, node);  in __alloc()
154 static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c)  in get_memcg()  argument
157 	if (c->objcg)  in get_memcg()
158 		return get_mem_cgroup_from_objcg(c->objcg);  in get_memcg()
168 static void inc_active(struct bpf_mem_cache *c, unsigned long *flags)  in inc_active()  argument
183 	WARN_ON_ONCE(local_inc_return(&c->active) != 1);  in inc_active()
186 static void dec_active(struct bpf_mem_cache *c, unsigned long *flags)  in dec_active()  argument
188 	local_dec(&c->active);  in dec_active()
193 static void add_obj_to_free_list(struct bpf_mem_cache *c, void *obj)  in add_obj_to_free_list()  argument
197 	inc_active(c, &flags);  in add_obj_to_free_list()
198 	__llist_add(obj, &c->free_llist);  in add_obj_to_free_list()
199 	c->free_cnt++;  in add_obj_to_free_list()
200 	dec_active(c, &flags);  in add_obj_to_free_list()
204 static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic)  in alloc_bulk()  argument
220 		obj = llist_del_first(&c->free_by_rcu_ttrace);  in alloc_bulk()
223 		add_obj_to_free_list(c, obj);  in alloc_bulk()
229 		obj = llist_del_first(&c->waiting_for_gp_ttrace);  in alloc_bulk()
232 		add_obj_to_free_list(c, obj);  in alloc_bulk()
237 	memcg = get_memcg(c);  in alloc_bulk()
245 		obj = __alloc(c, node, gfp);  in alloc_bulk()
248 		add_obj_to_free_list(c, obj);  in alloc_bulk()
279 	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace);  in __free_rcu()  local
281 	free_all(llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size);  in __free_rcu()
282 	atomic_set(&c->call_rcu_ttrace_in_progress, 0);  in __free_rcu()
296 static void enque_to_free(struct bpf_mem_cache *c, void *obj)  in enque_to_free()  argument
303 	llist_add(llnode, &c->free_by_rcu_ttrace);  in enque_to_free()
306 static void do_call_rcu_ttrace(struct bpf_mem_cache *c)  in do_call_rcu_ttrace()  argument
310 	if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) {  in do_call_rcu_ttrace()
311 		if (unlikely(READ_ONCE(c->draining))) {  in do_call_rcu_ttrace()
312 			llnode = llist_del_all(&c->free_by_rcu_ttrace);  in do_call_rcu_ttrace()
313 			free_all(llnode, !!c->percpu_size);  in do_call_rcu_ttrace()
318 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace));  in do_call_rcu_ttrace()
319 	llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace))  in do_call_rcu_ttrace()
320 		llist_add(llnode, &c->waiting_for_gp_ttrace);  in do_call_rcu_ttrace()
322 	if (unlikely(READ_ONCE(c->draining))) {  in do_call_rcu_ttrace()
323 		__free_rcu(&c->rcu_ttrace);  in do_call_rcu_ttrace()
332 	call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu_tasks_trace);  in do_call_rcu_ttrace()
335 static void free_bulk(struct bpf_mem_cache *c)  in free_bulk()  argument
337 	struct bpf_mem_cache *tgt = c->tgt;  in free_bulk()
342 	WARN_ON_ONCE(tgt->unit_size != c->unit_size);  in free_bulk()
345 		inc_active(c, &flags);  in free_bulk()
346 		llnode = __llist_del_first(&c->free_llist);  in free_bulk()
348 			cnt = --c->free_cnt;  in free_bulk()
351 		dec_active(c, &flags);  in free_bulk()
354 	} while (cnt > (c->high_watermark + c->low_watermark) / 2);  in free_bulk()
357 	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra))  in free_bulk()
364 	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);  in __free_by_rcu()  local
365 	struct bpf_mem_cache *tgt = c->tgt;  in __free_by_rcu()
368 	llnode = llist_del_all(&c->waiting_for_gp);  in __free_by_rcu()
372 	llist_add_batch(llnode, c->waiting_for_gp_tail, &tgt->free_by_rcu_ttrace);  in __free_by_rcu()
377 	atomic_set(&c->call_rcu_in_progress, 0);  in __free_by_rcu()
380 static void check_free_by_rcu(struct bpf_mem_cache *c)  in check_free_by_rcu()  argument
386 	if (unlikely(!llist_empty(&c->free_llist_extra_rcu))) {  in check_free_by_rcu()
387 		inc_active(c, &flags);  in check_free_by_rcu()
388 		llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra_rcu))  in check_free_by_rcu()
389 			if (__llist_add(llnode, &c->free_by_rcu))  in check_free_by_rcu()
390 				c->free_by_rcu_tail = llnode;  in check_free_by_rcu()
391 		dec_active(c, &flags);  in check_free_by_rcu()
394 	if (llist_empty(&c->free_by_rcu))  in check_free_by_rcu()
397 	if (atomic_xchg(&c->call_rcu_in_progress, 1)) {  in check_free_by_rcu()
410 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));  in check_free_by_rcu()
412 	inc_active(c, &flags);  in check_free_by_rcu()
413 	WRITE_ONCE(c->waiting_for_gp.first, __llist_del_all(&c->free_by_rcu));  in check_free_by_rcu()
414 	c->waiting_for_gp_tail = c->free_by_rcu_tail;  in check_free_by_rcu()
415 	dec_active(c, &flags);  in check_free_by_rcu()
417 	if (unlikely(READ_ONCE(c->draining))) {  in check_free_by_rcu()
418 		free_all(llist_del_all(&c->waiting_for_gp), !!c->percpu_size);  in check_free_by_rcu()
419 		atomic_set(&c->call_rcu_in_progress, 0);  in check_free_by_rcu()
421 		call_rcu_hurry(&c->rcu, __free_by_rcu);  in check_free_by_rcu()
427 	struct bpf_mem_cache *c = container_of(work, struct bpf_mem_cache, refill_work);  in bpf_mem_refill()  local
431 	cnt = c->free_cnt;  in bpf_mem_refill()
432 	if (cnt < c->low_watermark)  in bpf_mem_refill()
436 		alloc_bulk(c, c->batch, NUMA_NO_NODE, true);  in bpf_mem_refill()
437 	else if (cnt > c->high_watermark)  in bpf_mem_refill()
438 		free_bulk(c);  in bpf_mem_refill()
440 	check_free_by_rcu(c);  in bpf_mem_refill()
443 static void notrace irq_work_raise(struct bpf_mem_cache *c)  in irq_work_raise()  argument
445 	irq_work_queue(&c->refill_work);  in irq_work_raise()
462 static void init_refill_work(struct bpf_mem_cache *c)  in init_refill_work()  argument
464 	init_irq_work(&c->refill_work, bpf_mem_refill);  in init_refill_work()
465 	if (c->unit_size <= 256) {  in init_refill_work()
466 		c->low_watermark = 32;  in init_refill_work()
467 		c->high_watermark = 96;  in init_refill_work()
474 		c->low_watermark = max(32 * 256 / c->unit_size, 1);  in init_refill_work()
475 		c->high_watermark = max(96 * 256 / c->unit_size, 3);  in init_refill_work()
477 	c->batch = max((c->high_watermark - c->low_watermark) / 4 * 3, 1);  in init_refill_work()
480 static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu)  in prefill_mem_cache()  argument
486 	alloc_bulk(c, c->unit_size <= 256 ? 4 : 1, cpu_to_node(cpu), false);  in prefill_mem_cache()
489 static int check_obj_size(struct bpf_mem_cache *c, unsigned int idx)  in check_obj_size()  argument
498 	if (c->percpu_size)  in check_obj_size()
501 	first = c->free_llist.first;  in check_obj_size()
506 	if (obj_size != c->unit_size) {  in check_obj_size()
508 			  idx, obj_size, c->unit_size);  in check_obj_size()
526 	struct bpf_mem_cache *c, __percpu *pc;  in bpf_mem_alloc_init()  local
546 			c = per_cpu_ptr(pc, cpu);  in bpf_mem_alloc_init()
547 			c->unit_size = unit_size;  in bpf_mem_alloc_init()
548 			c->objcg = objcg;  in bpf_mem_alloc_init()
549 			c->percpu_size = percpu_size;  in bpf_mem_alloc_init()
550 			c->tgt = c;  in bpf_mem_alloc_init()
551 			init_refill_work(c);  in bpf_mem_alloc_init()
552 			prefill_mem_cache(c, cpu);  in bpf_mem_alloc_init()
572 			c = &cc->cache[i];  in bpf_mem_alloc_init()
573 			c->unit_size = sizes[i];  in bpf_mem_alloc_init()
574 			c->objcg = objcg;  in bpf_mem_alloc_init()
575 			c->tgt = c;  in bpf_mem_alloc_init()
577 			init_refill_work(c);  in bpf_mem_alloc_init()
583 			if (i != bpf_mem_cache_idx(c->unit_size))  in bpf_mem_alloc_init()
585 			prefill_mem_cache(c, cpu);  in bpf_mem_alloc_init()
586 			err = check_obj_size(c, i);  in bpf_mem_alloc_init()
602 static void drain_mem_cache(struct bpf_mem_cache *c)  in drain_mem_cache()  argument
604 	bool percpu = !!c->percpu_size;  in drain_mem_cache()
613 	free_all(llist_del_all(&c->free_by_rcu_ttrace), percpu);  in drain_mem_cache()
614 	free_all(llist_del_all(&c->waiting_for_gp_ttrace), percpu);  in drain_mem_cache()
615 	free_all(__llist_del_all(&c->free_llist), percpu);  in drain_mem_cache()
616 	free_all(__llist_del_all(&c->free_llist_extra), percpu);  in drain_mem_cache()
617 	free_all(__llist_del_all(&c->free_by_rcu), percpu);  in drain_mem_cache()
618 	free_all(__llist_del_all(&c->free_llist_extra_rcu), percpu);  in drain_mem_cache()
619 	free_all(llist_del_all(&c->waiting_for_gp), percpu);  in drain_mem_cache()
622 static void check_mem_cache(struct bpf_mem_cache *c)  in check_mem_cache()  argument
624 	WARN_ON_ONCE(!llist_empty(&c->free_by_rcu_ttrace));  in check_mem_cache()
625 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace));  in check_mem_cache()
626 	WARN_ON_ONCE(!llist_empty(&c->free_llist));  in check_mem_cache()
627 	WARN_ON_ONCE(!llist_empty(&c->free_llist_extra));  in check_mem_cache()
628 	WARN_ON_ONCE(!llist_empty(&c->free_by_rcu));  in check_mem_cache()
629 	WARN_ON_ONCE(!llist_empty(&c->free_llist_extra_rcu));  in check_mem_cache()
630 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));  in check_mem_cache()
636 	struct bpf_mem_cache *c;  in check_leaked_objs()  local
641 			c = per_cpu_ptr(ma->cache, cpu);  in check_leaked_objs()
642 			check_mem_cache(c);  in check_leaked_objs()
649 				c = &cc->cache[i];  in check_leaked_objs()
650 				check_mem_cache(c);  in check_leaked_objs()
720 	struct bpf_mem_cache *c;  in bpf_mem_alloc_destroy()  local
726 			c = per_cpu_ptr(ma->cache, cpu);  in bpf_mem_alloc_destroy()
727 			WRITE_ONCE(c->draining, true);  in bpf_mem_alloc_destroy()
728 			irq_work_sync(&c->refill_work);  in bpf_mem_alloc_destroy()
729 			drain_mem_cache(c);  in bpf_mem_alloc_destroy()
730 			rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress);  in bpf_mem_alloc_destroy()
731 			rcu_in_progress += atomic_read(&c->call_rcu_in_progress);  in bpf_mem_alloc_destroy()
734 		if (c->objcg)  in bpf_mem_alloc_destroy()
735 			obj_cgroup_put(c->objcg);  in bpf_mem_alloc_destroy()
743 				c = &cc->cache[i];  in bpf_mem_alloc_destroy()
744 				WRITE_ONCE(c->draining, true);  in bpf_mem_alloc_destroy()
745 				irq_work_sync(&c->refill_work);  in bpf_mem_alloc_destroy()
746 				drain_mem_cache(c);  in bpf_mem_alloc_destroy()
747 				rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress);  in bpf_mem_alloc_destroy()
748 				rcu_in_progress += atomic_read(&c->call_rcu_in_progress);  in bpf_mem_alloc_destroy()
751 		if (c->objcg)  in bpf_mem_alloc_destroy()
752 			obj_cgroup_put(c->objcg);  in bpf_mem_alloc_destroy()
760 static void notrace *unit_alloc(struct bpf_mem_cache *c)  in unit_alloc()  argument
777 	if (local_inc_return(&c->active) == 1) {  in unit_alloc()
778 		llnode = __llist_del_first(&c->free_llist);  in unit_alloc()
780 			cnt = --c->free_cnt;  in unit_alloc()
781 			*(struct bpf_mem_cache **)llnode = c;  in unit_alloc()
784 	local_dec(&c->active);  in unit_alloc()
789 	if (cnt < c->low_watermark)  in unit_alloc()
790 		irq_work_raise(c);  in unit_alloc()
798 static void notrace unit_free(struct bpf_mem_cache *c, void *ptr)  in unit_free()  argument
810 	c->tgt = *(struct bpf_mem_cache **)llnode;  in unit_free()
813 	if (local_inc_return(&c->active) == 1) {  in unit_free()
814 		__llist_add(llnode, &c->free_llist);  in unit_free()
815 		cnt = ++c->free_cnt;  in unit_free()
823 		llist_add(llnode, &c->free_llist_extra);  in unit_free()
825 	local_dec(&c->active);  in unit_free()
828 	if (cnt > c->high_watermark)  in unit_free()
830 		irq_work_raise(c);  in unit_free()
833 static void notrace unit_free_rcu(struct bpf_mem_cache *c, void *ptr)  in unit_free_rcu()  argument
838 	c->tgt = *(struct bpf_mem_cache **)llnode;  in unit_free_rcu()
841 	if (local_inc_return(&c->active) == 1) {  in unit_free_rcu()
842 		if (__llist_add(llnode, &c->free_by_rcu))  in unit_free_rcu()
843 			c->free_by_rcu_tail = llnode;  in unit_free_rcu()
845 		llist_add(llnode, &c->free_llist_extra_rcu);  in unit_free_rcu()
847 	local_dec(&c->active);  in unit_free_rcu()
850 	if (!atomic_read(&c->call_rcu_in_progress))  in unit_free_rcu()
851 		irq_work_raise(c);  in unit_free_rcu()
949 	struct bpf_mem_cache *c;  in bpf_mem_cache_alloc_flags()  local
952 	c = this_cpu_ptr(ma->cache);  in bpf_mem_cache_alloc_flags()
954 	ret = unit_alloc(c);  in bpf_mem_cache_alloc_flags()
958 		memcg = get_memcg(c);  in bpf_mem_cache_alloc_flags()
960 		ret = __alloc(c, NUMA_NO_NODE, GFP_KERNEL | __GFP_NOWARN | __GFP_ACCOUNT);  in bpf_mem_cache_alloc_flags()