521 int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)  in bpf_mem_alloc_init()  argument
554 		ma->cache = pc;  in bpf_mem_alloc_init()
593 	ma->caches = pcc;  in bpf_mem_alloc_init()
598 		bpf_mem_alloc_destroy(ma);  in bpf_mem_alloc_init()
633 static void check_leaked_objs(struct bpf_mem_alloc *ma)  in check_leaked_objs()  argument
639 	if (ma->cache) {  in check_leaked_objs()
641 			c = per_cpu_ptr(ma->cache, cpu);  in check_leaked_objs()
645 	if (ma->caches) {  in check_leaked_objs()
647 			cc = per_cpu_ptr(ma->caches, cpu);  in check_leaked_objs()
656 static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)  in free_mem_alloc_no_barrier()  argument
658 	check_leaked_objs(ma);  in free_mem_alloc_no_barrier()
659 	free_percpu(ma->cache);  in free_mem_alloc_no_barrier()
660 	free_percpu(ma->caches);  in free_mem_alloc_no_barrier()
661 	ma->cache = NULL;  in free_mem_alloc_no_barrier()
662 	ma->caches = NULL;  in free_mem_alloc_no_barrier()
665 static void free_mem_alloc(struct bpf_mem_alloc *ma)  in free_mem_alloc()  argument
681 	free_mem_alloc_no_barrier(ma);  in free_mem_alloc()
686 	struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work);  in free_mem_alloc_deferred()  local
688 	free_mem_alloc(ma);  in free_mem_alloc_deferred()
689 	kfree(ma);  in free_mem_alloc_deferred()
692 static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress)  in destroy_mem_alloc()  argument
700 		free_mem_alloc_no_barrier(ma);  in destroy_mem_alloc()
704 	copy = kmemdup(ma, sizeof(*ma), GFP_KERNEL);  in destroy_mem_alloc()
707 		free_mem_alloc(ma);  in destroy_mem_alloc()
712 	memset(ma, 0, sizeof(*ma));  in destroy_mem_alloc()
717 void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma)  in bpf_mem_alloc_destroy()  argument
723 	if (ma->cache) {  in bpf_mem_alloc_destroy()
726 			c = per_cpu_ptr(ma->cache, cpu);  in bpf_mem_alloc_destroy()
736 		destroy_mem_alloc(ma, rcu_in_progress);  in bpf_mem_alloc_destroy()
738 	if (ma->caches) {  in bpf_mem_alloc_destroy()
741 			cc = per_cpu_ptr(ma->caches, cpu);  in bpf_mem_alloc_destroy()
753 		destroy_mem_alloc(ma, rcu_in_progress);  in bpf_mem_alloc_destroy()
857 void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)  in bpf_mem_alloc()  argument
869 	ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx);  in bpf_mem_alloc()
873 void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_free()  argument
884 	unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr);  in bpf_mem_free()
887 void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_free_rcu()  argument
898 	unit_free_rcu(this_cpu_ptr(ma->caches)->cache + idx, ptr);  in bpf_mem_free_rcu()
901 void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma)  in bpf_mem_cache_alloc()  argument
905 	ret = unit_alloc(this_cpu_ptr(ma->cache));  in bpf_mem_cache_alloc()
909 void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_cache_free()  argument
914 	unit_free(this_cpu_ptr(ma->cache), ptr);  in bpf_mem_cache_free()
917 void notrace bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_cache_free_rcu()  argument
922 	unit_free_rcu(this_cpu_ptr(ma->cache), ptr);  in bpf_mem_cache_free_rcu()
947 void notrace *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags)  in bpf_mem_cache_alloc_flags()  argument
952 	c = this_cpu_ptr(ma->cache);  in bpf_mem_cache_alloc_flags()