81 	struct bpf_prog_aux *aux;  in bpf_prog_alloc_no_stats()  local
89 	aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags);  in bpf_prog_alloc_no_stats()
90 	if (aux == NULL) {  in bpf_prog_alloc_no_stats()
96 	fp->aux = aux;  in bpf_prog_alloc_no_stats()
97 	fp->aux->prog = fp;  in bpf_prog_alloc_no_stats()
100 	INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode);  in bpf_prog_alloc_no_stats()
101 	mutex_init(&fp->aux->used_maps_mutex);  in bpf_prog_alloc_no_stats()
102 	mutex_init(&fp->aux->dst_mutex);  in bpf_prog_alloc_no_stats()
117 	prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);  in bpf_prog_alloc()
118 	if (!prog->aux->stats) {  in bpf_prog_alloc()
119 		kfree(prog->aux);  in bpf_prog_alloc()
127 		pstats = per_cpu_ptr(prog->aux->stats, cpu);  in bpf_prog_alloc()
136 	if (!prog->aux->nr_linfo || !prog->jit_requested)  in bpf_prog_alloc_jited_linfo()
139 	prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,  in bpf_prog_alloc_jited_linfo()
140 					 sizeof(*prog->aux->jited_linfo),  in bpf_prog_alloc_jited_linfo()
142 	if (!prog->aux->jited_linfo)  in bpf_prog_alloc_jited_linfo()
150 	kfree(prog->aux->jited_linfo);  in bpf_prog_free_jited_linfo()
151 	prog->aux->jited_linfo = NULL;  in bpf_prog_free_jited_linfo()
156 	if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])  in bpf_prog_free_unused_jited_linfo()
175  *      The prog's idx to prog->aux->linfo and jited_linfo
191 	if (!prog->aux->jited_linfo)  in bpf_prog_fill_jited_linfo()
195 	linfo_idx = prog->aux->linfo_idx;  in bpf_prog_fill_jited_linfo()
196 	linfo = &prog->aux->linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
200 	jited_linfo = &prog->aux->jited_linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
203 	nr_linfo = prog->aux->nr_linfo - linfo_idx;  in bpf_prog_fill_jited_linfo()
216 	kvfree(prog->aux->linfo);  in bpf_prog_free_linfo()
233 	ret = __bpf_prog_charge(fp_old->aux->user, delta);  in bpf_prog_realloc()
239 		__bpf_prog_uncharge(fp_old->aux->user, delta);  in bpf_prog_realloc()
243 		fp->aux->prog = fp;  in bpf_prog_realloc()
245 		/* We keep fp->aux from fp_old around in the new  in bpf_prog_realloc()
248 		fp_old->aux = NULL;  in bpf_prog_realloc()
257 	if (fp->aux) {  in __bpf_prog_free()
258 		mutex_destroy(&fp->aux->used_maps_mutex);  in __bpf_prog_free()
259 		mutex_destroy(&fp->aux->dst_mutex);  in __bpf_prog_free()
260 		free_percpu(fp->aux->stats);  in __bpf_prog_free()
261 		kfree(fp->aux->poke_tab);  in __bpf_prog_free()
262 		kfree(fp->aux);  in __bpf_prog_free()
421 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo()
425 	linfo = prog->aux->linfo;  in bpf_adj_linfo()
513 	for (i = 0; i < fp->aux->func_cnt; i++)  in bpf_prog_kallsyms_del_subprogs()
514 		bpf_prog_kallsyms_del(fp->aux->func[i]);  in bpf_prog_kallsyms_del_subprogs()
538 	prog->aux->ksym.start = (unsigned long) prog->bpf_func;  in bpf_prog_ksym_set_addr()
539 	prog->aux->ksym.end   = addr + hdr->pages * PAGE_SIZE;  in bpf_prog_ksym_set_addr()
545 	char *sym = prog->aux->ksym.name;  in bpf_prog_ksym_set_name()
559 		     sizeof(prog->aux->name) > KSYM_NAME_LEN);  in bpf_prog_ksym_set_name()
564 	/* prog->aux->name will be ignored if full btf name is available */  in bpf_prog_ksym_set_name()
565 	if (prog->aux->func_info_cnt) {  in bpf_prog_ksym_set_name()
566 		type = btf_type_by_id(prog->aux->btf,  in bpf_prog_ksym_set_name()
567 				      prog->aux->func_info[prog->aux->func_idx].type_id);  in bpf_prog_ksym_set_name()
568 		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);  in bpf_prog_ksym_set_name()
573 	if (prog->aux->name[0])  in bpf_prog_ksym_set_name()
574 		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);  in bpf_prog_ksym_set_name()
646 	return list_empty(&fp->aux->ksym.lnode) ||  in bpf_prog_kallsyms_verify_off()
647 	       fp->aux->ksym.lnode.prev == LIST_POISON2;  in bpf_prog_kallsyms_verify_off()
658 	fp->aux->ksym.prog = true;  in bpf_prog_kallsyms_add()
660 	bpf_ksym_add(&fp->aux->ksym);  in bpf_prog_kallsyms_add()
668 	bpf_ksym_del(&fp->aux->ksym);  in bpf_prog_kallsyms_del()
733 	if (!prog->aux->num_exentries)  in search_bpf_extables()
736 	e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);  in search_bpf_extables()
773 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in bpf_jit_add_poke_descriptor()
775 	u32 slot = prog->aux->size_poke_tab;  in bpf_jit_add_poke_descriptor()
798 	prog->aux->size_poke_tab = size;  in bpf_jit_add_poke_descriptor()
799 	prog->aux->poke_tab = tab;  in bpf_jit_add_poke_descriptor()
933 		 * can pick them up from prog->aux.  in bpf_jit_get_func_addr()
937 		else if (prog->aux->func &&  in bpf_jit_get_func_addr()
938 			 off >= 0 && off < prog->aux->func_cnt)  in bpf_jit_get_func_addr()
939 			addr = (u8 *)prog->aux->func[off]->bpf_func;  in bpf_jit_get_func_addr()
956 			      const struct bpf_insn *aux,  in bpf_jit_blind_insn()  argument
1065 		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm);  in bpf_jit_blind_insn()
1068 		*to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX);  in bpf_jit_blind_insn()
1071 		*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm);  in bpf_jit_blind_insn()
1075 		*to++ = BPF_ALU64_REG(BPF_OR,  aux[0].dst_reg, BPF_REG_AX);  in bpf_jit_blind_insn()
1099 		/* aux->prog still points to the fp_other one, so  in bpf_prog_clone_create()
1111 	/* aux was stolen by the other clone, so we cannot free  in bpf_prog_clone_free()
1118 	fp->aux = NULL;  in bpf_prog_clone_free()
1124 	/* We have to repoint aux->prog to self, as we don't  in bpf_jit_prog_release_other()
1127 	fp->aux->prog = fp;  in bpf_jit_prog_release_other()
1133 	struct bpf_insn insn_buff[16], aux[2];  in bpf_jit_blind_constants()  local
1156 			memcpy(aux, insn, sizeof(aux));  in bpf_jit_blind_constants()
1158 		rewritten = bpf_jit_blind_insn(insn, aux, insn_buff,  in bpf_jit_blind_constants()
1159 						clone->aux->verifier_zext);  in bpf_jit_blind_constants()
1165 			/* Patching may have repointed aux->prog during  in bpf_jit_blind_constants()
1739 	if (!array->aux->type) {  in bpf_prog_array_compatible()
1743 		array->aux->type  = fp->type;  in bpf_prog_array_compatible()
1744 		array->aux->jited = fp->jited;  in bpf_prog_array_compatible()
1748 	return array->aux->type  == fp->type &&  in bpf_prog_array_compatible()
1749 	       array->aux->jited == fp->jited;  in bpf_prog_array_compatible()
1754 	struct bpf_prog_aux *aux = fp->aux;  in bpf_check_tail_call()  local
1757 	mutex_lock(&aux->used_maps_mutex);  in bpf_check_tail_call()
1758 	for (i = 0; i < aux->used_map_cnt; i++) {  in bpf_check_tail_call()
1759 		struct bpf_map *map = aux->used_maps[i];  in bpf_check_tail_call()
1773 	mutex_unlock(&aux->used_maps_mutex);  in bpf_check_tail_call()
1780 	u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);  in bpf_prog_select_func()
1812 	if (!bpf_prog_is_dev_bound(fp->aux)) {  in bpf_prog_select_runtime()
1923 		prog_ids[i] = item->prog->aux->id;  in bpf_prog_array_copy_core()
2110 void __bpf_free_used_maps(struct bpf_prog_aux *aux,  in __bpf_free_used_maps()  argument
2119 			map->ops->map_poke_untrack(map, aux);  in __bpf_free_used_maps()
2124 static void bpf_free_used_maps(struct bpf_prog_aux *aux)  in bpf_free_used_maps()  argument
2126 	__bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt);  in bpf_free_used_maps()
2127 	kfree(aux->used_maps);  in bpf_free_used_maps()
2132 	struct bpf_prog_aux *aux;  in bpf_prog_free_deferred()  local
2135 	aux = container_of(work, struct bpf_prog_aux, work);  in bpf_prog_free_deferred()
2136 	bpf_free_used_maps(aux);  in bpf_prog_free_deferred()
2137 	if (bpf_prog_is_dev_bound(aux))  in bpf_prog_free_deferred()
2138 		bpf_prog_offload_destroy(aux->prog);  in bpf_prog_free_deferred()
2140 	if (aux->prog->has_callchain_buf)  in bpf_prog_free_deferred()
2143 	if (aux->dst_trampoline)  in bpf_prog_free_deferred()
2144 		bpf_trampoline_put(aux->dst_trampoline);  in bpf_prog_free_deferred()
2145 	for (i = 0; i < aux->func_cnt; i++)  in bpf_prog_free_deferred()
2146 		bpf_jit_free(aux->func[i]);  in bpf_prog_free_deferred()
2147 	if (aux->func_cnt) {  in bpf_prog_free_deferred()
2148 		kfree(aux->func);  in bpf_prog_free_deferred()
2149 		bpf_prog_unlock_free(aux->prog);  in bpf_prog_free_deferred()
2151 		bpf_jit_free(aux->prog);  in bpf_prog_free_deferred()
2158 	struct bpf_prog_aux *aux = fp->aux;  in bpf_prog_free()  local
2160 	if (aux->dst_prog)  in bpf_prog_free()
2161 		bpf_prog_put(aux->dst_prog);  in bpf_prog_free()
2162 	INIT_WORK(&aux->work, bpf_prog_free_deferred);  in bpf_prog_free()
2163 	schedule_work(&aux->work);  in bpf_prog_free()