105 	fp->aux->prog = fp;  in bpf_prog_alloc_no_stats()
118 	struct bpf_prog *prog;  in bpf_prog_alloc()  local
121 	prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags);  in bpf_prog_alloc()
122 	if (!prog)  in bpf_prog_alloc()
125 	prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);  in bpf_prog_alloc()
126 	if (!prog->stats) {  in bpf_prog_alloc()
127 		free_percpu(prog->active);  in bpf_prog_alloc()
128 		kfree(prog->aux);  in bpf_prog_alloc()
129 		vfree(prog);  in bpf_prog_alloc()
136 		pstats = per_cpu_ptr(prog->stats, cpu);  in bpf_prog_alloc()
139 	return prog;  in bpf_prog_alloc()
143 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)  in bpf_prog_alloc_jited_linfo()  argument
145 	if (!prog->aux->nr_linfo || !prog->jit_requested)  in bpf_prog_alloc_jited_linfo()
148 	prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,  in bpf_prog_alloc_jited_linfo()
149 					  sizeof(*prog->aux->jited_linfo),  in bpf_prog_alloc_jited_linfo()
151 	if (!prog->aux->jited_linfo)  in bpf_prog_alloc_jited_linfo()
157 void bpf_prog_jit_attempt_done(struct bpf_prog *prog)  in bpf_prog_jit_attempt_done()  argument
159 	if (prog->aux->jited_linfo &&  in bpf_prog_jit_attempt_done()
160 	    (!prog->jited || !prog->aux->jited_linfo[0])) {  in bpf_prog_jit_attempt_done()
161 		kvfree(prog->aux->jited_linfo);  in bpf_prog_jit_attempt_done()
162 		prog->aux->jited_linfo = NULL;  in bpf_prog_jit_attempt_done()
165 	kfree(prog->aux->kfunc_tab);  in bpf_prog_jit_attempt_done()
166 	prog->aux->kfunc_tab = NULL;  in bpf_prog_jit_attempt_done()
193 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,  in bpf_prog_fill_jited_linfo()  argument
200 	if (!prog->aux->jited_linfo)  in bpf_prog_fill_jited_linfo()
204 	linfo_idx = prog->aux->linfo_idx;  in bpf_prog_fill_jited_linfo()
205 	linfo = &prog->aux->linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
207 	insn_end = insn_start + prog->len;  in bpf_prog_fill_jited_linfo()
209 	jited_linfo = &prog->aux->jited_linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
210 	jited_linfo[0] = prog->bpf_func;  in bpf_prog_fill_jited_linfo()
212 	nr_linfo = prog->aux->nr_linfo - linfo_idx;  in bpf_prog_fill_jited_linfo()
218 		jited_linfo[i] = prog->bpf_func +  in bpf_prog_fill_jited_linfo()
238 		fp->aux->prog = fp;  in bpf_prog_realloc()
374 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old,  in bpf_adj_branches()  argument
377 	u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0);  in bpf_adj_branches()
378 	struct bpf_insn *insn = prog->insnsi;  in bpf_adj_branches()
390 			insn = prog->insnsi + end_old;  in bpf_adj_branches()
414 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)  in bpf_adj_linfo()  argument
419 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo()
423 	linfo = prog->aux->linfo;  in bpf_adj_linfo()
434 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,  in bpf_patch_insn_single()  argument
444 		memcpy(prog->insnsi + off, patch, sizeof(*patch));  in bpf_patch_insn_single()
445 		return prog;  in bpf_patch_insn_single()
448 	insn_adj_cnt = prog->len + insn_delta;  in bpf_patch_insn_single()
456 	    (err = bpf_adj_branches(prog, off, off + 1, off + len, true)))  in bpf_patch_insn_single()
463 	prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),  in bpf_patch_insn_single()
495 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt)  in bpf_remove_insns()  argument
500 	memmove(prog->insnsi + off, prog->insnsi + off + cnt,  in bpf_remove_insns()
501 		sizeof(struct bpf_insn) * (prog->len - off - cnt));  in bpf_remove_insns()
502 	prog->len -= cnt;  in bpf_remove_insns()
504 	return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false));  in bpf_remove_insns()
530 bpf_prog_ksym_set_addr(struct bpf_prog *prog)  in bpf_prog_ksym_set_addr()  argument
532 	const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog);  in bpf_prog_ksym_set_addr()
535 	WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog));  in bpf_prog_ksym_set_addr()
537 	prog->aux->ksym.start = (unsigned long) prog->bpf_func;  in bpf_prog_ksym_set_addr()
538 	prog->aux->ksym.end   = addr + hdr->pages * PAGE_SIZE;  in bpf_prog_ksym_set_addr()
542 bpf_prog_ksym_set_name(struct bpf_prog *prog)  in bpf_prog_ksym_set_name()  argument
544 	char *sym = prog->aux->ksym.name;  in bpf_prog_ksym_set_name()
550 		     sizeof(prog->tag) * 2 +  in bpf_prog_ksym_set_name()
558 		     sizeof(prog->aux->name) > KSYM_NAME_LEN);  in bpf_prog_ksym_set_name()
561 	sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));  in bpf_prog_ksym_set_name()
564 	if (prog->aux->func_info_cnt) {  in bpf_prog_ksym_set_name()
565 		type = btf_type_by_id(prog->aux->btf,  in bpf_prog_ksym_set_name()
566 				      prog->aux->func_info[prog->aux->func_idx].type_id);  in bpf_prog_ksym_set_name()
567 		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);  in bpf_prog_ksym_set_name()
572 	if (prog->aux->name[0])  in bpf_prog_ksym_set_name()
573 		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);  in bpf_prog_ksym_set_name()
657 	fp->aux->ksym.prog = true;  in bpf_prog_kallsyms_add()
718 	return ksym && ksym->prog ?  in bpf_prog_ksym_find()
719 	       container_of(ksym, struct bpf_prog_aux, ksym)->prog :  in bpf_prog_ksym_find()
726 	struct bpf_prog *prog;  in search_bpf_extables()  local
729 	prog = bpf_prog_ksym_find(addr);  in search_bpf_extables()
730 	if (!prog)  in search_bpf_extables()
732 	if (!prog->aux->num_exentries)  in search_bpf_extables()
735 	e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);  in search_bpf_extables()
769 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,  in bpf_jit_add_poke_descriptor()  argument
772 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in bpf_jit_add_poke_descriptor()
774 	u32 slot = prog->aux->size_poke_tab;  in bpf_jit_add_poke_descriptor()
797 	prog->aux->size_poke_tab = size;  in bpf_jit_add_poke_descriptor()
798 	prog->aux->poke_tab = tab;  in bpf_jit_add_poke_descriptor()
921 int bpf_jit_get_func_addr(const struct bpf_prog *prog,  in bpf_jit_get_func_addr()  argument
937 		else if (prog->aux->func &&  in bpf_jit_get_func_addr()
938 			 off >= 0 && off < prog->aux->func_cnt)  in bpf_jit_get_func_addr()
939 			addr = (u8 *)prog->aux->func[off]->bpf_func;  in bpf_jit_get_func_addr()
1129 	fp->aux->prog = fp;  in bpf_jit_prog_release_other()
1133 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)  in bpf_jit_blind_constants()  argument
1141 	if (!bpf_jit_blinding_enabled(prog) || prog->blinded)  in bpf_jit_blind_constants()
1142 		return prog;  in bpf_jit_blind_constants()
1144 	clone = bpf_prog_clone_create(prog, GFP_USER);  in bpf_jit_blind_constants()
1171 			bpf_jit_prog_release_other(prog, clone);  in bpf_jit_blind_constants()
1564 		struct bpf_prog *prog;  in ___bpf_prog_run()  local
1574 		prog = READ_ONCE(array->ptrs[index]);  in ___bpf_prog_run()
1575 		if (!prog)  in ___bpf_prog_run()
1583 		insn = prog->insnsi;  in ___bpf_prog_run()
1955 	struct bpf_prog prog;  member
1957 	.prog = {
1998 	for (item = array->items; item->prog; item++)  in bpf_prog_array_length()
1999 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_length()
2008 	for (item = array->items; item->prog; item++)  in bpf_prog_array_is_empty()
2009 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_is_empty()
2021 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_copy_core()
2022 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy_core()
2024 		prog_ids[i] = item->prog->aux->id;  in bpf_prog_array_copy_core()
2031 	return !!(item->prog);  in bpf_prog_array_copy_core()
2065 	for (item = array->items; item->prog; item++)  in bpf_prog_array_delete_safe()
2066 		if (item->prog == old_prog) {  in bpf_prog_array_delete_safe()
2067 			WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe()
2089 	return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe_at()
2108 			     struct bpf_prog *prog)  in bpf_prog_array_update_at()  argument
2115 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_update_at()
2116 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_update_at()
2119 			WRITE_ONCE(item->prog, prog);  in bpf_prog_array_update_at()
2143 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
2144 			if (existing->prog == exclude_prog) {  in bpf_prog_array_copy()
2148 			if (existing->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
2150 			if (existing->prog == include_prog)  in bpf_prog_array_copy()
2178 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
2179 			if (existing->prog == exclude_prog ||  in bpf_prog_array_copy()
2180 			    existing->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
2183 			new->prog = existing->prog;  in bpf_prog_array_copy()
2189 		new->prog = include_prog;  in bpf_prog_array_copy()
2193 	new->prog = NULL;  in bpf_prog_array_copy()
2269 		bpf_prog_offload_destroy(aux->prog);  in bpf_prog_free_deferred()
2271 	if (aux->prog->has_callchain_buf)  in bpf_prog_free_deferred()
2286 		bpf_prog_unlock_free(aux->prog);  in bpf_prog_free_deferred()
2288 		bpf_jit_free(aux->prog);  in bpf_prog_free_deferred()
2390 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
2392 	return prog;  in bpf_int_jit_compile()
2398 void __weak bpf_jit_compile(struct bpf_prog *prog)  in bpf_jit_compile()  argument