97 	fp->aux->prog = fp;  in bpf_prog_alloc_no_stats()
110 	struct bpf_prog *prog;  in bpf_prog_alloc()  local
113 	prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags);  in bpf_prog_alloc()
114 	if (!prog)  in bpf_prog_alloc()
117 	prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);  in bpf_prog_alloc()
118 	if (!prog->aux->stats) {  in bpf_prog_alloc()
119 		kfree(prog->aux);  in bpf_prog_alloc()
120 		vfree(prog);  in bpf_prog_alloc()
127 		pstats = per_cpu_ptr(prog->aux->stats, cpu);  in bpf_prog_alloc()
130 	return prog;  in bpf_prog_alloc()
134 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)  in bpf_prog_alloc_jited_linfo()  argument
136 	if (!prog->aux->nr_linfo || !prog->jit_requested)  in bpf_prog_alloc_jited_linfo()
139 	prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,  in bpf_prog_alloc_jited_linfo()
140 					 sizeof(*prog->aux->jited_linfo),  in bpf_prog_alloc_jited_linfo()
142 	if (!prog->aux->jited_linfo)  in bpf_prog_alloc_jited_linfo()
148 void bpf_prog_free_jited_linfo(struct bpf_prog *prog)  in bpf_prog_free_jited_linfo()  argument
150 	kfree(prog->aux->jited_linfo);  in bpf_prog_free_jited_linfo()
151 	prog->aux->jited_linfo = NULL;  in bpf_prog_free_jited_linfo()
154 void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)  in bpf_prog_free_unused_jited_linfo()  argument
156 	if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])  in bpf_prog_free_unused_jited_linfo()
157 		bpf_prog_free_jited_linfo(prog);  in bpf_prog_free_unused_jited_linfo()
184 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,  in bpf_prog_fill_jited_linfo()  argument
191 	if (!prog->aux->jited_linfo)  in bpf_prog_fill_jited_linfo()
195 	linfo_idx = prog->aux->linfo_idx;  in bpf_prog_fill_jited_linfo()
196 	linfo = &prog->aux->linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
198 	insn_end = insn_start + prog->len;  in bpf_prog_fill_jited_linfo()
200 	jited_linfo = &prog->aux->jited_linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
201 	jited_linfo[0] = prog->bpf_func;  in bpf_prog_fill_jited_linfo()
203 	nr_linfo = prog->aux->nr_linfo - linfo_idx;  in bpf_prog_fill_jited_linfo()
209 		jited_linfo[i] = prog->bpf_func +  in bpf_prog_fill_jited_linfo()
213 void bpf_prog_free_linfo(struct bpf_prog *prog)  in bpf_prog_free_linfo()  argument
215 	bpf_prog_free_jited_linfo(prog);  in bpf_prog_free_linfo()
216 	kvfree(prog->aux->linfo);  in bpf_prog_free_linfo()
243 		fp->aux->prog = fp;  in bpf_prog_realloc()
376 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old,  in bpf_adj_branches()  argument
379 	u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0);  in bpf_adj_branches()
380 	struct bpf_insn *insn = prog->insnsi;  in bpf_adj_branches()
392 			insn = prog->insnsi + end_old;  in bpf_adj_branches()
416 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)  in bpf_adj_linfo()  argument
421 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo()
425 	linfo = prog->aux->linfo;  in bpf_adj_linfo()
436 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,  in bpf_patch_insn_single()  argument
446 		memcpy(prog->insnsi + off, patch, sizeof(*patch));  in bpf_patch_insn_single()
447 		return prog;  in bpf_patch_insn_single()
450 	insn_adj_cnt = prog->len + insn_delta;  in bpf_patch_insn_single()
458 	    (err = bpf_adj_branches(prog, off, off + 1, off + len, true)))  in bpf_patch_insn_single()
465 	prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),  in bpf_patch_insn_single()
497 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt)  in bpf_remove_insns()  argument
502 	memmove(prog->insnsi + off, prog->insnsi + off + cnt,  in bpf_remove_insns()
503 		sizeof(struct bpf_insn) * (prog->len - off - cnt));  in bpf_remove_insns()
504 	prog->len -= cnt;  in bpf_remove_insns()
506 	return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false));  in bpf_remove_insns()
531 bpf_prog_ksym_set_addr(struct bpf_prog *prog)  in bpf_prog_ksym_set_addr()  argument
533 	const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog);  in bpf_prog_ksym_set_addr()
536 	WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog));  in bpf_prog_ksym_set_addr()
538 	prog->aux->ksym.start = (unsigned long) prog->bpf_func;  in bpf_prog_ksym_set_addr()
539 	prog->aux->ksym.end   = addr + hdr->pages * PAGE_SIZE;  in bpf_prog_ksym_set_addr()
543 bpf_prog_ksym_set_name(struct bpf_prog *prog)  in bpf_prog_ksym_set_name()  argument
545 	char *sym = prog->aux->ksym.name;  in bpf_prog_ksym_set_name()
551 		     sizeof(prog->tag) * 2 +  in bpf_prog_ksym_set_name()
559 		     sizeof(prog->aux->name) > KSYM_NAME_LEN);  in bpf_prog_ksym_set_name()
562 	sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));  in bpf_prog_ksym_set_name()
565 	if (prog->aux->func_info_cnt) {  in bpf_prog_ksym_set_name()
566 		type = btf_type_by_id(prog->aux->btf,  in bpf_prog_ksym_set_name()
567 				      prog->aux->func_info[prog->aux->func_idx].type_id);  in bpf_prog_ksym_set_name()
568 		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);  in bpf_prog_ksym_set_name()
573 	if (prog->aux->name[0])  in bpf_prog_ksym_set_name()
574 		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);  in bpf_prog_ksym_set_name()
658 	fp->aux->ksym.prog = true;  in bpf_prog_kallsyms_add()
719 	return ksym && ksym->prog ?  in bpf_prog_ksym_find()
720 	       container_of(ksym, struct bpf_prog_aux, ksym)->prog :  in bpf_prog_ksym_find()
727 	struct bpf_prog *prog;  in search_bpf_extables()  local
730 	prog = bpf_prog_ksym_find(addr);  in search_bpf_extables()
731 	if (!prog)  in search_bpf_extables()
733 	if (!prog->aux->num_exentries)  in search_bpf_extables()
736 	e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);  in search_bpf_extables()
770 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,  in bpf_jit_add_poke_descriptor()  argument
773 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in bpf_jit_add_poke_descriptor()
775 	u32 slot = prog->aux->size_poke_tab;  in bpf_jit_add_poke_descriptor()
798 	prog->aux->size_poke_tab = size;  in bpf_jit_add_poke_descriptor()
799 	prog->aux->poke_tab = tab;  in bpf_jit_add_poke_descriptor()
921 int bpf_jit_get_func_addr(const struct bpf_prog *prog,  in bpf_jit_get_func_addr()  argument
937 		else if (prog->aux->func &&  in bpf_jit_get_func_addr()
938 			 off >= 0 && off < prog->aux->func_cnt)  in bpf_jit_get_func_addr()
939 			addr = (u8 *)prog->aux->func[off]->bpf_func;  in bpf_jit_get_func_addr()
1127 	fp->aux->prog = fp;  in bpf_jit_prog_release_other()
1131 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)  in bpf_jit_blind_constants()  argument
1139 	if (!bpf_jit_blinding_enabled(prog) || prog->blinded)  in bpf_jit_blind_constants()
1140 		return prog;  in bpf_jit_blind_constants()
1142 	clone = bpf_prog_clone_create(prog, GFP_USER);  in bpf_jit_blind_constants()
1169 			bpf_jit_prog_release_other(prog, clone);  in bpf_jit_blind_constants()
1535 		struct bpf_prog *prog;  in ___bpf_prog_run()  local
1545 		prog = READ_ONCE(array->ptrs[index]);  in ___bpf_prog_run()
1546 		if (!prog)  in ___bpf_prog_run()
1554 		insn = prog->insnsi;  in ___bpf_prog_run()
1854 	struct bpf_prog prog;  member
1856 	.prog = {
1897 	for (item = array->items; item->prog; item++)  in bpf_prog_array_length()
1898 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_length()
1907 	for (item = array->items; item->prog; item++)  in bpf_prog_array_is_empty()
1908 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_is_empty()
1920 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_copy_core()
1921 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy_core()
1923 		prog_ids[i] = item->prog->aux->id;  in bpf_prog_array_copy_core()
1930 	return !!(item->prog);  in bpf_prog_array_copy_core()
1964 	for (item = array->items; item->prog; item++)  in bpf_prog_array_delete_safe()
1965 		if (item->prog == old_prog) {  in bpf_prog_array_delete_safe()
1966 			WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe()
1988 	return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe_at()
2007 			     struct bpf_prog *prog)  in bpf_prog_array_update_at()  argument
2014 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_update_at()
2015 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_update_at()
2018 			WRITE_ONCE(item->prog, prog);  in bpf_prog_array_update_at()
2042 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
2043 			if (existing->prog == exclude_prog) {  in bpf_prog_array_copy()
2047 			if (existing->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
2049 			if (existing->prog == include_prog)  in bpf_prog_array_copy()
2076 		for (; existing->prog; existing++)  in bpf_prog_array_copy()
2077 			if (existing->prog != exclude_prog &&  in bpf_prog_array_copy()
2078 			    existing->prog != &dummy_bpf_prog.prog) {  in bpf_prog_array_copy()
2079 				array->items[new_prog_idx++].prog =  in bpf_prog_array_copy()
2080 					existing->prog;  in bpf_prog_array_copy()
2084 		array->items[new_prog_idx++].prog = include_prog;  in bpf_prog_array_copy()
2085 	array->items[new_prog_idx].prog = NULL;  in bpf_prog_array_copy()
2138 		bpf_prog_offload_destroy(aux->prog);  in bpf_prog_free_deferred()
2140 	if (aux->prog->has_callchain_buf)  in bpf_prog_free_deferred()
2149 		bpf_prog_unlock_free(aux->prog);  in bpf_prog_free_deferred()
2151 		bpf_jit_free(aux->prog);  in bpf_prog_free_deferred()
2252 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
2254 	return prog;  in bpf_int_jit_compile()
2260 void __weak bpf_jit_compile(struct bpf_prog *prog)  in bpf_jit_compile()  argument