116 	fp->aux->prog = fp;  in bpf_prog_alloc_no_stats()
133 	struct bpf_prog *prog;  in bpf_prog_alloc()  local
136 	prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags);  in bpf_prog_alloc()
137 	if (!prog)  in bpf_prog_alloc()
140 	prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);  in bpf_prog_alloc()
141 	if (!prog->stats) {  in bpf_prog_alloc()
142 		free_percpu(prog->active);  in bpf_prog_alloc()
143 		kfree(prog->aux);  in bpf_prog_alloc()
144 		vfree(prog);  in bpf_prog_alloc()
151 		pstats = per_cpu_ptr(prog->stats, cpu);  in bpf_prog_alloc()
154 	return prog;  in bpf_prog_alloc()
158 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)  in bpf_prog_alloc_jited_linfo()  argument
160 	if (!prog->aux->nr_linfo || !prog->jit_requested)  in bpf_prog_alloc_jited_linfo()
163 	prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,  in bpf_prog_alloc_jited_linfo()
164 					  sizeof(*prog->aux->jited_linfo),  in bpf_prog_alloc_jited_linfo()
166 	if (!prog->aux->jited_linfo)  in bpf_prog_alloc_jited_linfo()
172 void bpf_prog_jit_attempt_done(struct bpf_prog *prog)  in bpf_prog_jit_attempt_done()  argument
174 	if (prog->aux->jited_linfo &&  in bpf_prog_jit_attempt_done()
175 	    (!prog->jited || !prog->aux->jited_linfo[0])) {  in bpf_prog_jit_attempt_done()
176 		kvfree(prog->aux->jited_linfo);  in bpf_prog_jit_attempt_done()
177 		prog->aux->jited_linfo = NULL;  in bpf_prog_jit_attempt_done()
180 	kfree(prog->aux->kfunc_tab);  in bpf_prog_jit_attempt_done()
181 	prog->aux->kfunc_tab = NULL;  in bpf_prog_jit_attempt_done()
208 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,  in bpf_prog_fill_jited_linfo()  argument
215 	if (!prog->aux->jited_linfo)  in bpf_prog_fill_jited_linfo()
219 	linfo_idx = prog->aux->linfo_idx;  in bpf_prog_fill_jited_linfo()
220 	linfo = &prog->aux->linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
222 	insn_end = insn_start + prog->len;  in bpf_prog_fill_jited_linfo()
224 	jited_linfo = &prog->aux->jited_linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
225 	jited_linfo[0] = prog->bpf_func;  in bpf_prog_fill_jited_linfo()
227 	nr_linfo = prog->aux->nr_linfo - linfo_idx;  in bpf_prog_fill_jited_linfo()
233 		jited_linfo[i] = prog->bpf_func +  in bpf_prog_fill_jited_linfo()
253 		fp->aux->prog = fp;  in bpf_prog_realloc()
398 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old,  in bpf_adj_branches()  argument
401 	u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0);  in bpf_adj_branches()
402 	struct bpf_insn *insn = prog->insnsi;  in bpf_adj_branches()
414 			insn = prog->insnsi + end_old;  in bpf_adj_branches()
445 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)  in bpf_adj_linfo()  argument
450 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo()
454 	linfo = prog->aux->linfo;  in bpf_adj_linfo()
465 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,  in bpf_patch_insn_single()  argument
475 		memcpy(prog->insnsi + off, patch, sizeof(*patch));  in bpf_patch_insn_single()
476 		return prog;  in bpf_patch_insn_single()
479 	insn_adj_cnt = prog->len + insn_delta;  in bpf_patch_insn_single()
487 	    (err = bpf_adj_branches(prog, off, off + 1, off + len, true)))  in bpf_patch_insn_single()
494 	prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),  in bpf_patch_insn_single()
526 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt)  in bpf_remove_insns()  argument
531 	memmove(prog->insnsi + off, prog->insnsi + off + cnt,  in bpf_remove_insns()
532 		sizeof(struct bpf_insn) * (prog->len - off - cnt));  in bpf_remove_insns()
533 	prog->len -= cnt;  in bpf_remove_insns()
535 	return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false));  in bpf_remove_insns()
561 bpf_prog_ksym_set_addr(struct bpf_prog *prog)  in bpf_prog_ksym_set_addr()  argument
563 	WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog));  in bpf_prog_ksym_set_addr()
565 	prog->aux->ksym.start = (unsigned long) prog->bpf_func;  in bpf_prog_ksym_set_addr()
566 	prog->aux->ksym.end   = prog->aux->ksym.start + prog->jited_len;  in bpf_prog_ksym_set_addr()
570 bpf_prog_ksym_set_name(struct bpf_prog *prog)  in bpf_prog_ksym_set_name()  argument
572 	char *sym = prog->aux->ksym.name;  in bpf_prog_ksym_set_name()
578 		     sizeof(prog->tag) * 2 +  in bpf_prog_ksym_set_name()
586 		     sizeof(prog->aux->name) > KSYM_NAME_LEN);  in bpf_prog_ksym_set_name()
589 	sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));  in bpf_prog_ksym_set_name()
592 	if (prog->aux->func_info_cnt) {  in bpf_prog_ksym_set_name()
593 		type = btf_type_by_id(prog->aux->btf,  in bpf_prog_ksym_set_name()
594 				      prog->aux->func_info[prog->aux->func_idx].type_id);  in bpf_prog_ksym_set_name()
595 		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);  in bpf_prog_ksym_set_name()
600 	if (prog->aux->name[0])  in bpf_prog_ksym_set_name()
601 		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);  in bpf_prog_ksym_set_name()
679 	fp->aux->ksym.prog = true;  in bpf_prog_kallsyms_add()
740 	return ksym && ksym->prog ?  in bpf_prog_ksym_find()
741 	       container_of(ksym, struct bpf_prog_aux, ksym)->prog :  in bpf_prog_ksym_find()
748 	struct bpf_prog *prog;  in search_bpf_extables()  local
751 	prog = bpf_prog_ksym_find(addr);  in search_bpf_extables()
752 	if (!prog)  in search_bpf_extables()
754 	if (!prog->aux->num_exentries)  in search_bpf_extables()
757 	e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);  in search_bpf_extables()
791 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,  in bpf_jit_add_poke_descriptor()  argument
794 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in bpf_jit_add_poke_descriptor()
796 	u32 slot = prog->aux->size_poke_tab;  in bpf_jit_add_poke_descriptor()
819 	prog->aux->size_poke_tab = size;  in bpf_jit_add_poke_descriptor()
820 	prog->aux->poke_tab = tab;  in bpf_jit_add_poke_descriptor()
1120 int bpf_jit_binary_pack_finalize(struct bpf_prog *prog,  in bpf_jit_binary_pack_finalize()  argument
1193 int bpf_jit_get_func_addr(const struct bpf_prog *prog,  in bpf_jit_get_func_addr()  argument
1210 		else if (prog->aux->func &&  in bpf_jit_get_func_addr()
1211 			 off >= 0 && off < prog->aux->func_cnt)  in bpf_jit_get_func_addr()
1212 			addr = (u8 *)prog->aux->func[off]->bpf_func;  in bpf_jit_get_func_addr()
1217 		err = bpf_get_kfunc_addr(prog, insn->imm, insn->off, &addr);  in bpf_jit_get_func_addr()
1407 	fp->aux->prog = fp;  in bpf_jit_prog_release_other()
1411 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)  in bpf_jit_blind_constants()  argument
1419 	if (!prog->blinding_requested || prog->blinded)  in bpf_jit_blind_constants()
1420 		return prog;  in bpf_jit_blind_constants()
1422 	clone = bpf_prog_clone_create(prog, GFP_USER);  in bpf_jit_blind_constants()
1459 			bpf_jit_prog_release_other(prog, clone);  in bpf_jit_blind_constants()
1968 		struct bpf_prog *prog;  in ___bpf_prog_run()  local
1979 		prog = READ_ONCE(array->ptrs[index]);  in ___bpf_prog_run()
1980 		if (!prog)  in ___bpf_prog_run()
1988 		insn = prog->insnsi;  in ___bpf_prog_run()
2383 	struct bpf_prog prog;  member
2385 	.prog = {
2439 	for (item = array->items; item->prog; item++)  in bpf_prog_array_length()
2440 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_length()
2449 	for (item = array->items; item->prog; item++)  in bpf_prog_array_is_empty()
2450 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_is_empty()
2462 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_copy_core()
2463 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy_core()
2465 		prog_ids[i] = item->prog->aux->id;  in bpf_prog_array_copy_core()
2472 	return !!(item->prog);  in bpf_prog_array_copy_core()
2506 	for (item = array->items; item->prog; item++)  in bpf_prog_array_delete_safe()
2507 		if (item->prog == old_prog) {  in bpf_prog_array_delete_safe()
2508 			WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe()
2530 	return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe_at()
2549 			     struct bpf_prog *prog)  in bpf_prog_array_update_at()  argument
2556 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_update_at()
2557 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_update_at()
2560 			WRITE_ONCE(item->prog, prog);  in bpf_prog_array_update_at()
2584 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
2585 			if (existing->prog == exclude_prog) {  in bpf_prog_array_copy()
2589 			if (existing->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
2591 			if (existing->prog == include_prog)  in bpf_prog_array_copy()
2619 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
2620 			if (existing->prog == exclude_prog ||  in bpf_prog_array_copy()
2621 			    existing->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
2624 			new->prog = existing->prog;  in bpf_prog_array_copy()
2630 		new->prog = include_prog;  in bpf_prog_array_copy()
2634 	new->prog = NULL;  in bpf_prog_array_copy()
2717 		bpf_prog_dev_bound_destroy(aux->prog);  in bpf_prog_free_deferred()
2719 	if (aux->prog->has_callchain_buf)  in bpf_prog_free_deferred()
2734 		bpf_prog_unlock_free(aux->prog);  in bpf_prog_free_deferred()
2736 		bpf_jit_free(aux->prog);  in bpf_prog_free_deferred()
2846 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
2848 	return prog;  in bpf_int_jit_compile()
2854 void __weak bpf_jit_compile(struct bpf_prog *prog)  in bpf_jit_compile()  argument