97 	fp->aux->prog = fp;  in bpf_prog_alloc()
132 		fp->aux->prog = fp;  in bpf_prog_realloc()
256 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta,  in bpf_adj_branches()  argument
259 	u32 i, insn_cnt = prog->len + (probe_pass ? delta : 0);  in bpf_adj_branches()
260 	struct bpf_insn *insn = prog->insnsi;  in bpf_adj_branches()
295 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,  in bpf_patch_insn_single()  argument
304 		memcpy(prog->insnsi + off, patch, sizeof(*patch));  in bpf_patch_insn_single()
305 		return prog;  in bpf_patch_insn_single()
308 	insn_adj_cnt = prog->len + insn_delta;  in bpf_patch_insn_single()
316 	    bpf_adj_branches(prog, off, insn_delta, true))  in bpf_patch_insn_single()
323 	prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),  in bpf_patch_insn_single()
374 bpf_get_prog_addr_region(const struct bpf_prog *prog,  in bpf_get_prog_addr_region()  argument
378 	const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog);  in bpf_get_prog_addr_region()
381 	WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog));  in bpf_get_prog_addr_region()
387 static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)  in bpf_get_prog_name()  argument
392 		     sizeof(prog->tag) * 2 +  in bpf_get_prog_name()
400 		     sizeof(prog->aux->name) > KSYM_NAME_LEN);  in bpf_get_prog_name()
403 	sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));  in bpf_get_prog_name()
404 	if (prog->aux->name[0])  in bpf_get_prog_name()
405 		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);  in bpf_get_prog_name()
417 	bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);  in bpf_get_prog_addr_start()
435 	bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);  in bpf_tree_comp()
511 	       container_of(n, struct bpf_prog_aux, ksym_tnode)->prog :  in bpf_prog_kallsyms_find()
519 	struct bpf_prog *prog;  in __bpf_address_lookup()  local
523 	prog = bpf_prog_kallsyms_find(addr);  in __bpf_address_lookup()
524 	if (prog) {  in __bpf_address_lookup()
525 		bpf_get_prog_addr_region(prog, &symbol_start, &symbol_end);  in __bpf_address_lookup()
526 		bpf_get_prog_name(prog, sym);  in __bpf_address_lookup()
566 		bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);  in bpf_get_kallsym()
567 		bpf_get_prog_name(aux->prog, sym);  in bpf_get_kallsym()
762 	fp->aux->prog = fp;  in bpf_jit_prog_release_other()
766 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)  in bpf_jit_blind_constants()  argument
774 	if (!bpf_jit_blinding_enabled(prog) || prog->blinded)  in bpf_jit_blind_constants()
775 		return prog;  in bpf_jit_blind_constants()
777 	clone = bpf_prog_clone_create(prog, GFP_USER);  in bpf_jit_blind_constants()
803 			bpf_jit_prog_release_other(prog, clone);  in bpf_jit_blind_constants()
1126 		struct bpf_prog *prog;  in ___bpf_prog_run()  local
1136 		prog = READ_ONCE(array->ptrs[index]);  in ___bpf_prog_run()
1137 		if (!prog)  in ___bpf_prog_run()
1145 		insn = prog->insnsi;  in ___bpf_prog_run()
1521 	struct bpf_prog prog;  member
1523 	.prog = {
1567 	for (; item->prog; item++)  in bpf_prog_array_length()
1568 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_length()
1583 	for (; item->prog; item++) {  in bpf_prog_array_copy_core()
1584 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy_core()
1586 		prog_ids[i] = item->prog->aux->id;  in bpf_prog_array_copy_core()
1593 	return !!(item->prog);  in bpf_prog_array_copy_core()
1633 	for (; item->prog; item++)  in bpf_prog_array_delete_safe()
1634 		if (item->prog == old_prog) {  in bpf_prog_array_delete_safe()
1635 			WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe()
1656 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
1657 			if (existing->prog == exclude_prog) {  in bpf_prog_array_copy()
1661 			if (existing->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
1663 			if (existing->prog == include_prog)  in bpf_prog_array_copy()
1690 		for (; existing->prog; existing++)  in bpf_prog_array_copy()
1691 			if (existing->prog != exclude_prog &&  in bpf_prog_array_copy()
1692 			    existing->prog != &dummy_bpf_prog.prog) {  in bpf_prog_array_copy()
1693 				array->items[new_prog_idx++].prog =  in bpf_prog_array_copy()
1694 					existing->prog;  in bpf_prog_array_copy()
1698 		array->items[new_prog_idx++].prog = include_prog;  in bpf_prog_array_copy()
1699 	array->items[new_prog_idx].prog = NULL;  in bpf_prog_array_copy()
1731 		bpf_prog_offload_destroy(aux->prog);  in bpf_prog_free_deferred()
1733 	if (aux->prog->has_callchain_buf)  in bpf_prog_free_deferred()
1740 		bpf_prog_unlock_free(aux->prog);  in bpf_prog_free_deferred()
1742 		bpf_jit_free(aux->prog);  in bpf_prog_free_deferred()
1827 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
1829 	return prog;  in bpf_int_jit_compile()
1835 void __weak bpf_jit_compile(struct bpf_prog *prog)  in bpf_jit_compile()  argument