108 bool bpf_prog_has_trampoline(const struct bpf_prog *prog)  in bpf_prog_has_trampoline()  argument
110 	enum bpf_attach_type eatype = prog->expected_attach_type;  in bpf_prog_has_trampoline()
111 	enum bpf_prog_type ptype = prog->type;  in bpf_prog_has_trampoline()
275 			*ip_arg |= link->link.prog->call_get_func_ip;  in bpf_trampoline_get_progs()
510 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)  in bpf_attach_type_to_tramp()  argument
512 	switch (prog->expected_attach_type) {  in bpf_attach_type_to_tramp()
520 		if (!prog->aux->attach_func_proto->type)  in bpf_attach_type_to_tramp()
539 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_link_prog()
553 		tr->extension_prog = link->link.prog;  in __bpf_trampoline_link_prog()
555 					  link->link.prog->bpf_func);  in __bpf_trampoline_link_prog()
563 		if (link_exiting->link.prog != link->link.prog)  in __bpf_trampoline_link_prog()
594 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_unlink_prog()
645 static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,  in cgroup_shim_alloc()  argument
666 	p->aux->attach_func_proto = prog->aux->attach_func_proto;  in cgroup_shim_alloc()
667 	p->aux->attach_btf_id = prog->aux->attach_btf_id;  in cgroup_shim_alloc()
668 	p->aux->attach_btf = prog->aux->attach_btf;  in cgroup_shim_alloc()
688 			struct bpf_prog *p = link->link.prog;  in cgroup_shim_find()
698 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,  in bpf_trampoline_link_cgroup_shim()  argument
708 	err = bpf_check_attach_target(NULL, prog, NULL,  in bpf_trampoline_link_cgroup_shim()
709 				      prog->aux->attach_btf_id,  in bpf_trampoline_link_cgroup_shim()
714 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_link_cgroup_shim()
715 					 prog->aux->attach_btf_id);  in bpf_trampoline_link_cgroup_shim()
717 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_link_cgroup_shim()
736 	shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);  in bpf_trampoline_link_cgroup_shim()
764 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)  in bpf_trampoline_unlink_cgroup_shim()  argument
771 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_unlink_cgroup_shim()
772 					 prog->aux->attach_btf_id);  in bpf_trampoline_unlink_cgroup_shim()
774 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_unlink_cgroup_shim()
867 u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_enter()  argument
875 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter()
876 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter()
882 static void notrace update_prog_stats(struct bpf_prog *prog,  in update_prog_stats()  argument
896 		stats = this_cpu_ptr(prog->stats);  in update_prog_stats()
904 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_exit()  argument
909 	update_prog_stats(prog, start);  in __bpf_prog_exit()
910 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit()
915 u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,  in __bpf_prog_enter_lsm_cgroup()  argument
930 void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_lsm_cgroup()  argument
940 u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_enter_sleepable()  argument
946 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_sleepable()
947 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_sleepable()
956 void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable()  argument
961 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable()
962 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_sleepable()
967 u64 notrace __bpf_prog_enter_struct_ops(struct bpf_prog *prog,  in __bpf_prog_enter_struct_ops()  argument
979 void notrace __bpf_prog_exit_struct_ops(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_struct_ops()  argument
985 	update_prog_stats(prog, start);  in __bpf_prog_exit_struct_ops()