193 			*ip_arg |= aux->prog->call_get_func_ip;  in bpf_trampoline_get_progs()
194 			*progs++ = aux->prog;  in bpf_trampoline_get_progs()
391 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)  in bpf_attach_type_to_tramp()  argument
393 	switch (prog->expected_attach_type) {  in bpf_attach_type_to_tramp()
401 		if (!prog->aux->attach_func_proto->type)  in bpf_attach_type_to_tramp()
413 int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)  in bpf_trampoline_link_prog()  argument
419 	kind = bpf_attach_type_to_tramp(prog);  in bpf_trampoline_link_prog()
435 		tr->extension_prog = prog;  in bpf_trampoline_link_prog()
437 					 prog->bpf_func);  in bpf_trampoline_link_prog()
444 	if (!hlist_unhashed(&prog->aux->tramp_hlist)) {  in bpf_trampoline_link_prog()
449 	hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);  in bpf_trampoline_link_prog()
453 		hlist_del_init(&prog->aux->tramp_hlist);  in bpf_trampoline_link_prog()
462 int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)  in bpf_trampoline_unlink_prog()  argument
467 	kind = bpf_attach_type_to_tramp(prog);  in bpf_trampoline_unlink_prog()
476 	hlist_del_init(&prog->aux->tramp_hlist);  in bpf_trampoline_unlink_prog()
541 static void notrace inc_misses_counter(struct bpf_prog *prog)  in inc_misses_counter()  argument
545 	stats = this_cpu_ptr(prog->stats);  in inc_misses_counter()
564 u64 notrace __bpf_prog_enter(struct bpf_prog *prog)  in __bpf_prog_enter()  argument
569 	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter()
570 		inc_misses_counter(prog);  in __bpf_prog_enter()
576 static void notrace update_prog_stats(struct bpf_prog *prog,  in update_prog_stats()  argument
588 		stats = this_cpu_ptr(prog->stats);  in update_prog_stats()
596 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)  in __bpf_prog_exit()  argument
599 	update_prog_stats(prog, start);  in __bpf_prog_exit()
600 	__this_cpu_dec(*(prog->active));  in __bpf_prog_exit()
605 u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)  in __bpf_prog_enter_sleepable()  argument
610 	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_sleepable()
611 		inc_misses_counter(prog);  in __bpf_prog_enter_sleepable()
617 void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)  in __bpf_prog_exit_sleepable()  argument
619 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable()
620 	__this_cpu_dec(*(prog->active));  in __bpf_prog_exit_sleepable()