63 	struct bpf_trampoline *tr;  in bpf_trampoline_lookup()  local
69 	hlist_for_each_entry(tr, head, hlist) {  in bpf_trampoline_lookup()
70 		if (tr->key == key) {  in bpf_trampoline_lookup()
71 			refcount_inc(&tr->refcnt);  in bpf_trampoline_lookup()
75 	tr = kzalloc(sizeof(*tr), GFP_KERNEL);  in bpf_trampoline_lookup()
76 	if (!tr)  in bpf_trampoline_lookup()
79 	tr->key = key;  in bpf_trampoline_lookup()
80 	INIT_HLIST_NODE(&tr->hlist);  in bpf_trampoline_lookup()
81 	hlist_add_head(&tr->hlist, head);  in bpf_trampoline_lookup()
82 	refcount_set(&tr->refcnt, 1);  in bpf_trampoline_lookup()
83 	mutex_init(&tr->mutex);  in bpf_trampoline_lookup()
85 		INIT_HLIST_HEAD(&tr->progs_hlist[i]);  in bpf_trampoline_lookup()
88 	return tr;  in bpf_trampoline_lookup()
91 static int bpf_trampoline_module_get(struct bpf_trampoline *tr)  in bpf_trampoline_module_get()  argument
97 	mod = __module_text_address((unsigned long) tr->func.addr);  in bpf_trampoline_module_get()
101 	tr->mod = mod;  in bpf_trampoline_module_get()
105 static void bpf_trampoline_module_put(struct bpf_trampoline *tr)  in bpf_trampoline_module_put()  argument
107 	module_put(tr->mod);  in bpf_trampoline_module_put()
108 	tr->mod = NULL;  in bpf_trampoline_module_put()
123 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)  in unregister_fentry()  argument
125 	void *ip = tr->func.addr;  in unregister_fentry()
128 	if (tr->func.ftrace_managed)  in unregister_fentry()
134 		bpf_trampoline_module_put(tr);  in unregister_fentry()
138 static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr)  in modify_fentry()  argument
140 	void *ip = tr->func.addr;  in modify_fentry()
143 	if (tr->func.ftrace_managed)  in modify_fentry()
151 static int register_fentry(struct bpf_trampoline *tr, void *new_addr)  in register_fentry()  argument
153 	void *ip = tr->func.addr;  in register_fentry()
159 	tr->func.ftrace_managed = ret;  in register_fentry()
161 	if (bpf_trampoline_module_get(tr))  in register_fentry()
164 	if (tr->func.ftrace_managed)  in register_fentry()
170 		bpf_trampoline_module_put(tr);  in register_fentry()
175 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)  in bpf_trampoline_get_progs()  argument
188 		tprogs[kind].nr_progs = tr->progs_cnt[kind];  in bpf_trampoline_get_progs()
189 		*total += tr->progs_cnt[kind];  in bpf_trampoline_get_progs()
192 		hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) {  in bpf_trampoline_get_progs()
333 static int bpf_trampoline_update(struct bpf_trampoline *tr)  in bpf_trampoline_update()  argument
341 	tprogs = bpf_trampoline_get_progs(tr, &total, &ip_arg);  in bpf_trampoline_update()
346 		err = unregister_fentry(tr, tr->cur_image->image);  in bpf_trampoline_update()
347 		bpf_tramp_image_put(tr->cur_image);  in bpf_trampoline_update()
348 		tr->cur_image = NULL;  in bpf_trampoline_update()
349 		tr->selector = 0;  in bpf_trampoline_update()
353 	im = bpf_tramp_image_alloc(tr->key, tr->selector);  in bpf_trampoline_update()
367 					  &tr->func.model, flags, tprogs,  in bpf_trampoline_update()
368 					  tr->func.addr);  in bpf_trampoline_update()
372 	WARN_ON(tr->cur_image && tr->selector == 0);  in bpf_trampoline_update()
373 	WARN_ON(!tr->cur_image && tr->selector);  in bpf_trampoline_update()
374 	if (tr->cur_image)  in bpf_trampoline_update()
376 		err = modify_fentry(tr, tr->cur_image->image, im->image);  in bpf_trampoline_update()
379 		err = register_fentry(tr, im->image);  in bpf_trampoline_update()
382 	if (tr->cur_image)  in bpf_trampoline_update()
383 		bpf_tramp_image_put(tr->cur_image);  in bpf_trampoline_update()
384 	tr->cur_image = im;  in bpf_trampoline_update()
385 	tr->selector++;  in bpf_trampoline_update()
413 int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)  in bpf_trampoline_link_prog()  argument
420 	mutex_lock(&tr->mutex);  in bpf_trampoline_link_prog()
421 	if (tr->extension_prog) {  in bpf_trampoline_link_prog()
428 	cnt = tr->progs_cnt[BPF_TRAMP_FENTRY] + tr->progs_cnt[BPF_TRAMP_FEXIT];  in bpf_trampoline_link_prog()
435 		tr->extension_prog = prog;  in bpf_trampoline_link_prog()
436 		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,  in bpf_trampoline_link_prog()
449 	hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);  in bpf_trampoline_link_prog()
450 	tr->progs_cnt[kind]++;  in bpf_trampoline_link_prog()
451 	err = bpf_trampoline_update(tr);  in bpf_trampoline_link_prog()
454 		tr->progs_cnt[kind]--;  in bpf_trampoline_link_prog()
457 	mutex_unlock(&tr->mutex);  in bpf_trampoline_link_prog()
462 int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)  in bpf_trampoline_unlink_prog()  argument
468 	mutex_lock(&tr->mutex);  in bpf_trampoline_unlink_prog()
470 		WARN_ON_ONCE(!tr->extension_prog);  in bpf_trampoline_unlink_prog()
471 		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,  in bpf_trampoline_unlink_prog()
472 					 tr->extension_prog->bpf_func, NULL);  in bpf_trampoline_unlink_prog()
473 		tr->extension_prog = NULL;  in bpf_trampoline_unlink_prog()
477 	tr->progs_cnt[kind]--;  in bpf_trampoline_unlink_prog()
478 	err = bpf_trampoline_update(tr);  in bpf_trampoline_unlink_prog()
480 	mutex_unlock(&tr->mutex);  in bpf_trampoline_unlink_prog()
487 	struct bpf_trampoline *tr;  in bpf_trampoline_get()  local
489 	tr = bpf_trampoline_lookup(key);  in bpf_trampoline_get()
490 	if (!tr)  in bpf_trampoline_get()
493 	mutex_lock(&tr->mutex);  in bpf_trampoline_get()
494 	if (tr->func.addr)  in bpf_trampoline_get()
497 	memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));  in bpf_trampoline_get()
498 	tr->func.addr = (void *)tgt_info->tgt_addr;  in bpf_trampoline_get()
500 	mutex_unlock(&tr->mutex);  in bpf_trampoline_get()
501 	return tr;  in bpf_trampoline_get()
504 void bpf_trampoline_put(struct bpf_trampoline *tr)  in bpf_trampoline_put()  argument
506 	if (!tr)  in bpf_trampoline_put()
509 	if (!refcount_dec_and_test(&tr->refcnt))  in bpf_trampoline_put()
511 	WARN_ON_ONCE(mutex_is_locked(&tr->mutex));  in bpf_trampoline_put()
512 	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FENTRY])))  in bpf_trampoline_put()
514 	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT])))  in bpf_trampoline_put()
522 	hlist_del(&tr->hlist);  in bpf_trampoline_put()
523 	kfree(tr);  in bpf_trampoline_put()
625 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)  in __bpf_tramp_enter()  argument
627 	percpu_ref_get(&tr->pcref);  in __bpf_tramp_enter()
630 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr)  in __bpf_tramp_exit()  argument
632 	percpu_ref_put(&tr->pcref);  in __bpf_tramp_exit()
636 arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,  in arch_prepare_bpf_trampoline()  argument