60 static void bpf_trampoline_ksym_add(struct bpf_trampoline *tr)  in bpf_trampoline_ksym_add()  argument
62 	struct bpf_ksym *ksym = &tr->ksym;  in bpf_trampoline_ksym_add()
64 	snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", tr->key);  in bpf_trampoline_ksym_add()
65 	bpf_image_ksym_add(tr->image, ksym);  in bpf_trampoline_ksym_add()
70 	struct bpf_trampoline *tr;  in bpf_trampoline_lookup()  local
77 	hlist_for_each_entry(tr, head, hlist) {  in bpf_trampoline_lookup()
78 		if (tr->key == key) {  in bpf_trampoline_lookup()
79 			refcount_inc(&tr->refcnt);  in bpf_trampoline_lookup()
83 	tr = kzalloc(sizeof(*tr), GFP_KERNEL);  in bpf_trampoline_lookup()
84 	if (!tr)  in bpf_trampoline_lookup()
90 		kfree(tr);  in bpf_trampoline_lookup()
91 		tr = NULL;  in bpf_trampoline_lookup()
95 	tr->key = key;  in bpf_trampoline_lookup()
96 	INIT_HLIST_NODE(&tr->hlist);  in bpf_trampoline_lookup()
97 	hlist_add_head(&tr->hlist, head);  in bpf_trampoline_lookup()
98 	refcount_set(&tr->refcnt, 1);  in bpf_trampoline_lookup()
99 	mutex_init(&tr->mutex);  in bpf_trampoline_lookup()
101 		INIT_HLIST_HEAD(&tr->progs_hlist[i]);  in bpf_trampoline_lookup()
102 	tr->image = image;  in bpf_trampoline_lookup()
103 	INIT_LIST_HEAD_RCU(&tr->ksym.lnode);  in bpf_trampoline_lookup()
104 	bpf_trampoline_ksym_add(tr);  in bpf_trampoline_lookup()
107 	return tr;  in bpf_trampoline_lookup()
122 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)  in unregister_fentry()  argument
124 	void *ip = tr->func.addr;  in unregister_fentry()
127 	if (tr->func.ftrace_managed)  in unregister_fentry()
134 static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr)  in modify_fentry()  argument
136 	void *ip = tr->func.addr;  in modify_fentry()
139 	if (tr->func.ftrace_managed)  in modify_fentry()
147 static int register_fentry(struct bpf_trampoline *tr, void *new_addr)  in register_fentry()  argument
149 	void *ip = tr->func.addr;  in register_fentry()
155 	tr->func.ftrace_managed = ret;  in register_fentry()
157 	if (tr->func.ftrace_managed)  in register_fentry()
165 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total)  in bpf_trampoline_get_progs()  argument
178 		tprogs[kind].nr_progs = tr->progs_cnt[kind];  in bpf_trampoline_get_progs()
179 		*total += tr->progs_cnt[kind];  in bpf_trampoline_get_progs()
182 		hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist)  in bpf_trampoline_get_progs()
188 static int bpf_trampoline_update(struct bpf_trampoline *tr)  in bpf_trampoline_update()  argument
190 	void *old_image = tr->image + ((tr->selector + 1) & 1) * PAGE_SIZE/2;  in bpf_trampoline_update()
191 	void *new_image = tr->image + (tr->selector & 1) * PAGE_SIZE/2;  in bpf_trampoline_update()
196 	tprogs = bpf_trampoline_get_progs(tr, &total);  in bpf_trampoline_update()
201 		err = unregister_fentry(tr, old_image);  in bpf_trampoline_update()
202 		tr->selector = 0;  in bpf_trampoline_update()
223 					  &tr->func.model, flags, tprogs,  in bpf_trampoline_update()
224 					  tr->func.addr);  in bpf_trampoline_update()
228 	if (tr->selector)  in bpf_trampoline_update()
230 		err = modify_fentry(tr, old_image, new_image);  in bpf_trampoline_update()
233 		err = register_fentry(tr, new_image);  in bpf_trampoline_update()
236 	tr->selector++;  in bpf_trampoline_update()
264 int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)  in bpf_trampoline_link_prog()  argument
271 	mutex_lock(&tr->mutex);  in bpf_trampoline_link_prog()
272 	if (tr->extension_prog) {  in bpf_trampoline_link_prog()
279 	cnt = tr->progs_cnt[BPF_TRAMP_FENTRY] + tr->progs_cnt[BPF_TRAMP_FEXIT];  in bpf_trampoline_link_prog()
286 		tr->extension_prog = prog;  in bpf_trampoline_link_prog()
287 		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,  in bpf_trampoline_link_prog()
300 	hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);  in bpf_trampoline_link_prog()
301 	tr->progs_cnt[kind]++;  in bpf_trampoline_link_prog()
302 	err = bpf_trampoline_update(tr);  in bpf_trampoline_link_prog()
305 		tr->progs_cnt[kind]--;  in bpf_trampoline_link_prog()
308 	mutex_unlock(&tr->mutex);  in bpf_trampoline_link_prog()
313 int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)  in bpf_trampoline_unlink_prog()  argument
319 	mutex_lock(&tr->mutex);  in bpf_trampoline_unlink_prog()
321 		WARN_ON_ONCE(!tr->extension_prog);  in bpf_trampoline_unlink_prog()
322 		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,  in bpf_trampoline_unlink_prog()
323 					 tr->extension_prog->bpf_func, NULL);  in bpf_trampoline_unlink_prog()
324 		tr->extension_prog = NULL;  in bpf_trampoline_unlink_prog()
328 	tr->progs_cnt[kind]--;  in bpf_trampoline_unlink_prog()
329 	err = bpf_trampoline_update(tr);  in bpf_trampoline_unlink_prog()
331 	mutex_unlock(&tr->mutex);  in bpf_trampoline_unlink_prog()
338 	struct bpf_trampoline *tr;  in bpf_trampoline_get()  local
340 	tr = bpf_trampoline_lookup(key);  in bpf_trampoline_get()
341 	if (!tr)  in bpf_trampoline_get()
344 	mutex_lock(&tr->mutex);  in bpf_trampoline_get()
345 	if (tr->func.addr)  in bpf_trampoline_get()
348 	memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));  in bpf_trampoline_get()
349 	tr->func.addr = (void *)tgt_info->tgt_addr;  in bpf_trampoline_get()
351 	mutex_unlock(&tr->mutex);  in bpf_trampoline_get()
352 	return tr;  in bpf_trampoline_get()
355 void bpf_trampoline_put(struct bpf_trampoline *tr)  in bpf_trampoline_put()  argument
357 	if (!tr)  in bpf_trampoline_put()
360 	if (!refcount_dec_and_test(&tr->refcnt))  in bpf_trampoline_put()
362 	WARN_ON_ONCE(mutex_is_locked(&tr->mutex));  in bpf_trampoline_put()
363 	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FENTRY])))  in bpf_trampoline_put()
365 	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT])))  in bpf_trampoline_put()
367 	bpf_image_ksym_del(&tr->ksym);  in bpf_trampoline_put()
377 	bpf_jit_free_exec(tr->image);  in bpf_trampoline_put()
378 	hlist_del(&tr->hlist);  in bpf_trampoline_put()
379 	kfree(tr);  in bpf_trampoline_put()