1202 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)  in find_prog_type()  argument
1213 	if (!bpf_prog_is_dev_bound(prog->aux))  in find_prog_type()
1214 		prog->aux->ops = ops;  in find_prog_type()
1216 		prog->aux->ops = &bpf_offload_prog_ops;  in find_prog_type()
1217 	prog->type = type;  in find_prog_type()
1230 		bpf_cgroup_storage_release(aux->prog,  in free_used_maps()
1262 static int bpf_prog_charge_memlock(struct bpf_prog *prog)  in bpf_prog_charge_memlock()  argument
1267 	ret = __bpf_prog_charge(user, prog->pages);  in bpf_prog_charge_memlock()
1273 	prog->aux->user = user;  in bpf_prog_charge_memlock()
1277 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)  in bpf_prog_uncharge_memlock()  argument
1279 	struct user_struct *user = prog->aux->user;  in bpf_prog_uncharge_memlock()
1281 	__bpf_prog_uncharge(user, prog->pages);  in bpf_prog_uncharge_memlock()
1285 static int bpf_prog_alloc_id(struct bpf_prog *prog)  in bpf_prog_alloc_id()  argument
1291 	id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);  in bpf_prog_alloc_id()
1293 		prog->aux->id = id;  in bpf_prog_alloc_id()
1304 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)  in bpf_prog_free_id()  argument
1311 	if (!prog->aux->id)  in bpf_prog_free_id()
1319 	idr_remove(&prog_idr, prog->aux->id);  in bpf_prog_free_id()
1320 	prog->aux->id = 0;  in bpf_prog_free_id()
1334 	bpf_prog_uncharge_memlock(aux->prog);  in __bpf_prog_put_rcu()
1336 	bpf_prog_free(aux->prog);  in __bpf_prog_put_rcu()
1339 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)  in __bpf_prog_put_noref()  argument
1341 	bpf_prog_kallsyms_del_all(prog);  in __bpf_prog_put_noref()
1342 	btf_put(prog->aux->btf);  in __bpf_prog_put_noref()
1343 	bpf_prog_free_linfo(prog);  in __bpf_prog_put_noref()
1346 		call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);  in __bpf_prog_put_noref()
1348 		__bpf_prog_put_rcu(&prog->aux->rcu);  in __bpf_prog_put_noref()
1351 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)  in __bpf_prog_put()  argument
1353 	if (atomic_dec_and_test(&prog->aux->refcnt)) {  in __bpf_prog_put()
1354 		perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);  in __bpf_prog_put()
1356 		bpf_prog_free_id(prog, do_idr_lock);  in __bpf_prog_put()
1357 		__bpf_prog_put_noref(prog, true);  in __bpf_prog_put()
1361 void bpf_prog_put(struct bpf_prog *prog)  in bpf_prog_put()  argument
1363 	__bpf_prog_put(prog, true);  in bpf_prog_put()
1369 	struct bpf_prog *prog = filp->private_data;  in bpf_prog_release()  local
1371 	bpf_prog_put(prog);  in bpf_prog_release()
1375 static void bpf_prog_get_stats(const struct bpf_prog *prog,  in bpf_prog_get_stats()  argument
1386 		st = per_cpu_ptr(prog->aux->stats, cpu);  in bpf_prog_get_stats()
1402 	const struct bpf_prog *prog = filp->private_data;  in bpf_prog_show_fdinfo()  local
1403 	char prog_tag[sizeof(prog->tag) * 2 + 1] = { };  in bpf_prog_show_fdinfo()
1406 	bpf_prog_get_stats(prog, &stats);  in bpf_prog_show_fdinfo()
1407 	bin2hex(prog_tag, prog->tag, sizeof(prog->tag));  in bpf_prog_show_fdinfo()
1416 		   prog->type,  in bpf_prog_show_fdinfo()
1417 		   prog->jited,  in bpf_prog_show_fdinfo()
1419 		   prog->pages * 1ULL << PAGE_SHIFT,  in bpf_prog_show_fdinfo()
1420 		   prog->aux->id,  in bpf_prog_show_fdinfo()
1435 int bpf_prog_new_fd(struct bpf_prog *prog)  in bpf_prog_new_fd()  argument
1439 	ret = security_bpf_prog(prog);  in bpf_prog_new_fd()
1443 	return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,  in bpf_prog_new_fd()
1459 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)  in bpf_prog_add()  argument
1461 	if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {  in bpf_prog_add()
1462 		atomic_sub(i, &prog->aux->refcnt);  in bpf_prog_add()
1465 	return prog;  in bpf_prog_add()
1469 void bpf_prog_sub(struct bpf_prog *prog, int i)  in bpf_prog_sub()  argument
1476 	WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);  in bpf_prog_sub()
1480 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)  in bpf_prog_inc()  argument
1482 	return bpf_prog_add(prog, 1);  in bpf_prog_inc()
1487 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)  in bpf_prog_inc_not_zero()  argument
1491 	refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0);  in bpf_prog_inc_not_zero()
1494 		__bpf_prog_put(prog, false);  in bpf_prog_inc_not_zero()
1501 	return prog;  in bpf_prog_inc_not_zero()
1505 bool bpf_prog_get_ok(struct bpf_prog *prog,  in bpf_prog_get_ok()  argument
1512 	if (prog->type != *attach_type)  in bpf_prog_get_ok()
1514 	if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)  in bpf_prog_get_ok()
1524 	struct bpf_prog *prog;  in __bpf_prog_get()  local
1526 	prog = ____bpf_prog_get(f);  in __bpf_prog_get()
1527 	if (IS_ERR(prog))  in __bpf_prog_get()
1528 		return prog;  in __bpf_prog_get()
1529 	if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {  in __bpf_prog_get()
1530 		prog = ERR_PTR(-EINVAL);  in __bpf_prog_get()
1534 	prog = bpf_prog_inc(prog);  in __bpf_prog_get()
1537 	return prog;  in __bpf_prog_get()
1633 	struct bpf_prog *prog;  in bpf_prog_load()  local
1674 	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);  in bpf_prog_load()
1675 	if (!prog)  in bpf_prog_load()
1678 	prog->expected_attach_type = attr->expected_attach_type;  in bpf_prog_load()
1680 	prog->aux->offload_requested = !!attr->prog_ifindex;  in bpf_prog_load()
1682 	err = security_bpf_prog_alloc(prog->aux);  in bpf_prog_load()
1686 	err = bpf_prog_charge_memlock(prog);  in bpf_prog_load()
1690 	prog->len = attr->insn_cnt;  in bpf_prog_load()
1693 	if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),  in bpf_prog_load()
1694 			   bpf_prog_insn_size(prog)) != 0)  in bpf_prog_load()
1697 	prog->orig_prog = NULL;  in bpf_prog_load()
1698 	prog->jited = 0;  in bpf_prog_load()
1700 	atomic_set(&prog->aux->refcnt, 1);  in bpf_prog_load()
1701 	prog->gpl_compatible = is_gpl ? 1 : 0;  in bpf_prog_load()
1703 	if (bpf_prog_is_dev_bound(prog->aux)) {  in bpf_prog_load()
1704 		err = bpf_prog_offload_init(prog, attr);  in bpf_prog_load()
1710 	err = find_prog_type(type, prog);  in bpf_prog_load()
1714 	prog->aux->load_time = ktime_get_boottime_ns();  in bpf_prog_load()
1715 	err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);  in bpf_prog_load()
1720 	err = bpf_check(&prog, attr, uattr);  in bpf_prog_load()
1724 	prog = bpf_prog_select_runtime(prog, &err);  in bpf_prog_load()
1728 	err = bpf_prog_alloc_id(prog);  in bpf_prog_load()
1746 	bpf_prog_kallsyms_add(prog);  in bpf_prog_load()
1747 	perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);  in bpf_prog_load()
1749 	err = bpf_prog_new_fd(prog);  in bpf_prog_load()
1751 		bpf_prog_put(prog);  in bpf_prog_load()
1759 	__bpf_prog_put_noref(prog, prog->aux->func_cnt);  in bpf_prog_load()
1762 	bpf_prog_uncharge_memlock(prog);  in bpf_prog_load()
1764 	security_bpf_prog_free(prog->aux);  in bpf_prog_load()
1766 	bpf_prog_free(prog);  in bpf_prog_load()
1792 	struct bpf_prog *prog;  member
1799 	if (raw_tp->prog) {  in bpf_raw_tracepoint_release()
1800 		bpf_probe_unregister(raw_tp->btp, raw_tp->prog);  in bpf_raw_tracepoint_release()
1801 		bpf_prog_put(raw_tp->prog);  in bpf_raw_tracepoint_release()
1820 	struct bpf_prog *prog;  in bpf_raw_tracepoint_open()  local
1840 	prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);  in bpf_raw_tracepoint_open()
1841 	if (IS_ERR(prog)) {  in bpf_raw_tracepoint_open()
1842 		err = PTR_ERR(prog);  in bpf_raw_tracepoint_open()
1845 	if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT &&  in bpf_raw_tracepoint_open()
1846 	    prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {  in bpf_raw_tracepoint_open()
1851 	err = bpf_probe_register(raw_tp->btp, prog);  in bpf_raw_tracepoint_open()
1855 	raw_tp->prog = prog;  in bpf_raw_tracepoint_open()
1859 		bpf_probe_unregister(raw_tp->btp, prog);  in bpf_raw_tracepoint_open()
1866 	bpf_prog_put(prog);  in bpf_raw_tracepoint_open()
1874 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,  in bpf_prog_attach_check_attach_type()  argument
1877 	switch (prog->type) {  in bpf_prog_attach_check_attach_type()
1881 		return attach_type == prog->expected_attach_type ? 0 : -EINVAL;  in bpf_prog_attach_check_attach_type()
1883 		return prog->enforce_expected_attach_type &&  in bpf_prog_attach_check_attach_type()
1884 			prog->expected_attach_type != attach_type ?  in bpf_prog_attach_check_attach_type()
1899 	struct bpf_prog *prog;  in bpf_prog_attach()  local
1961 	prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);  in bpf_prog_attach()
1962 	if (IS_ERR(prog))  in bpf_prog_attach()
1963 		return PTR_ERR(prog);  in bpf_prog_attach()
1965 	if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {  in bpf_prog_attach()
1966 		bpf_prog_put(prog);  in bpf_prog_attach()
1973 		ret = sock_map_get_from_fd(attr, prog);  in bpf_prog_attach()
1976 		ret = lirc_prog_attach(attr, prog);  in bpf_prog_attach()
1979 		ret = skb_flow_dissector_bpf_prog_attach(attr, prog);  in bpf_prog_attach()
1982 		ret = cgroup_bpf_prog_attach(attr, ptype, prog);  in bpf_prog_attach()
1986 		bpf_prog_put(prog);  in bpf_prog_attach()
2099 	struct bpf_prog *prog;  in bpf_prog_test_run()  local
2115 	prog = bpf_prog_get(attr->test.prog_fd);  in bpf_prog_test_run()
2116 	if (IS_ERR(prog))  in bpf_prog_test_run()
2117 		return PTR_ERR(prog);  in bpf_prog_test_run()
2119 	if (prog->aux->ops->test_run)  in bpf_prog_test_run()
2120 		ret = prog->aux->ops->test_run(prog, attr, uattr);  in bpf_prog_test_run()
2122 	bpf_prog_put(prog);  in bpf_prog_test_run()
2158 	struct bpf_prog *prog;  in bpf_prog_get_fd_by_id()  local
2169 	prog = idr_find(&prog_idr, id);  in bpf_prog_get_fd_by_id()
2170 	if (prog)  in bpf_prog_get_fd_by_id()
2171 		prog = bpf_prog_inc_not_zero(prog);  in bpf_prog_get_fd_by_id()
2173 		prog = ERR_PTR(-ENOENT);  in bpf_prog_get_fd_by_id()
2176 	if (IS_ERR(prog))  in bpf_prog_get_fd_by_id()
2177 		return PTR_ERR(prog);  in bpf_prog_get_fd_by_id()
2179 	fd = bpf_prog_new_fd(prog);  in bpf_prog_get_fd_by_id()
2181 		bpf_prog_put(prog);  in bpf_prog_get_fd_by_id()
2224 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,  in bpf_map_from_imm()  argument
2231 	for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {  in bpf_map_from_imm()
2232 		map = prog->aux->used_maps[i];  in bpf_map_from_imm()
2248 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)  in bpf_insn_prepare_dump()  argument
2256 	insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),  in bpf_insn_prepare_dump()
2261 	for (i = 0; i < prog->len; i++) {  in bpf_insn_prepare_dump()
2280 		map = bpf_map_from_imm(prog, imm, &off, &type);  in bpf_insn_prepare_dump()
2323 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,  in bpf_prog_get_info_by_fd()  argument
2343 	info.type = prog->type;  in bpf_prog_get_info_by_fd()
2344 	info.id = prog->aux->id;  in bpf_prog_get_info_by_fd()
2345 	info.load_time = prog->aux->load_time;  in bpf_prog_get_info_by_fd()
2347 					       prog->aux->user->uid);  in bpf_prog_get_info_by_fd()
2348 	info.gpl_compatible = prog->gpl_compatible;  in bpf_prog_get_info_by_fd()
2350 	memcpy(info.tag, prog->tag, sizeof(prog->tag));  in bpf_prog_get_info_by_fd()
2351 	memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));  in bpf_prog_get_info_by_fd()
2354 	info.nr_map_ids = prog->aux->used_map_cnt;  in bpf_prog_get_info_by_fd()
2361 			if (put_user(prog->aux->used_maps[i]->id,  in bpf_prog_get_info_by_fd()
2370 	bpf_prog_get_stats(prog, &stats);  in bpf_prog_get_info_by_fd()
2386 	info.xlated_prog_len = bpf_prog_insn_size(prog);  in bpf_prog_get_info_by_fd()
2391 		if (prog->blinded && !bpf_dump_raw_ok()) {  in bpf_prog_get_info_by_fd()
2395 		insns_sanitized = bpf_insn_prepare_dump(prog);  in bpf_prog_get_info_by_fd()
2406 	if (bpf_prog_is_dev_bound(prog->aux)) {  in bpf_prog_get_info_by_fd()
2407 		err = bpf_prog_offload_info_fill(&info, prog);  in bpf_prog_get_info_by_fd()
2418 	if (prog->aux->func_cnt) {  in bpf_prog_get_info_by_fd()
2422 		for (i = 0; i < prog->aux->func_cnt; i++)  in bpf_prog_get_info_by_fd()
2423 			info.jited_prog_len += prog->aux->func[i]->jited_len;  in bpf_prog_get_info_by_fd()
2425 		info.jited_prog_len = prog->jited_len;  in bpf_prog_get_info_by_fd()
2436 			if (prog->aux->func_cnt) {  in bpf_prog_get_info_by_fd()
2441 				for (i = 0; i < prog->aux->func_cnt; i++) {  in bpf_prog_get_info_by_fd()
2442 					len = prog->aux->func[i]->jited_len;  in bpf_prog_get_info_by_fd()
2444 					img = (u8 *) prog->aux->func[i]->bpf_func;  in bpf_prog_get_info_by_fd()
2453 				if (copy_to_user(uinsns, prog->bpf_func, ulen))  in bpf_prog_get_info_by_fd()
2462 	info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;  in bpf_prog_get_info_by_fd()
2474 			if (prog->aux->func_cnt) {  in bpf_prog_get_info_by_fd()
2477 						prog->aux->func[i]->bpf_func;  in bpf_prog_get_info_by_fd()
2483 				ksym_addr = (unsigned long) prog->bpf_func;  in bpf_prog_get_info_by_fd()
2493 	info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;  in bpf_prog_get_info_by_fd()
2502 			if (prog->aux->func_cnt) {  in bpf_prog_get_info_by_fd()
2505 						prog->aux->func[i]->jited_len;  in bpf_prog_get_info_by_fd()
2510 				func_len = prog->jited_len;  in bpf_prog_get_info_by_fd()
2519 	if (prog->aux->btf)  in bpf_prog_get_info_by_fd()
2520 		info.btf_id = btf_id(prog->aux->btf);  in bpf_prog_get_info_by_fd()
2523 	info.nr_func_info = prog->aux->func_info_cnt;  in bpf_prog_get_info_by_fd()
2529 		if (copy_to_user(user_finfo, prog->aux->func_info,  in bpf_prog_get_info_by_fd()
2535 	info.nr_line_info = prog->aux->nr_linfo;  in bpf_prog_get_info_by_fd()
2541 		if (copy_to_user(user_linfo, prog->aux->linfo,  in bpf_prog_get_info_by_fd()
2547 	if (prog->aux->jited_linfo)  in bpf_prog_get_info_by_fd()
2548 		info.nr_jited_line_info = prog->aux->nr_linfo;  in bpf_prog_get_info_by_fd()
2559 				if (put_user((__u64)(long)prog->aux->jited_linfo[i],  in bpf_prog_get_info_by_fd()
2569 	info.nr_prog_tags = prog->aux->func_cnt ? : 1;  in bpf_prog_get_info_by_fd()
2576 		if (prog->aux->func_cnt) {  in bpf_prog_get_info_by_fd()
2579 						 prog->aux->func[i]->tag,  in bpf_prog_get_info_by_fd()
2585 					 prog->tag, BPF_TAG_SIZE))  in bpf_prog_get_info_by_fd()
2807 					     raw_tp->prog->aux->id,  in bpf_task_fd_query()