970 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)  in find_prog_type()  argument
981 	if (!bpf_prog_is_dev_bound(prog->aux))  in find_prog_type()
982 		prog->aux->ops = ops;  in find_prog_type()
984 		prog->aux->ops = &bpf_offload_prog_ops;  in find_prog_type()
985 	prog->type = type;  in find_prog_type()
995 		bpf_cgroup_storage_release(aux->prog, aux->cgroup_storage);  in free_used_maps()
1025 static int bpf_prog_charge_memlock(struct bpf_prog *prog)  in bpf_prog_charge_memlock()  argument
1030 	ret = __bpf_prog_charge(user, prog->pages);  in bpf_prog_charge_memlock()
1036 	prog->aux->user = user;  in bpf_prog_charge_memlock()
1040 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)  in bpf_prog_uncharge_memlock()  argument
1042 	struct user_struct *user = prog->aux->user;  in bpf_prog_uncharge_memlock()
1044 	__bpf_prog_uncharge(user, prog->pages);  in bpf_prog_uncharge_memlock()
1048 static int bpf_prog_alloc_id(struct bpf_prog *prog)  in bpf_prog_alloc_id()  argument
1054 	id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);  in bpf_prog_alloc_id()
1056 		prog->aux->id = id;  in bpf_prog_alloc_id()
1067 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)  in bpf_prog_free_id()  argument
1074 	if (!prog->aux->id)  in bpf_prog_free_id()
1082 	idr_remove(&prog_idr, prog->aux->id);  in bpf_prog_free_id()
1083 	prog->aux->id = 0;  in bpf_prog_free_id()
1096 	bpf_prog_uncharge_memlock(aux->prog);  in __bpf_prog_put_rcu()
1098 	bpf_prog_free(aux->prog);  in __bpf_prog_put_rcu()
1101 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)  in __bpf_prog_put()  argument
1103 	if (atomic_dec_and_test(&prog->aux->refcnt)) {  in __bpf_prog_put()
1105 		bpf_prog_free_id(prog, do_idr_lock);  in __bpf_prog_put()
1106 		bpf_prog_kallsyms_del_all(prog);  in __bpf_prog_put()
1108 		call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);  in __bpf_prog_put()
1112 void bpf_prog_put(struct bpf_prog *prog)  in bpf_prog_put()  argument
1114 	__bpf_prog_put(prog, true);  in bpf_prog_put()
1120 	struct bpf_prog *prog = filp->private_data;  in bpf_prog_release()  local
1122 	bpf_prog_put(prog);  in bpf_prog_release()
1129 	const struct bpf_prog *prog = filp->private_data;  in bpf_prog_show_fdinfo()  local
1130 	char prog_tag[sizeof(prog->tag) * 2 + 1] = { };  in bpf_prog_show_fdinfo()
1132 	bin2hex(prog_tag, prog->tag, sizeof(prog->tag));  in bpf_prog_show_fdinfo()
1139 		   prog->type,  in bpf_prog_show_fdinfo()
1140 		   prog->jited,  in bpf_prog_show_fdinfo()
1142 		   prog->pages * 1ULL << PAGE_SHIFT,  in bpf_prog_show_fdinfo()
1143 		   prog->aux->id);  in bpf_prog_show_fdinfo()
1156 int bpf_prog_new_fd(struct bpf_prog *prog)  in bpf_prog_new_fd()  argument
1160 	ret = security_bpf_prog(prog);  in bpf_prog_new_fd()
1164 	return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,  in bpf_prog_new_fd()
1180 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)  in bpf_prog_add()  argument
1182 	if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {  in bpf_prog_add()
1183 		atomic_sub(i, &prog->aux->refcnt);  in bpf_prog_add()
1186 	return prog;  in bpf_prog_add()
1190 void bpf_prog_sub(struct bpf_prog *prog, int i)  in bpf_prog_sub()  argument
1197 	WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);  in bpf_prog_sub()
1201 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)  in bpf_prog_inc()  argument
1203 	return bpf_prog_add(prog, 1);  in bpf_prog_inc()
1208 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)  in bpf_prog_inc_not_zero()  argument
1212 	refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0);  in bpf_prog_inc_not_zero()
1215 		__bpf_prog_put(prog, false);  in bpf_prog_inc_not_zero()
1222 	return prog;  in bpf_prog_inc_not_zero()
1226 bool bpf_prog_get_ok(struct bpf_prog *prog,  in bpf_prog_get_ok()  argument
1233 	if (prog->type != *attach_type)  in bpf_prog_get_ok()
1235 	if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)  in bpf_prog_get_ok()
1245 	struct bpf_prog *prog;  in __bpf_prog_get()  local
1247 	prog = ____bpf_prog_get(f);  in __bpf_prog_get()
1248 	if (IS_ERR(prog))  in __bpf_prog_get()
1249 		return prog;  in __bpf_prog_get()
1250 	if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {  in __bpf_prog_get()
1251 		prog = ERR_PTR(-EINVAL);  in __bpf_prog_get()
1255 	prog = bpf_prog_inc(prog);  in __bpf_prog_get()
1258 	return prog;  in __bpf_prog_get()
1336 	struct bpf_prog *prog;  in bpf_prog_load()  local
1373 	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);  in bpf_prog_load()
1374 	if (!prog)  in bpf_prog_load()
1377 	prog->expected_attach_type = attr->expected_attach_type;  in bpf_prog_load()
1379 	prog->aux->offload_requested = !!attr->prog_ifindex;  in bpf_prog_load()
1381 	err = security_bpf_prog_alloc(prog->aux);  in bpf_prog_load()
1385 	err = bpf_prog_charge_memlock(prog);  in bpf_prog_load()
1389 	prog->len = attr->insn_cnt;  in bpf_prog_load()
1392 	if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),  in bpf_prog_load()
1393 			   bpf_prog_insn_size(prog)) != 0)  in bpf_prog_load()
1396 	prog->orig_prog = NULL;  in bpf_prog_load()
1397 	prog->jited = 0;  in bpf_prog_load()
1399 	atomic_set(&prog->aux->refcnt, 1);  in bpf_prog_load()
1400 	prog->gpl_compatible = is_gpl ? 1 : 0;  in bpf_prog_load()
1402 	if (bpf_prog_is_dev_bound(prog->aux)) {  in bpf_prog_load()
1403 		err = bpf_prog_offload_init(prog, attr);  in bpf_prog_load()
1409 	err = find_prog_type(type, prog);  in bpf_prog_load()
1413 	prog->aux->load_time = ktime_get_boot_ns();  in bpf_prog_load()
1414 	err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);  in bpf_prog_load()
1419 	err = bpf_check(&prog, attr);  in bpf_prog_load()
1423 	prog = bpf_prog_select_runtime(prog, &err);  in bpf_prog_load()
1427 	err = bpf_prog_alloc_id(prog);  in bpf_prog_load()
1431 	err = bpf_prog_new_fd(prog);  in bpf_prog_load()
1439 		bpf_prog_put(prog);  in bpf_prog_load()
1443 	bpf_prog_kallsyms_add(prog);  in bpf_prog_load()
1447 	bpf_prog_kallsyms_del_subprogs(prog);  in bpf_prog_load()
1448 	free_used_maps(prog->aux);  in bpf_prog_load()
1450 	bpf_prog_uncharge_memlock(prog);  in bpf_prog_load()
1452 	security_bpf_prog_free(prog->aux);  in bpf_prog_load()
1454 	bpf_prog_free(prog);  in bpf_prog_load()
1480 	struct bpf_prog *prog;  member
1487 	if (raw_tp->prog) {  in bpf_raw_tracepoint_release()
1488 		bpf_probe_unregister(raw_tp->btp, raw_tp->prog);  in bpf_raw_tracepoint_release()
1489 		bpf_prog_put(raw_tp->prog);  in bpf_raw_tracepoint_release()
1507 	struct bpf_prog *prog;  in bpf_raw_tracepoint_open()  local
1525 	prog = bpf_prog_get_type(attr->raw_tracepoint.prog_fd,  in bpf_raw_tracepoint_open()
1527 	if (IS_ERR(prog)) {  in bpf_raw_tracepoint_open()
1528 		err = PTR_ERR(prog);  in bpf_raw_tracepoint_open()
1532 	err = bpf_probe_register(raw_tp->btp, prog);  in bpf_raw_tracepoint_open()
1536 	raw_tp->prog = prog;  in bpf_raw_tracepoint_open()
1540 		bpf_probe_unregister(raw_tp->btp, prog);  in bpf_raw_tracepoint_open()
1547 	bpf_prog_put(prog);  in bpf_raw_tracepoint_open()
1553 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,  in bpf_prog_attach_check_attach_type()  argument
1556 	switch (prog->type) {  in bpf_prog_attach_check_attach_type()
1559 		return attach_type == prog->expected_attach_type ? 0 : -EINVAL;  in bpf_prog_attach_check_attach_type()
1573 	struct bpf_prog *prog;  in bpf_prog_attach()  local
1623 	prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);  in bpf_prog_attach()
1624 	if (IS_ERR(prog))  in bpf_prog_attach()
1625 		return PTR_ERR(prog);  in bpf_prog_attach()
1627 	if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {  in bpf_prog_attach()
1628 		bpf_prog_put(prog);  in bpf_prog_attach()
1635 		ret = sockmap_get_from_fd(attr, ptype, prog);  in bpf_prog_attach()
1638 		ret = lirc_prog_attach(attr, prog);  in bpf_prog_attach()
1641 		ret = cgroup_bpf_prog_attach(attr, ptype, prog);  in bpf_prog_attach()
1645 		bpf_prog_put(prog);  in bpf_prog_attach()
1740 	struct bpf_prog *prog;  in bpf_prog_test_run()  local
1748 	prog = bpf_prog_get(attr->test.prog_fd);  in bpf_prog_test_run()
1749 	if (IS_ERR(prog))  in bpf_prog_test_run()
1750 		return PTR_ERR(prog);  in bpf_prog_test_run()
1752 	if (prog->aux->ops->test_run)  in bpf_prog_test_run()
1753 		ret = prog->aux->ops->test_run(prog, attr, uattr);  in bpf_prog_test_run()
1755 	bpf_prog_put(prog);  in bpf_prog_test_run()
1791 	struct bpf_prog *prog;  in bpf_prog_get_fd_by_id()  local
1802 	prog = idr_find(&prog_idr, id);  in bpf_prog_get_fd_by_id()
1803 	if (prog)  in bpf_prog_get_fd_by_id()
1804 		prog = bpf_prog_inc_not_zero(prog);  in bpf_prog_get_fd_by_id()
1806 		prog = ERR_PTR(-ENOENT);  in bpf_prog_get_fd_by_id()
1809 	if (IS_ERR(prog))  in bpf_prog_get_fd_by_id()
1810 		return PTR_ERR(prog);  in bpf_prog_get_fd_by_id()
1812 	fd = bpf_prog_new_fd(prog);  in bpf_prog_get_fd_by_id()
1814 		bpf_prog_put(prog);  in bpf_prog_get_fd_by_id()
1857 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,  in bpf_map_from_imm()  argument
1862 	for (i = 0; i < prog->aux->used_map_cnt; i++)  in bpf_map_from_imm()
1863 		if (prog->aux->used_maps[i] == (void *)addr)  in bpf_map_from_imm()
1864 			return prog->aux->used_maps[i];  in bpf_map_from_imm()
1868 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)  in bpf_insn_prepare_dump()  argument
1875 	insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),  in bpf_insn_prepare_dump()
1880 	for (i = 0; i < prog->len; i++) {  in bpf_insn_prepare_dump()
1899 		map = bpf_map_from_imm(prog, imm);  in bpf_insn_prepare_dump()
1908 		    imm == (unsigned long)prog->aux) {  in bpf_insn_prepare_dump()
1918 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,  in bpf_prog_get_info_by_fd()  argument
1937 	info.type = prog->type;  in bpf_prog_get_info_by_fd()
1938 	info.id = prog->aux->id;  in bpf_prog_get_info_by_fd()
1939 	info.load_time = prog->aux->load_time;  in bpf_prog_get_info_by_fd()
1941 					       prog->aux->user->uid);  in bpf_prog_get_info_by_fd()
1942 	info.gpl_compatible = prog->gpl_compatible;  in bpf_prog_get_info_by_fd()
1944 	memcpy(info.tag, prog->tag, sizeof(prog->tag));  in bpf_prog_get_info_by_fd()
1945 	memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));  in bpf_prog_get_info_by_fd()
1948 	info.nr_map_ids = prog->aux->used_map_cnt;  in bpf_prog_get_info_by_fd()
1955 			if (put_user(prog->aux->used_maps[i]->id,  in bpf_prog_get_info_by_fd()
1968 	info.xlated_prog_len = bpf_prog_insn_size(prog);  in bpf_prog_get_info_by_fd()
1973 		if (prog->blinded && !bpf_dump_raw_ok()) {  in bpf_prog_get_info_by_fd()
1977 		insns_sanitized = bpf_insn_prepare_dump(prog);  in bpf_prog_get_info_by_fd()
1988 	if (bpf_prog_is_dev_bound(prog->aux)) {  in bpf_prog_get_info_by_fd()
1989 		err = bpf_prog_offload_info_fill(&info, prog);  in bpf_prog_get_info_by_fd()
2000 	if (prog->aux->func_cnt) {  in bpf_prog_get_info_by_fd()
2004 		for (i = 0; i < prog->aux->func_cnt; i++)  in bpf_prog_get_info_by_fd()
2005 			info.jited_prog_len += prog->aux->func[i]->jited_len;  in bpf_prog_get_info_by_fd()
2007 		info.jited_prog_len = prog->jited_len;  in bpf_prog_get_info_by_fd()
2018 			if (prog->aux->func_cnt) {  in bpf_prog_get_info_by_fd()
2023 				for (i = 0; i < prog->aux->func_cnt; i++) {  in bpf_prog_get_info_by_fd()
2024 					len = prog->aux->func[i]->jited_len;  in bpf_prog_get_info_by_fd()
2026 					img = (u8 *) prog->aux->func[i]->bpf_func;  in bpf_prog_get_info_by_fd()
2035 				if (copy_to_user(uinsns, prog->bpf_func, ulen))  in bpf_prog_get_info_by_fd()
2044 	info.nr_jited_ksyms = prog->aux->func_cnt;  in bpf_prog_get_info_by_fd()
2057 				ksym_addr = (ulong) prog->aux->func[i]->bpf_func;  in bpf_prog_get_info_by_fd()
2068 	info.nr_jited_func_lens = prog->aux->func_cnt;  in bpf_prog_get_info_by_fd()
2078 				func_len = prog->aux->func[i]->jited_len;  in bpf_prog_get_info_by_fd()
2304 					     raw_tp->prog->aux->id,  in bpf_task_fd_query()