31 bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,  in bpf_prog_run_array_cg()  argument
46 	array = rcu_dereference(cgrp->effective[atype]);  in bpf_prog_run_array_cg()
71 	struct cgroup *cgrp;  in __cgroup_bpf_run_lsm_sock()  local
80 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_lsm_sock()
81 	if (likely(cgrp))  in __cgroup_bpf_run_lsm_sock()
82 		ret = bpf_prog_run_array_cg(&cgrp->bpf,  in __cgroup_bpf_run_lsm_sock()
93 	struct cgroup *cgrp;  in __cgroup_bpf_run_lsm_socket()  local
102 	cgrp = sock_cgroup_ptr(&sock->sk->sk_cgrp_data);  in __cgroup_bpf_run_lsm_socket()
103 	if (likely(cgrp))  in __cgroup_bpf_run_lsm_socket()
104 		ret = bpf_prog_run_array_cg(&cgrp->bpf,  in __cgroup_bpf_run_lsm_socket()
114 	struct cgroup *cgrp;  in __cgroup_bpf_run_lsm_current()  local
121 	cgrp = task_dfl_cgroup(current);  in __cgroup_bpf_run_lsm_current()
122 	if (likely(cgrp))  in __cgroup_bpf_run_lsm_current()
123 		ret = bpf_prog_run_array_cg(&cgrp->bpf,  in __cgroup_bpf_run_lsm_current()
192 void cgroup_bpf_offline(struct cgroup *cgrp)  in cgroup_bpf_offline()  argument
194 	cgroup_get(cgrp);  in cgroup_bpf_offline()
195 	percpu_ref_kill(&cgrp->bpf.refcnt);  in cgroup_bpf_offline()
210 				     struct cgroup *cgrp)  in bpf_cgroup_storages_alloc()  argument
216 	key.cgroup_inode_id = cgroup_id(cgrp);  in bpf_cgroup_storages_alloc()
250 				     struct cgroup *cgrp,  in bpf_cgroup_storages_link()  argument
256 		bpf_cgroup_storage_link(storages[stype], cgrp, attach_type);  in bpf_cgroup_storages_link()
277 	struct cgroup *p, *cgrp = container_of(work, struct cgroup,  in cgroup_bpf_release()  local
280 	struct list_head *storages = &cgrp->bpf.storages;  in cgroup_bpf_release()
287 	for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) {  in cgroup_bpf_release()
288 		struct hlist_head *progs = &cgrp->bpf.progs[atype];  in cgroup_bpf_release()
308 				cgrp->bpf.effective[atype],  in cgroup_bpf_release()
320 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))  in cgroup_bpf_release()
323 	percpu_ref_exit(&cgrp->bpf.refcnt);  in cgroup_bpf_release()
324 	cgroup_put(cgrp);  in cgroup_bpf_release()
334 	struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);  in cgroup_bpf_release_fn()  local
336 	INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);  in cgroup_bpf_release_fn()
337 	queue_work(system_wq, &cgrp->bpf.release_work);  in cgroup_bpf_release_fn()
372 static bool hierarchy_allows_attach(struct cgroup *cgrp,  in hierarchy_allows_attach()  argument
377 	p = cgroup_parent(cgrp);  in hierarchy_allows_attach()
401 static int compute_effective_progs(struct cgroup *cgrp,  in compute_effective_progs()  argument
408 	struct cgroup *p = cgrp;  in compute_effective_progs()
424 	p = cgrp;  in compute_effective_progs()
445 static void activate_effective_progs(struct cgroup *cgrp,  in activate_effective_progs()  argument
449 	old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array,  in activate_effective_progs()
461 int cgroup_bpf_inherit(struct cgroup *cgrp)  in cgroup_bpf_inherit()  argument
466 #define	NR ARRAY_SIZE(cgrp->bpf.effective)  in cgroup_bpf_inherit()
471 	ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,  in cgroup_bpf_inherit()
476 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))  in cgroup_bpf_inherit()
480 		INIT_HLIST_HEAD(&cgrp->bpf.progs[i]);  in cgroup_bpf_inherit()
482 	INIT_LIST_HEAD(&cgrp->bpf.storages);  in cgroup_bpf_inherit()
485 		if (compute_effective_progs(cgrp, i, &arrays[i]))  in cgroup_bpf_inherit()
489 		activate_effective_progs(cgrp, i, arrays[i]);  in cgroup_bpf_inherit()
496 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))  in cgroup_bpf_inherit()
499 	percpu_ref_exit(&cgrp->bpf.refcnt);  in cgroup_bpf_inherit()
504 static int update_effective_progs(struct cgroup *cgrp,  in update_effective_progs()  argument
511 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
523 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
544 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
607 static int __cgroup_bpf_attach(struct cgroup *cgrp,  in __cgroup_bpf_attach()  argument
637 	progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_attach()
639 	if (!hierarchy_allows_attach(cgrp, atype))  in __cgroup_bpf_attach()
642 	if (!hlist_empty(progs) && cgrp->bpf.flags[atype] != saved_flags)  in __cgroup_bpf_attach()
658 				      prog ? : link->link.prog, cgrp))  in __cgroup_bpf_attach()
685 	cgrp->bpf.flags[atype] = saved_flags;  in __cgroup_bpf_attach()
693 	err = update_effective_progs(cgrp, atype);  in __cgroup_bpf_attach()
704 	bpf_cgroup_storages_link(new_storage, cgrp, type);  in __cgroup_bpf_attach()
724 static int cgroup_bpf_attach(struct cgroup *cgrp,  in cgroup_bpf_attach()  argument
733 	ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags);  in cgroup_bpf_attach()
741 static void replace_effective_prog(struct cgroup *cgrp,  in replace_effective_prog()  argument
753 	css_for_each_descendant_pre(css, &cgrp->self) {  in replace_effective_prog()
792 static int __cgroup_bpf_replace(struct cgroup *cgrp,  in __cgroup_bpf_replace()  argument
806 	progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_replace()
821 	replace_effective_prog(cgrp, atype, link);  in __cgroup_bpf_replace()
892 static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog,  in purge_effective_progs()  argument
904 	css_for_each_descendant_pre(css, &cgrp->self) {  in purge_effective_progs()
949 static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in __cgroup_bpf_detach()  argument
968 	progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_detach()
969 	flags = cgrp->bpf.flags[atype];  in __cgroup_bpf_detach()
984 	if (update_effective_progs(cgrp, atype)) {  in __cgroup_bpf_detach()
988 		purge_effective_progs(cgrp, old_prog, link, atype);  in __cgroup_bpf_detach()
997 		cgrp->bpf.flags[atype] = 0;  in __cgroup_bpf_detach()
1007 static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in cgroup_bpf_detach()  argument
1013 	ret = __cgroup_bpf_detach(cgrp, prog, NULL, type);  in cgroup_bpf_detach()
1019 static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,  in __cgroup_bpf_query()  argument
1049 		flags = cgrp->bpf.flags[from_atype];  in __cgroup_bpf_query()
1054 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],  in __cgroup_bpf_query()
1058 			total_cnt += prog_list_length(&cgrp->bpf.progs[atype]);  in __cgroup_bpf_query()
1079 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],  in __cgroup_bpf_query()
1089 			progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_query()
1102 				flags = cgrp->bpf.flags[atype];  in __cgroup_bpf_query()
1118 static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,  in cgroup_bpf_query()  argument
1124 	ret = __cgroup_bpf_query(cgrp, attr, uattr);  in cgroup_bpf_query()
1133 	struct cgroup *cgrp;  in cgroup_bpf_prog_attach()  local
1136 	cgrp = cgroup_get_from_fd(attr->target_fd);  in cgroup_bpf_prog_attach()
1137 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_attach()
1138 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_attach()
1144 			cgroup_put(cgrp);  in cgroup_bpf_prog_attach()
1149 	ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL,  in cgroup_bpf_prog_attach()
1154 	cgroup_put(cgrp);  in cgroup_bpf_prog_attach()
1161 	struct cgroup *cgrp;  in cgroup_bpf_prog_detach()  local
1164 	cgrp = cgroup_get_from_fd(attr->target_fd);  in cgroup_bpf_prog_detach()
1165 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_detach()
1166 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_detach()
1172 	ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type);  in cgroup_bpf_prog_detach()
1176 	cgroup_put(cgrp);  in cgroup_bpf_prog_detach()
1277 	struct cgroup *cgrp;  in cgroup_bpf_link_attach()  local
1283 	cgrp = cgroup_get_from_fd(attr->link_create.target_fd);  in cgroup_bpf_link_attach()
1284 	if (IS_ERR(cgrp))  in cgroup_bpf_link_attach()
1285 		return PTR_ERR(cgrp);  in cgroup_bpf_link_attach()
1294 	link->cgroup = cgrp;  in cgroup_bpf_link_attach()
1303 	err = cgroup_bpf_attach(cgrp, NULL, NULL, link,  in cgroup_bpf_link_attach()
1313 	cgroup_put(cgrp);  in cgroup_bpf_link_attach()
1320 	struct cgroup *cgrp;  in cgroup_bpf_prog_query()  local
1323 	cgrp = cgroup_get_from_fd(attr->query.target_fd);  in cgroup_bpf_prog_query()
1324 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_query()
1325 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_query()
1327 	ret = cgroup_bpf_query(cgrp, attr, uattr);  in cgroup_bpf_prog_query()
1329 	cgroup_put(cgrp);  in cgroup_bpf_prog_query()
1363 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_skb()  local
1372 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_skb()
1384 		ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, skb,  in __cgroup_bpf_run_filter_skb()
1410 		ret = bpf_prog_run_array_cg(&cgrp->bpf, atype,  in __cgroup_bpf_run_filter_skb()
1440 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sk()  local
1442 	return bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0,  in __cgroup_bpf_run_filter_sk()
1474 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_sock_addr()  local
1487 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sock_addr()
1488 	return bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run,  in __cgroup_bpf_run_filter_sock_addr()
1513 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sock_ops()  local
1515 	return bpf_prog_run_array_cg(&cgrp->bpf, atype, sock_ops, bpf_prog_run,  in __cgroup_bpf_run_filter_sock_ops()
1523 	struct cgroup *cgrp;  in __cgroup_bpf_check_dev_permission()  local
1532 	cgrp = task_dfl_cgroup(current);  in __cgroup_bpf_check_dev_permission()
1533 	ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,  in __cgroup_bpf_check_dev_permission()
1697 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_sysctl()  local
1723 	cgrp = task_dfl_cgroup(current);  in __cgroup_bpf_run_filter_sysctl()
1724 	ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,  in __cgroup_bpf_run_filter_sysctl()
1791 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_setsockopt()  local
1817 	ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_SETSOCKOPT,  in __cgroup_bpf_run_filter_setsockopt()
1876 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_getsockopt()  local
1917 	ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,  in __cgroup_bpf_run_filter_getsockopt()
1946 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_getsockopt_kern()  local
1966 	ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,  in __cgroup_bpf_run_filter_getsockopt_kern()