25 void cgroup_bpf_offline(struct cgroup *cgrp)  in cgroup_bpf_offline()  argument
27 	cgroup_get(cgrp);  in cgroup_bpf_offline()
28 	percpu_ref_kill(&cgrp->bpf.refcnt);  in cgroup_bpf_offline()
43 				     struct cgroup *cgrp)  in bpf_cgroup_storages_alloc()  argument
49 	key.cgroup_inode_id = cgroup_id(cgrp);  in bpf_cgroup_storages_alloc()
83 				     struct cgroup *cgrp,  in bpf_cgroup_storages_link()  argument
89 		bpf_cgroup_storage_link(storages[stype], cgrp, attach_type);  in bpf_cgroup_storages_link()
110 	struct cgroup *p, *cgrp = container_of(work, struct cgroup,  in cgroup_bpf_release()  local
113 	struct list_head *storages = &cgrp->bpf.storages;  in cgroup_bpf_release()
120 	for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {  in cgroup_bpf_release()
121 		struct list_head *progs = &cgrp->bpf.progs[type];  in cgroup_bpf_release()
134 				cgrp->bpf.effective[type],  in cgroup_bpf_release()
146 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))  in cgroup_bpf_release()
149 	percpu_ref_exit(&cgrp->bpf.refcnt);  in cgroup_bpf_release()
150 	cgroup_put(cgrp);  in cgroup_bpf_release()
160 	struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);  in cgroup_bpf_release_fn()  local
162 	INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);  in cgroup_bpf_release_fn()
163 	queue_work(system_wq, &cgrp->bpf.release_work);  in cgroup_bpf_release_fn()
198 static bool hierarchy_allows_attach(struct cgroup *cgrp,  in hierarchy_allows_attach()  argument
203 	p = cgroup_parent(cgrp);  in hierarchy_allows_attach()
227 static int compute_effective_progs(struct cgroup *cgrp,  in compute_effective_progs()  argument
234 	struct cgroup *p = cgrp;  in compute_effective_progs()
250 	p = cgrp;  in compute_effective_progs()
271 static void activate_effective_progs(struct cgroup *cgrp,  in activate_effective_progs()  argument
275 	old_array = rcu_replace_pointer(cgrp->bpf.effective[type], old_array,  in activate_effective_progs()
287 int cgroup_bpf_inherit(struct cgroup *cgrp)  in cgroup_bpf_inherit()  argument
292 #define	NR ARRAY_SIZE(cgrp->bpf.effective)  in cgroup_bpf_inherit()
297 	ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,  in cgroup_bpf_inherit()
302 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))  in cgroup_bpf_inherit()
306 		INIT_LIST_HEAD(&cgrp->bpf.progs[i]);  in cgroup_bpf_inherit()
308 	INIT_LIST_HEAD(&cgrp->bpf.storages);  in cgroup_bpf_inherit()
311 		if (compute_effective_progs(cgrp, i, &arrays[i]))  in cgroup_bpf_inherit()
315 		activate_effective_progs(cgrp, i, arrays[i]);  in cgroup_bpf_inherit()
322 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))  in cgroup_bpf_inherit()
325 	percpu_ref_exit(&cgrp->bpf.refcnt);  in cgroup_bpf_inherit()
330 static int update_effective_progs(struct cgroup *cgrp,  in update_effective_progs()  argument
337 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
349 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
370 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
433 int __cgroup_bpf_attach(struct cgroup *cgrp,  in __cgroup_bpf_attach()  argument
439 	struct list_head *progs = &cgrp->bpf.progs[type];  in __cgroup_bpf_attach()
457 	if (!hierarchy_allows_attach(cgrp, type))  in __cgroup_bpf_attach()
460 	if (!list_empty(progs) && cgrp->bpf.flags[type] != saved_flags)  in __cgroup_bpf_attach()
476 				      prog ? : link->link.prog, cgrp))  in __cgroup_bpf_attach()
493 	cgrp->bpf.flags[type] = saved_flags;  in __cgroup_bpf_attach()
495 	err = update_effective_progs(cgrp, type);  in __cgroup_bpf_attach()
503 	bpf_cgroup_storages_link(new_storage, cgrp, type);  in __cgroup_bpf_attach()
522 static void replace_effective_prog(struct cgroup *cgrp,  in replace_effective_prog()  argument
534 	css_for_each_descendant_pre(css, &cgrp->self) {  in replace_effective_prog()
573 static int __cgroup_bpf_replace(struct cgroup *cgrp,  in __cgroup_bpf_replace()  argument
577 	struct list_head *progs = &cgrp->bpf.progs[link->type];  in __cgroup_bpf_replace()
595 	replace_effective_prog(cgrp, link->type, link);  in __cgroup_bpf_replace()
667 int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in __cgroup_bpf_detach()  argument
670 	struct list_head *progs = &cgrp->bpf.progs[type];  in __cgroup_bpf_detach()
671 	u32 flags = cgrp->bpf.flags[type];  in __cgroup_bpf_detach()
689 	err = update_effective_progs(cgrp, type);  in __cgroup_bpf_detach()
698 		cgrp->bpf.flags[type] = 0;  in __cgroup_bpf_detach()
712 int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,  in __cgroup_bpf_query()  argument
717 	struct list_head *progs = &cgrp->bpf.progs[type];  in __cgroup_bpf_query()
718 	u32 flags = cgrp->bpf.flags[type];  in __cgroup_bpf_query()
723 	effective = rcu_dereference_protected(cgrp->bpf.effective[type],  in __cgroup_bpf_query()
766 	struct cgroup *cgrp;  in cgroup_bpf_prog_attach()  local
769 	cgrp = cgroup_get_from_fd(attr->target_fd);  in cgroup_bpf_prog_attach()
770 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_attach()
771 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_attach()
777 			cgroup_put(cgrp);  in cgroup_bpf_prog_attach()
782 	ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL,  in cgroup_bpf_prog_attach()
787 	cgroup_put(cgrp);  in cgroup_bpf_prog_attach()
794 	struct cgroup *cgrp;  in cgroup_bpf_prog_detach()  local
797 	cgrp = cgroup_get_from_fd(attr->target_fd);  in cgroup_bpf_prog_detach()
798 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_detach()
799 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_detach()
805 	ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type);  in cgroup_bpf_prog_detach()
809 	cgroup_put(cgrp);  in cgroup_bpf_prog_detach()
908 	struct cgroup *cgrp;  in cgroup_bpf_link_attach()  local
914 	cgrp = cgroup_get_from_fd(attr->link_create.target_fd);  in cgroup_bpf_link_attach()
915 	if (IS_ERR(cgrp))  in cgroup_bpf_link_attach()
916 		return PTR_ERR(cgrp);  in cgroup_bpf_link_attach()
925 	link->cgroup = cgrp;  in cgroup_bpf_link_attach()
934 	err = cgroup_bpf_attach(cgrp, NULL, NULL, link, link->type,  in cgroup_bpf_link_attach()
944 	cgroup_put(cgrp);  in cgroup_bpf_link_attach()
951 	struct cgroup *cgrp;  in cgroup_bpf_prog_query()  local
954 	cgrp = cgroup_get_from_fd(attr->query.target_fd);  in cgroup_bpf_prog_query()
955 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_query()
956 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_query()
958 	ret = cgroup_bpf_query(cgrp, attr, uattr);  in cgroup_bpf_prog_query()
960 	cgroup_put(cgrp);  in cgroup_bpf_prog_query()
994 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_skb()  local
1003 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_skb()
1013 			cgrp->bpf.effective[type], skb, __bpf_prog_run_save_cb);  in __cgroup_bpf_run_filter_skb()
1015 		ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,  in __cgroup_bpf_run_filter_skb()
1043 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sk()  local
1046 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN);  in __cgroup_bpf_run_filter_sk()
1075 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_sock_addr()  local
1089 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sock_addr()
1090 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);  in __cgroup_bpf_run_filter_sock_addr()
1116 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sock_ops()  local
1119 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops,  in __cgroup_bpf_run_filter_sock_ops()
1128 	struct cgroup *cgrp;  in __cgroup_bpf_check_dev_permission()  local
1137 	cgrp = task_dfl_cgroup(current);  in __cgroup_bpf_check_dev_permission()
1138 	allow = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx,  in __cgroup_bpf_check_dev_permission()
1243 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_sysctl()  local
1269 	cgrp = task_dfl_cgroup(current);  in __cgroup_bpf_run_filter_sysctl()
1270 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);  in __cgroup_bpf_run_filter_sysctl()
1287 static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,  in __cgroup_bpf_prog_array_is_empty()  argument
1294 	prog_array = rcu_dereference(cgrp->bpf.effective[attach_type]);  in __cgroup_bpf_prog_array_is_empty()
1331 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_setsockopt()  local
1344 	    __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))  in __cgroup_bpf_run_filter_setsockopt()
1365 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_SETSOCKOPT],  in __cgroup_bpf_run_filter_setsockopt()
1408 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_getsockopt()  local
1422 	    __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))  in __cgroup_bpf_run_filter_getsockopt()
1452 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],  in __cgroup_bpf_run_filter_getsockopt()