25 void cgroup_bpf_offline(struct cgroup *cgrp)  in cgroup_bpf_offline()  argument
27 	cgroup_get(cgrp);  in cgroup_bpf_offline()
28 	percpu_ref_kill(&cgrp->bpf.refcnt);  in cgroup_bpf_offline()
38 	struct cgroup *cgrp = container_of(work, struct cgroup,  in cgroup_bpf_release()  local
46 	for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {  in cgroup_bpf_release()
47 		struct list_head *progs = &cgrp->bpf.progs[type];  in cgroup_bpf_release()
61 				cgrp->bpf.effective[type],  in cgroup_bpf_release()
68 	percpu_ref_exit(&cgrp->bpf.refcnt);  in cgroup_bpf_release()
69 	cgroup_put(cgrp);  in cgroup_bpf_release()
79 	struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);  in cgroup_bpf_release_fn()  local
81 	INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);  in cgroup_bpf_release_fn()
82 	queue_work(system_wq, &cgrp->bpf.release_work);  in cgroup_bpf_release_fn()
105 static bool hierarchy_allows_attach(struct cgroup *cgrp,  in hierarchy_allows_attach()  argument
111 	p = cgroup_parent(cgrp);  in hierarchy_allows_attach()
135 static int compute_effective_progs(struct cgroup *cgrp,  in compute_effective_progs()  argument
142 	struct cgroup *p = cgrp;  in compute_effective_progs()
158 	p = cgrp;  in compute_effective_progs()
179 static void activate_effective_progs(struct cgroup *cgrp,  in activate_effective_progs()  argument
183 	rcu_swap_protected(cgrp->bpf.effective[type], old_array,  in activate_effective_progs()
195 int cgroup_bpf_inherit(struct cgroup *cgrp)  in cgroup_bpf_inherit()  argument
200 #define	NR ARRAY_SIZE(cgrp->bpf.effective)  in cgroup_bpf_inherit()
204 	ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,  in cgroup_bpf_inherit()
210 		INIT_LIST_HEAD(&cgrp->bpf.progs[i]);  in cgroup_bpf_inherit()
213 		if (compute_effective_progs(cgrp, i, &arrays[i]))  in cgroup_bpf_inherit()
217 		activate_effective_progs(cgrp, i, arrays[i]);  in cgroup_bpf_inherit()
224 	percpu_ref_exit(&cgrp->bpf.refcnt);  in cgroup_bpf_inherit()
229 static int update_effective_progs(struct cgroup *cgrp,  in update_effective_progs()  argument
236 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
248 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
269 	css_for_each_descendant_pre(css, &cgrp->self) {  in update_effective_progs()
291 int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,  in __cgroup_bpf_attach()  argument
294 	struct list_head *progs = &cgrp->bpf.progs[type];  in __cgroup_bpf_attach()
307 	if (!hierarchy_allows_attach(cgrp, type, flags))  in __cgroup_bpf_attach()
310 	if (!list_empty(progs) && cgrp->bpf.flags[type] != flags)  in __cgroup_bpf_attach()
376 	cgrp->bpf.flags[type] = flags;  in __cgroup_bpf_attach()
378 	err = update_effective_progs(cgrp, type);  in __cgroup_bpf_attach()
393 		bpf_cgroup_storage_link(storage[stype], cgrp, type);  in __cgroup_bpf_attach()
402 		bpf_cgroup_storage_link(old_storage[stype], cgrp, type);  in __cgroup_bpf_attach()
420 int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in __cgroup_bpf_detach()  argument
423 	struct list_head *progs = &cgrp->bpf.progs[type];  in __cgroup_bpf_detach()
425 	u32 flags = cgrp->bpf.flags[type];  in __cgroup_bpf_detach()
465 	err = update_effective_progs(cgrp, type);  in __cgroup_bpf_detach()
478 		cgrp->bpf.flags[type] = 0;  in __cgroup_bpf_detach()
491 int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,  in __cgroup_bpf_query()  argument
496 	struct list_head *progs = &cgrp->bpf.progs[type];  in __cgroup_bpf_query()
497 	u32 flags = cgrp->bpf.flags[type];  in __cgroup_bpf_query()
501 	effective = rcu_dereference_protected(cgrp->bpf.effective[type],  in __cgroup_bpf_query()
542 	struct cgroup *cgrp;  in cgroup_bpf_prog_attach()  local
545 	cgrp = cgroup_get_from_fd(attr->target_fd);  in cgroup_bpf_prog_attach()
546 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_attach()
547 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_attach()
549 	ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type,  in cgroup_bpf_prog_attach()
551 	cgroup_put(cgrp);  in cgroup_bpf_prog_attach()
558 	struct cgroup *cgrp;  in cgroup_bpf_prog_detach()  local
561 	cgrp = cgroup_get_from_fd(attr->target_fd);  in cgroup_bpf_prog_detach()
562 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_detach()
563 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_detach()
569 	ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0);  in cgroup_bpf_prog_detach()
573 	cgroup_put(cgrp);  in cgroup_bpf_prog_detach()
580 	struct cgroup *cgrp;  in cgroup_bpf_prog_query()  local
583 	cgrp = cgroup_get_from_fd(attr->query.target_fd);  in cgroup_bpf_prog_query()
584 	if (IS_ERR(cgrp))  in cgroup_bpf_prog_query()
585 		return PTR_ERR(cgrp);  in cgroup_bpf_prog_query()
587 	ret = cgroup_bpf_query(cgrp, attr, uattr);  in cgroup_bpf_prog_query()
589 	cgroup_put(cgrp);  in cgroup_bpf_prog_query()
623 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_skb()  local
632 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_skb()
642 			cgrp->bpf.effective[type], skb, __bpf_prog_run_save_cb);  in __cgroup_bpf_run_filter_skb()
644 		ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,  in __cgroup_bpf_run_filter_skb()
672 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sk()  local
675 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN);  in __cgroup_bpf_run_filter_sk()
704 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_sock_addr()  local
718 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sock_addr()
719 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);  in __cgroup_bpf_run_filter_sock_addr()
745 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_sock_ops()  local
748 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops,  in __cgroup_bpf_run_filter_sock_ops()
757 	struct cgroup *cgrp;  in __cgroup_bpf_check_dev_permission()  local
766 	cgrp = task_dfl_cgroup(current);  in __cgroup_bpf_check_dev_permission()
767 	allow = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx,  in __cgroup_bpf_check_dev_permission()
891 	struct cgroup *cgrp;  in __cgroup_bpf_run_filter_sysctl()  local
925 	cgrp = task_dfl_cgroup(current);  in __cgroup_bpf_run_filter_sysctl()
926 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);  in __cgroup_bpf_run_filter_sysctl()
943 static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,  in __cgroup_bpf_prog_array_is_empty()  argument
950 	prog_array = rcu_dereference(cgrp->bpf.effective[attach_type]);  in __cgroup_bpf_prog_array_is_empty()
980 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_setsockopt()  local
993 	    __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))  in __cgroup_bpf_run_filter_setsockopt()
1014 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_SETSOCKOPT],  in __cgroup_bpf_run_filter_setsockopt()
1052 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);  in __cgroup_bpf_run_filter_getsockopt()  local
1066 	    __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))  in __cgroup_bpf_run_filter_getsockopt()
1098 	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],  in __cgroup_bpf_run_filter_getsockopt()