Lines Matching refs:cgrp
161 struct cgroup_root cgrp_dfl_root = { .cgrp.rstat_cpu = &cgrp_dfl_root_rstat_cpu };
216 static int cgroup_apply_control(struct cgroup *cgrp);
217 static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
220 static int cgroup_destroy_locked(struct cgroup *cgrp);
221 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
226 struct cgroup *cgrp, struct cftype cfts[],
298 bool cgroup_on_dfl(const struct cgroup *cgrp) in cgroup_on_dfl() argument
300 return cgrp->root == &cgrp_dfl_root; in cgroup_on_dfl()
334 static bool cgroup_has_tasks(struct cgroup *cgrp) in cgroup_has_tasks() argument
336 return cgrp->nr_populated_csets; in cgroup_has_tasks()
339 bool cgroup_is_threaded(struct cgroup *cgrp) in cgroup_is_threaded() argument
341 return cgrp->dom_cgrp != cgrp; in cgroup_is_threaded()
345 static bool cgroup_is_mixable(struct cgroup *cgrp) in cgroup_is_mixable() argument
352 return !cgroup_parent(cgrp); in cgroup_is_mixable()
356 static bool cgroup_can_be_thread_root(struct cgroup *cgrp) in cgroup_can_be_thread_root() argument
359 if (cgroup_is_mixable(cgrp)) in cgroup_can_be_thread_root()
363 if (cgroup_is_threaded(cgrp)) in cgroup_can_be_thread_root()
367 if (cgrp->nr_populated_domain_children) in cgroup_can_be_thread_root()
371 if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_can_be_thread_root()
378 bool cgroup_is_thread_root(struct cgroup *cgrp) in cgroup_is_thread_root() argument
381 if (cgroup_is_threaded(cgrp)) in cgroup_is_thread_root()
385 if (cgrp->nr_threaded_children) in cgroup_is_thread_root()
392 if (cgroup_has_tasks(cgrp) && in cgroup_is_thread_root()
393 (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask)) in cgroup_is_thread_root()
400 static bool cgroup_is_valid_domain(struct cgroup *cgrp) in cgroup_is_valid_domain() argument
403 if (cgroup_is_threaded(cgrp)) in cgroup_is_valid_domain()
407 while ((cgrp = cgroup_parent(cgrp))) { in cgroup_is_valid_domain()
408 if (!cgroup_is_mixable(cgrp) && cgroup_is_thread_root(cgrp)) in cgroup_is_valid_domain()
410 if (cgroup_is_threaded(cgrp)) in cgroup_is_valid_domain()
418 static u16 cgroup_control(struct cgroup *cgrp) in cgroup_control() argument
420 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_control()
421 u16 root_ss_mask = cgrp->root->subsys_mask; in cgroup_control()
427 if (cgroup_is_threaded(cgrp)) in cgroup_control()
432 if (cgroup_on_dfl(cgrp)) in cgroup_control()
439 static u16 cgroup_ss_mask(struct cgroup *cgrp) in cgroup_ss_mask() argument
441 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_ss_mask()
447 if (cgroup_is_threaded(cgrp)) in cgroup_ss_mask()
452 return cgrp->root->subsys_mask; in cgroup_ss_mask()
466 static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, in cgroup_css() argument
470 return rcu_dereference_check(cgrp->subsys[ss->id], in cgroup_css()
473 return &cgrp->self; in cgroup_css()
484 static struct cgroup_subsys_state *cgroup_tryget_css(struct cgroup *cgrp, in cgroup_tryget_css() argument
490 css = cgroup_css(cgrp, ss); in cgroup_tryget_css()
508 static struct cgroup_subsys_state *cgroup_e_css_by_mask(struct cgroup *cgrp, in cgroup_e_css_by_mask() argument
514 return &cgrp->self; in cgroup_e_css_by_mask()
520 while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) { in cgroup_e_css_by_mask()
521 cgrp = cgroup_parent(cgrp); in cgroup_e_css_by_mask()
522 if (!cgrp) in cgroup_e_css_by_mask()
526 return cgroup_css(cgrp, ss); in cgroup_e_css_by_mask()
542 struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, in cgroup_e_css() argument
548 css = cgroup_css(cgrp, ss); in cgroup_e_css()
552 cgrp = cgroup_parent(cgrp); in cgroup_e_css()
553 } while (cgrp); in cgroup_e_css()
569 struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, in cgroup_get_e_css() argument
577 css = cgroup_css(cgrp, ss); in cgroup_get_e_css()
581 cgrp = cgroup_parent(cgrp); in cgroup_get_e_css()
582 } while (cgrp); in cgroup_get_e_css()
591 static void cgroup_get_live(struct cgroup *cgrp) in cgroup_get_live() argument
593 WARN_ON_ONCE(cgroup_is_dead(cgrp)); in cgroup_get_live()
594 css_get(&cgrp->self); in cgroup_get_live()
602 int __cgroup_task_count(const struct cgroup *cgrp) in __cgroup_task_count() argument
609 list_for_each_entry(link, &cgrp->cset_links, cset_link) in __cgroup_task_count()
619 int cgroup_task_count(const struct cgroup *cgrp) in cgroup_task_count() argument
624 count = __cgroup_task_count(cgrp); in cgroup_task_count()
632 struct cgroup *cgrp = of->kn->parent->priv; in of_css() local
644 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); in of_css()
646 return &cgrp->self; in of_css()
658 #define for_each_css(css, ssid, cgrp) \ argument
661 (cgrp)->subsys[(ssid)], \
673 #define for_each_e_css(css, ssid, cgrp) \ argument
675 if (!((css) = cgroup_e_css_by_mask(cgrp, \
705 #define cgroup_for_each_live_child(child, cgrp) \ argument
706 list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
713 #define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \ argument
714 css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \
722 #define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \ argument
723 css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \
755 .dfl_cgrp = &cgrp_dfl_root.cgrp,
798 static void cgroup_update_populated(struct cgroup *cgrp, bool populated) in cgroup_update_populated() argument
806 bool was_populated = cgroup_is_populated(cgrp); in cgroup_update_populated()
809 cgrp->nr_populated_csets += adj; in cgroup_update_populated()
812 cgrp->nr_populated_threaded_children += adj; in cgroup_update_populated()
814 cgrp->nr_populated_domain_children += adj; in cgroup_update_populated()
817 if (was_populated == cgroup_is_populated(cgrp)) in cgroup_update_populated()
820 cgroup1_check_for_release(cgrp); in cgroup_update_populated()
821 TRACE_CGROUP_PATH(notify_populated, cgrp, in cgroup_update_populated()
822 cgroup_is_populated(cgrp)); in cgroup_update_populated()
823 cgroup_file_notify(&cgrp->events_file); in cgroup_update_populated()
825 child = cgrp; in cgroup_update_populated()
826 cgrp = cgroup_parent(cgrp); in cgroup_update_populated()
827 } while (cgrp); in cgroup_update_populated()
845 cgroup_update_populated(link->cgrp, populated); in css_set_update_populated()
958 if (cgroup_parent(link->cgrp)) in put_css_set_locked()
959 cgroup_put(link->cgrp); in put_css_set_locked()
1031 cgrp1 = link1->cgrp; in compare_css_sets()
1032 cgrp2 = link2->cgrp; in compare_css_sets()
1061 struct cgroup *cgrp, in find_existing_css_set() argument
1064 struct cgroup_root *root = cgrp->root; in find_existing_css_set()
1081 template[i] = cgroup_e_css_by_mask(cgrp, ss); in find_existing_css_set()
1093 if (!compare_css_sets(cset, old_cset, cgrp, template)) in find_existing_css_set()
1147 struct cgroup *cgrp) in link_css_set() argument
1153 if (cgroup_on_dfl(cgrp)) in link_css_set()
1154 cset->dfl_cgrp = cgrp; in link_css_set()
1158 link->cgrp = cgrp; in link_css_set()
1164 list_move_tail(&link->cset_link, &cgrp->cset_links); in link_css_set()
1167 if (cgroup_parent(cgrp)) in link_css_set()
1168 cgroup_get_live(cgrp); in link_css_set()
1180 struct cgroup *cgrp) in find_css_set() argument
1195 cset = find_existing_css_set(old_cset, cgrp, template); in find_css_set()
1232 struct cgroup *c = link->cgrp; in find_css_set()
1234 if (c->root == cgrp->root) in find_css_set()
1235 c = cgrp; in find_css_set()
1320 struct cgroup *cgrp = &root->cgrp; in cgroup_destroy_root() local
1325 cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); in cgroup_destroy_root()
1328 BUG_ON(!list_empty(&cgrp->self.children)); in cgroup_destroy_root()
1339 list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { in cgroup_destroy_root()
1376 res = &root->cgrp; in current_cgns_cgroup_from_root()
1381 struct cgroup *c = link->cgrp; in current_cgns_cgroup_from_root()
1405 res = &root->cgrp; in cset_cgroup_from_root()
1412 struct cgroup *c = link->cgrp; in cset_cgroup_from_root()
1468 static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, in cgroup_file_name() argument
1474 !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) { in cgroup_file_name()
1478 dbg, cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name, in cgroup_file_name()
1565 struct cgroup *cgrp; in cgroup_kn_unlock() local
1568 cgrp = kn->priv; in cgroup_kn_unlock()
1570 cgrp = kn->parent->priv; in cgroup_kn_unlock()
1575 cgroup_put(cgrp); in cgroup_kn_unlock()
1597 struct cgroup *cgrp; in cgroup_kn_lock_live() local
1600 cgrp = kn->priv; in cgroup_kn_lock_live()
1602 cgrp = kn->parent->priv; in cgroup_kn_lock_live()
1610 if (!cgroup_tryget(cgrp)) in cgroup_kn_lock_live()
1615 cgroup_lock_and_drain_offline(cgrp); in cgroup_kn_lock_live()
1619 if (!cgroup_is_dead(cgrp)) in cgroup_kn_lock_live()
1620 return cgrp; in cgroup_kn_lock_live()
1626 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) in cgroup_rm_file() argument
1633 struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss); in cgroup_rm_file()
1643 kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); in cgroup_rm_file()
1652 struct cgroup *cgrp = css->cgroup; in css_clear_dir() local
1661 if (cgroup_on_dfl(cgrp)) in css_clear_dir()
1666 cgroup_addrm_files(css, cgrp, cfts, false); in css_clear_dir()
1669 cgroup_addrm_files(css, cgrp, cfts, false); in css_clear_dir()
1681 struct cgroup *cgrp = css->cgroup; in css_populate_dir() local
1685 if ((css->flags & CSS_VISIBLE) || !cgrp->kn) in css_populate_dir()
1689 if (cgroup_on_dfl(cgrp)) in css_populate_dir()
1694 ret = cgroup_addrm_files(&cgrp->self, cgrp, cfts, true); in css_populate_dir()
1699 ret = cgroup_addrm_files(css, cgrp, cfts, true); in css_populate_dir()
1714 cgroup_addrm_files(css, cgrp, cfts, false); in css_populate_dir()
1721 struct cgroup *dcgrp = &dst_root->cgrp; in rebind_subsystems()
1733 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) && in rebind_subsystems()
1744 struct cgroup *scgrp = &src_root->cgrp; in rebind_subsystems()
1945 static void init_cgroup_housekeeping(struct cgroup *cgrp) in init_cgroup_housekeeping() argument
1950 INIT_LIST_HEAD(&cgrp->self.sibling); in init_cgroup_housekeeping()
1951 INIT_LIST_HEAD(&cgrp->self.children); in init_cgroup_housekeeping()
1952 INIT_LIST_HEAD(&cgrp->cset_links); in init_cgroup_housekeeping()
1953 INIT_LIST_HEAD(&cgrp->pidlists); in init_cgroup_housekeeping()
1954 mutex_init(&cgrp->pidlist_mutex); in init_cgroup_housekeeping()
1955 cgrp->self.cgroup = cgrp; in init_cgroup_housekeeping()
1956 cgrp->self.flags |= CSS_ONLINE; in init_cgroup_housekeeping()
1957 cgrp->dom_cgrp = cgrp; in init_cgroup_housekeeping()
1958 cgrp->max_descendants = INT_MAX; in init_cgroup_housekeeping()
1959 cgrp->max_depth = INT_MAX; in init_cgroup_housekeeping()
1960 INIT_LIST_HEAD(&cgrp->rstat_css_list); in init_cgroup_housekeeping()
1961 prev_cputime_init(&cgrp->prev_cputime); in init_cgroup_housekeeping()
1964 INIT_LIST_HEAD(&cgrp->e_csets[ssid]); in init_cgroup_housekeeping()
1966 init_waitqueue_head(&cgrp->offline_waitq); in init_cgroup_housekeeping()
1967 INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); in init_cgroup_housekeeping()
1973 struct cgroup *cgrp = &root->cgrp; in init_cgroup_root() local
1977 cgrp->root = root; in init_cgroup_root()
1978 init_cgroup_housekeeping(cgrp); in init_cgroup_root()
1987 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); in init_cgroup_root()
1993 struct cgroup *root_cgrp = &root->cgrp; in cgroup_setup_root()
2110 struct cgroup *cgrp; in cgroup_do_get_tree() local
2115 cgrp = cset_cgroup_from_root(ctx->ns->root_cset, ctx->root); in cgroup_do_get_tree()
2120 nsdentry = kernfs_node_dentry(cgrp->kn, sb); in cgroup_do_get_tree()
2131 cgroup_put(&ctx->root->cgrp); in cgroup_do_get_tree()
2156 cgroup_get_live(&cgrp_dfl_root.cgrp); in cgroup_get_tree()
2223 if (list_empty(&root->cgrp.self.children) && root != &cgrp_dfl_root && in cgroup_kill_sb()
2224 !percpu_ref_is_dying(&root->cgrp.self.refcnt)) in cgroup_kill_sb()
2225 percpu_ref_kill(&root->cgrp.self.refcnt); in cgroup_kill_sb()
2226 cgroup_put(&root->cgrp); in cgroup_kill_sb()
2290 int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, in cgroup_path_ns_locked() argument
2293 struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root); in cgroup_path_ns_locked()
2295 return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen); in cgroup_path_ns_locked()
2298 int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen, in cgroup_path_ns() argument
2306 ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns); in cgroup_path_ns()
2331 struct cgroup *cgrp; in task_cgroup_path() local
2341 cgrp = task_cgroup_from_root(task, root); in task_cgroup_path()
2342 ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns); in task_cgroup_path()
2908 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_controllers_show() local
2910 cgroup_print_ss_mask(seq, cgroup_control(cgrp)); in cgroup_controllers_show()
2917 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_subtree_control_show() local
2919 cgroup_print_ss_mask(seq, cgrp->subtree_control); in cgroup_subtree_control_show()
2932 static int cgroup_update_dfl_csses(struct cgroup *cgrp) in cgroup_update_dfl_csses() argument
2946 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_update_dfl_csses()
2984 void cgroup_lock_and_drain_offline(struct cgroup *cgrp) in cgroup_lock_and_drain_offline() argument
2995 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_lock_and_drain_offline()
3025 static void cgroup_save_control(struct cgroup *cgrp) in cgroup_save_control() argument
3030 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_save_control()
3045 static void cgroup_propagate_control(struct cgroup *cgrp) in cgroup_propagate_control() argument
3050 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_propagate_control()
3066 static void cgroup_restore_control(struct cgroup *cgrp) in cgroup_restore_control() argument
3071 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_restore_control()
3081 struct cgroup *cgrp = css->cgroup; in css_visible() local
3083 if (cgroup_control(cgrp) & (1 << ss->id)) in css_visible()
3085 if (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) in css_visible()
3087 return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl; in css_visible()
3103 static int cgroup_apply_control_enable(struct cgroup *cgrp) in cgroup_apply_control_enable() argument
3110 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_apply_control_enable()
3149 static void cgroup_apply_control_disable(struct cgroup *cgrp) in cgroup_apply_control_disable() argument
3156 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_apply_control_disable()
3194 static int cgroup_apply_control(struct cgroup *cgrp) in cgroup_apply_control() argument
3198 cgroup_propagate_control(cgrp); in cgroup_apply_control()
3200 ret = cgroup_apply_control_enable(cgrp); in cgroup_apply_control()
3209 ret = cgroup_update_dfl_csses(cgrp); in cgroup_apply_control()
3223 static void cgroup_finalize_control(struct cgroup *cgrp, int ret) in cgroup_finalize_control() argument
3226 cgroup_restore_control(cgrp); in cgroup_finalize_control()
3227 cgroup_propagate_control(cgrp); in cgroup_finalize_control()
3230 cgroup_apply_control_disable(cgrp); in cgroup_finalize_control()
3233 static int cgroup_vet_subtree_control_enable(struct cgroup *cgrp, u16 enable) in cgroup_vet_subtree_control_enable() argument
3242 if (!cgroup_is_valid_domain(cgrp->dom_cgrp)) in cgroup_vet_subtree_control_enable()
3246 if (cgroup_is_mixable(cgrp)) in cgroup_vet_subtree_control_enable()
3251 if (cgroup_is_thread_root(cgrp) || cgroup_is_threaded(cgrp)) in cgroup_vet_subtree_control_enable()
3259 if (cgroup_can_be_thread_root(cgrp) || cgroup_is_threaded(cgrp)) in cgroup_vet_subtree_control_enable()
3267 if (cgroup_has_tasks(cgrp)) in cgroup_vet_subtree_control_enable()
3279 struct cgroup *cgrp, *child; in cgroup_subtree_control_write() local
3312 cgrp = cgroup_kn_lock_live(of->kn, true); in cgroup_subtree_control_write()
3313 if (!cgrp) in cgroup_subtree_control_write()
3318 if (cgrp->subtree_control & (1 << ssid)) { in cgroup_subtree_control_write()
3323 if (!(cgroup_control(cgrp) & (1 << ssid))) { in cgroup_subtree_control_write()
3328 if (!(cgrp->subtree_control & (1 << ssid))) { in cgroup_subtree_control_write()
3334 cgroup_for_each_live_child(child, cgrp) { in cgroup_subtree_control_write()
3348 ret = cgroup_vet_subtree_control_enable(cgrp, enable); in cgroup_subtree_control_write()
3353 cgroup_save_control(cgrp); in cgroup_subtree_control_write()
3355 cgrp->subtree_control |= enable; in cgroup_subtree_control_write()
3356 cgrp->subtree_control &= ~disable; in cgroup_subtree_control_write()
3358 ret = cgroup_apply_control(cgrp); in cgroup_subtree_control_write()
3359 cgroup_finalize_control(cgrp, ret); in cgroup_subtree_control_write()
3363 kernfs_activate(cgrp->kn); in cgroup_subtree_control_write()
3378 static int cgroup_enable_threaded(struct cgroup *cgrp) in cgroup_enable_threaded() argument
3380 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_enable_threaded()
3389 if (cgroup_is_threaded(cgrp)) in cgroup_enable_threaded()
3398 if (cgroup_is_populated(cgrp) || in cgroup_enable_threaded()
3399 cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_enable_threaded()
3411 cgroup_save_control(cgrp); in cgroup_enable_threaded()
3413 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) in cgroup_enable_threaded()
3414 if (dsct == cgrp || cgroup_is_threaded(dsct)) in cgroup_enable_threaded()
3417 ret = cgroup_apply_control(cgrp); in cgroup_enable_threaded()
3421 cgroup_finalize_control(cgrp, ret); in cgroup_enable_threaded()
3427 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_type_show() local
3429 if (cgroup_is_threaded(cgrp)) in cgroup_type_show()
3431 else if (!cgroup_is_valid_domain(cgrp)) in cgroup_type_show()
3433 else if (cgroup_is_thread_root(cgrp)) in cgroup_type_show()
3444 struct cgroup *cgrp; in cgroup_type_write() local
3451 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_type_write()
3452 if (!cgrp) in cgroup_type_write()
3456 ret = cgroup_enable_threaded(cgrp); in cgroup_type_write()
3464 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_descendants_show() local
3465 int descendants = READ_ONCE(cgrp->max_descendants); in cgroup_max_descendants_show()
3478 struct cgroup *cgrp; in cgroup_max_descendants_write() local
3494 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_descendants_write()
3495 if (!cgrp) in cgroup_max_descendants_write()
3498 cgrp->max_descendants = descendants; in cgroup_max_descendants_write()
3507 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_depth_show() local
3508 int depth = READ_ONCE(cgrp->max_depth); in cgroup_max_depth_show()
3521 struct cgroup *cgrp; in cgroup_max_depth_write() local
3537 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_depth_write()
3538 if (!cgrp) in cgroup_max_depth_write()
3541 cgrp->max_depth = depth; in cgroup_max_depth_write()
3550 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_events_show() local
3552 seq_printf(seq, "populated %d\n", cgroup_is_populated(cgrp)); in cgroup_events_show()
3553 seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags)); in cgroup_events_show()
3571 struct cgroup *cgrp, int ssid) in cgroup_extra_stat_show() argument
3580 css = cgroup_tryget_css(cgrp, ss); in cgroup_extra_stat_show()
3591 struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup; in cpu_stat_show() local
3596 ret = cgroup_extra_stat_show(seq, cgrp, cpu_cgrp_id); in cpu_stat_show()
3628 struct cgroup *cgrp; in cgroup_pressure_write() local
3630 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_pressure_write()
3631 if (!cgrp) in cgroup_pressure_write()
3634 cgroup_get(cgrp); in cgroup_pressure_write()
3637 new = psi_trigger_create(&cgrp->psi, buf, nbytes, res); in cgroup_pressure_write()
3639 cgroup_put(cgrp); in cgroup_pressure_write()
3645 cgroup_put(cgrp); in cgroup_pressure_write()
3685 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_freeze_show() local
3687 seq_printf(seq, "%d\n", cgrp->freezer.freeze); in cgroup_freeze_show()
3695 struct cgroup *cgrp; in cgroup_freeze_write() local
3706 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_freeze_write()
3707 if (!cgrp) in cgroup_freeze_write()
3710 cgroup_freeze(cgrp, freeze); in cgroup_freeze_write()
3738 struct cgroup *cgrp = of->kn->parent->priv; in cgroup_file_write() local
3749 if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) && in cgroup_file_write()
3751 ns != &init_cgroup_ns && ns->root_cset->dfl_cgrp == cgrp) in cgroup_file_write()
3764 css = cgroup_css(cgrp, cft->ss); in cgroup_file_write()
3868 static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp, in cgroup_add_file() argument
3879 kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), in cgroup_add_file()
3917 struct cgroup *cgrp, struct cftype cfts[], in cgroup_addrm_files() argument
3928 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
3930 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
3932 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) in cgroup_addrm_files()
3934 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) in cgroup_addrm_files()
3939 ret = cgroup_add_file(css, cgrp, cft); in cgroup_addrm_files()
3948 cgroup_rm_file(cgrp, cft); in cgroup_addrm_files()
3957 struct cgroup *root = &ss->root->cgrp; in cgroup_apply_cftypes()
3965 struct cgroup *cgrp = css->cgroup; in cgroup_apply_cftypes() local
3970 ret = cgroup_addrm_files(css, cgrp, cfts, is_add); in cgroup_apply_cftypes()
4668 struct cgroup *cgrp = seq_css(s)->cgroup; in __cgroup_procs_start() local
4683 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
4686 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
4694 struct cgroup *cgrp = seq_css(s)->cgroup; in cgroup_procs_start() local
4702 if (cgroup_is_threaded(cgrp)) in cgroup_procs_start()
4958 struct cgroup *cgrp = css->cgroup; in css_free_rwork_fn() local
4969 cgroup_put(cgrp); in css_free_rwork_fn()
4975 atomic_dec(&cgrp->root->nr_cgrps); in css_free_rwork_fn()
4976 cgroup1_pidlist_destroy_all(cgrp); in css_free_rwork_fn()
4977 cancel_work_sync(&cgrp->release_agent_work); in css_free_rwork_fn()
4979 if (cgroup_parent(cgrp)) { in css_free_rwork_fn()
4986 cgroup_put(cgroup_parent(cgrp)); in css_free_rwork_fn()
4987 kernfs_put(cgrp->kn); in css_free_rwork_fn()
4988 psi_cgroup_free(cgrp); in css_free_rwork_fn()
4989 if (cgroup_on_dfl(cgrp)) in css_free_rwork_fn()
4990 cgroup_rstat_exit(cgrp); in css_free_rwork_fn()
4991 kfree(cgrp); in css_free_rwork_fn()
4998 cgroup_destroy_root(cgrp->root); in css_free_rwork_fn()
5008 struct cgroup *cgrp = css->cgroup; in css_release_work_fn() local
5018 cgroup_rstat_flush(cgrp); in css_release_work_fn()
5029 TRACE_CGROUP_PATH(release, cgrp); in css_release_work_fn()
5031 if (cgroup_on_dfl(cgrp)) in css_release_work_fn()
5032 cgroup_rstat_flush(cgrp); in css_release_work_fn()
5035 for (tcgrp = cgroup_parent(cgrp); tcgrp; in css_release_work_fn()
5040 cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id); in css_release_work_fn()
5041 cgrp->id = -1; in css_release_work_fn()
5050 if (cgrp->kn) in css_release_work_fn()
5051 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, in css_release_work_fn()
5071 struct cgroup_subsys *ss, struct cgroup *cgrp) in init_and_link_css() argument
5075 cgroup_get_live(cgrp); in init_and_link_css()
5078 css->cgroup = cgrp; in init_and_link_css()
5087 if (cgroup_parent(cgrp)) { in init_and_link_css()
5088 css->parent = cgroup_css(cgroup_parent(cgrp), ss); in init_and_link_css()
5092 if (cgroup_on_dfl(cgrp) && ss->css_rstat_flush) in init_and_link_css()
5093 list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list); in init_and_link_css()
5095 BUG_ON(cgroup_css(cgrp, ss)); in init_and_link_css()
5147 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, in css_create() argument
5150 struct cgroup *parent = cgroup_parent(cgrp); in css_create()
5163 init_and_link_css(css, ss, cgrp); in css_create()
5210 struct cgroup *cgrp, *tcgrp; in cgroup_create() local
5215 cgrp = kzalloc(struct_size(cgrp, ancestor_ids, (level + 1)), in cgroup_create()
5217 if (!cgrp) in cgroup_create()
5220 ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); in cgroup_create()
5225 ret = cgroup_rstat_init(cgrp); in cgroup_create()
5234 cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL); in cgroup_create()
5235 if (cgrp->id < 0) { in cgroup_create()
5240 init_cgroup_housekeeping(cgrp); in cgroup_create()
5242 cgrp->self.parent = &parent->self; in cgroup_create()
5243 cgrp->root = root; in cgroup_create()
5244 cgrp->level = level; in cgroup_create()
5246 ret = psi_cgroup_alloc(cgrp); in cgroup_create()
5250 ret = cgroup_bpf_inherit(cgrp); in cgroup_create()
5258 cgrp->freezer.e_freeze = parent->freezer.e_freeze; in cgroup_create()
5259 if (cgrp->freezer.e_freeze) { in cgroup_create()
5266 set_bit(CGRP_FREEZE, &cgrp->flags); in cgroup_create()
5267 set_bit(CGRP_FROZEN, &cgrp->flags); in cgroup_create()
5271 for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) { in cgroup_create()
5272 cgrp->ancestor_ids[tcgrp->level] = tcgrp->id; in cgroup_create()
5274 if (tcgrp != cgrp) { in cgroup_create()
5282 if (cgrp->freezer.e_freeze) in cgroup_create()
5289 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); in cgroup_create()
5292 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); in cgroup_create()
5294 cgrp->self.serial_nr = css_serial_nr_next++; in cgroup_create()
5297 list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); in cgroup_create()
5305 cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id); in cgroup_create()
5311 if (!cgroup_on_dfl(cgrp)) in cgroup_create()
5312 cgrp->subtree_control = cgroup_control(cgrp); in cgroup_create()
5314 cgroup_propagate_control(cgrp); in cgroup_create()
5316 return cgrp; in cgroup_create()
5319 psi_cgroup_free(cgrp); in cgroup_create()
5321 cgroup_idr_remove(&root->cgroup_idr, cgrp->id); in cgroup_create()
5324 cgroup_rstat_exit(cgrp); in cgroup_create()
5326 percpu_ref_exit(&cgrp->self.refcnt); in cgroup_create()
5328 kfree(cgrp); in cgroup_create()
5357 struct cgroup *parent, *cgrp; in cgroup_mkdir() local
5374 cgrp = cgroup_create(parent); in cgroup_mkdir()
5375 if (IS_ERR(cgrp)) { in cgroup_mkdir()
5376 ret = PTR_ERR(cgrp); in cgroup_mkdir()
5381 kn = kernfs_create_dir(parent->kn, name, mode, cgrp); in cgroup_mkdir()
5386 cgrp->kn = kn; in cgroup_mkdir()
5398 ret = css_populate_dir(&cgrp->self); in cgroup_mkdir()
5402 ret = cgroup_apply_control_enable(cgrp); in cgroup_mkdir()
5406 TRACE_CGROUP_PATH(mkdir, cgrp); in cgroup_mkdir()
5415 cgroup_destroy_locked(cgrp); in cgroup_mkdir()
5522 static int cgroup_destroy_locked(struct cgroup *cgrp) in cgroup_destroy_locked() argument
5525 struct cgroup *tcgrp, *parent = cgroup_parent(cgrp); in cgroup_destroy_locked()
5536 if (cgroup_is_populated(cgrp)) in cgroup_destroy_locked()
5544 if (css_has_online_children(&cgrp->self)) in cgroup_destroy_locked()
5553 cgrp->self.flags &= ~CSS_ONLINE; in cgroup_destroy_locked()
5556 list_for_each_entry(link, &cgrp->cset_links, cset_link) in cgroup_destroy_locked()
5561 for_each_css(css, ssid, cgrp) in cgroup_destroy_locked()
5565 css_clear_dir(&cgrp->self); in cgroup_destroy_locked()
5566 kernfs_remove(cgrp->kn); in cgroup_destroy_locked()
5568 if (parent && cgroup_is_threaded(cgrp)) in cgroup_destroy_locked()
5572 for (tcgrp = cgroup_parent(cgrp); tcgrp; tcgrp = cgroup_parent(tcgrp)) { in cgroup_destroy_locked()
5579 if (test_bit(CGRP_FROZEN, &cgrp->flags)) in cgroup_destroy_locked()
5586 cgroup_bpf_offline(cgrp); in cgroup_destroy_locked()
5589 percpu_ref_kill(&cgrp->self.refcnt); in cgroup_destroy_locked()
5596 struct cgroup *cgrp; in cgroup_rmdir() local
5599 cgrp = cgroup_kn_lock_live(kn, false); in cgroup_rmdir()
5600 if (!cgrp) in cgroup_rmdir()
5603 ret = cgroup_destroy_locked(cgrp); in cgroup_rmdir()
5605 TRACE_CGROUP_PATH(rmdir, cgrp); in cgroup_rmdir()
5631 css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss)); in cgroup_init_subsys()
5634 init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); in cgroup_init_subsys()
5685 cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; in cgroup_init_early()
5761 &cgrp_dfl_root.cgrp.e_csets[ssid]); in cgroup_init()
5873 struct cgroup *cgrp; in proc_cgroup_show() local
5890 cgrp = task_cgroup_from_root(tsk, root); in proc_cgroup_show()
5901 if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) { in proc_cgroup_show()
5902 retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX, in proc_cgroup_show()
5914 if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp)) in proc_cgroup_show()
6192 struct cgroup *cgrp; in css_tryget_online_from_dir() local
6206 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in css_tryget_online_from_dir()
6207 if (cgrp) in css_tryget_online_from_dir()
6208 css = cgroup_css(cgrp, ss); in css_tryget_online_from_dir()
6243 struct cgroup *cgrp; in cgroup_get_from_path() local
6247 kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path); in cgroup_get_from_path()
6250 cgrp = kn->priv; in cgroup_get_from_path()
6251 cgroup_get_live(cgrp); in cgroup_get_from_path()
6253 cgrp = ERR_PTR(-ENOTDIR); in cgroup_get_from_path()
6257 cgrp = ERR_PTR(-ENOENT); in cgroup_get_from_path()
6261 return cgrp; in cgroup_get_from_path()
6277 struct cgroup *cgrp; in cgroup_get_from_fd() local
6289 cgrp = css->cgroup; in cgroup_get_from_fd()
6290 if (!cgroup_on_dfl(cgrp)) { in cgroup_get_from_fd()
6291 cgroup_put(cgrp); in cgroup_get_from_fd()
6295 return cgrp; in cgroup_get_from_fd()
6402 struct cgroup *cgrp = sock_cgroup_ptr(skcd); in cgroup_sk_free() local
6404 cgroup_bpf_put(cgrp); in cgroup_sk_free()
6405 cgroup_put(cgrp); in cgroup_sk_free()
6411 int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, in cgroup_bpf_attach() argument
6417 ret = __cgroup_bpf_attach(cgrp, prog, type, flags); in cgroup_bpf_attach()
6421 int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, in cgroup_bpf_detach() argument
6427 ret = __cgroup_bpf_detach(cgrp, prog, type); in cgroup_bpf_detach()
6431 int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, in cgroup_bpf_query() argument
6437 ret = __cgroup_bpf_query(cgrp, attr, uattr); in cgroup_bpf_query()