2  *  Generic process-grouping system.
12  *  --------------------------------------------------
14  *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
17  *  sysfs is Copyright (c) 2001-3 Patrick Mochel
19  *  2003-10-10 Written by Simon Derr.
20  *  2003-10-22 Updates by Stephen Hemminger.
21  *  2004 May-July Rework by Paul Jackson.
22  *  ---------------------------------------------------
31 #include "cgroup-internal.h"
33 #include <linux/bpf-cgroup.h>
48 #include <linux/percpu-rwsem.h>
60 #include <linux/psi.h>
73  * that attempts to access what would be a 0-element array (i.e. sized
83  * css_set_lock protects task->cgroups pointer, the list of css_set
108  * Protects cgroup_file->kn for !self csses.  It synchronizes notifications
109  * against file removal/re-creation across css hiding.
194  * Also, as csses are always appended to the parent's ->children list, it
252  * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
268  * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
278  * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
281  * - When mounting an existing superblock, mount options should match.
283  * - rename(2) is disallowed.
285  * - "tasks" is removed.  Everything should be at process granularity.  Use
288  * - "cgroup.procs" is not sorted.  pids will be unique unless they got
289  *   recycled in-between reads.
291  * - "release_agent" and "notify_on_release" are removed.  Replacement
294  * - "cgroup.clone_children" is removed.
296  * - "cgroup.subtree_populated" is available.  Its value is 0 if the cgroup
301  * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
302  *   take masks of ancestors with non-empty cpus/mems, instead of being
305  * - cpuset: a task can be moved into an empty cpuset, and again it takes
308  * - blkcg: blk-throttle becomes properly hierarchical.
310  * - debug: disallowed on the default hierarchy.
314 	return cgrp->root == &cgrp_dfl_root;  in cgroup_on_dfl()
350 	return cgrp->nr_populated_csets;  in cgroup_has_tasks()
355 	return cgrp->dom_cgrp != cgrp;  in cgroup_is_threaded()
363 	 * the no-internal-process constraint, so it can serve as a thread  in cgroup_is_mixable()
381 	if (cgrp->nr_populated_domain_children)  in cgroup_can_be_thread_root()
385 	if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask)  in cgroup_can_be_thread_root()
399 	if (cgrp->nr_threaded_children)  in cgroup_is_thread_root()
407 	    (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask))  in cgroup_is_thread_root()
435 	u16 root_ss_mask = cgrp->root->subsys_mask;  in cgroup_control()
438 		u16 ss_mask = parent->subtree_control;  in cgroup_control()
458 		u16 ss_mask = parent->subtree_ss_mask;  in cgroup_ss_mask()
466 	return cgrp->root->subsys_mask;  in cgroup_ss_mask()
470  * cgroup_css - obtain a cgroup's css for the specified subsystem
472  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
484 		return rcu_dereference_check(cgrp->subsys[ss->id],  in cgroup_css()
487 		return &cgrp->self;  in cgroup_css()
491  * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem
513  * cgroup_e_css_by_mask - obtain a cgroup's effective css for the specified ss
515  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
520  * function is guaranteed to return non-NULL css.
528 		return &cgrp->self;  in cgroup_e_css_by_mask()
534 	while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) {  in cgroup_e_css_by_mask()
544  * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
572 	return init_css_set.subsys[ss->id];  in cgroup_e_css()
576  * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
604 	css = init_css_set.subsys[ss->id];  in cgroup_get_e_css()
615 	css_get(&cgrp->self);  in cgroup_get_live()
619  * __cgroup_task_count - count the number of tasks in a cgroup. The caller
630 	list_for_each_entry(link, &cgrp->cset_links, cset_link)  in __cgroup_task_count()
631 		count += link->cset->nr_tasks;  in __cgroup_task_count()
637  * cgroup_task_count - count the number of tasks in a cgroup.
653 	struct cgroup *cgrp = of->kn->parent->priv;  in of_css()
664 	if (CGROUP_HAS_SUBSYS_CONFIG && cft->ss)  in of_css()
665 		return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);  in of_css()
667 		return &cgrp->self;  in of_css()
672  * for_each_css - iterate all css's of a cgroup
682 				(cgrp)->subsys[(ssid)],			\
687  * for_each_e_css - iterate all effective css's of a cgroup
702  * do_each_subsys_mask - filter for_each_subsys with a bitmask
707  * The block will only run for cases where the ssid-th bit (1 << ssid) of
727 	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
737 		       (dsct) = (d_css)->cgroup;			\
746 		       (dsct) = (d_css)->cgroup;			\
752  * The default css_set - used by init and its children prior to any
755  * reference-counted, to improve performance when child cgroups
772 	 * The following field is re-initialized when this cset gets linked
784 	return cset->dom_cset != cset;  in css_set_threaded()
788  * css_set_populated - does a css_set contain any tasks?
791  * css_set_populated() should be the same as !!cset->nr_tasks at steady
794  * properly updated. Hence, we can't just look at ->nr_tasks here.
800 	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);  in css_set_populated()
804  * cgroup_update_populated - update the populated count of a cgroup
809  * task or losing the last.  Update @cgrp->nr_populated_* accordingly.  The
815  * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and
823 	int adj = populated ? 1 : -1;  in cgroup_update_populated()
831 			cgrp->nr_populated_csets += adj;  in cgroup_update_populated()
834 				cgrp->nr_populated_threaded_children += adj;  in cgroup_update_populated()
836 				cgrp->nr_populated_domain_children += adj;  in cgroup_update_populated()
845 		cgroup_file_notify(&cgrp->events_file);  in cgroup_update_populated()
853  * css_set_update_populated - update populated state of a css_set
866 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)  in css_set_update_populated()
867 		cgroup_update_populated(link->cgrp, populated);  in css_set_update_populated()
881 	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)  in css_set_skip_task_iters()
886  * css_set_move_task - move a task from one css_set to another
890  * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
910 		WARN_ON_ONCE(list_empty(&task->cg_list));  in css_set_move_task()
913 		list_del_init(&task->cg_list);  in css_set_move_task()
917 		WARN_ON_ONCE(!list_empty(&task->cg_list));  in css_set_move_task()
926 		WARN_ON_ONCE(task->flags & PF_EXITING);  in css_set_move_task()
929 		list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks :  in css_set_move_task()
930 							     &to_cset->tasks);  in css_set_move_task()
963 	if (!refcount_dec_and_test(&cset->refcount))  in put_css_set_locked()
966 	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));  in put_css_set_locked()
970 		list_del(&cset->e_cset_node[ssid]);  in put_css_set_locked()
971 		css_put(cset->subsys[ssid]);  in put_css_set_locked()
973 	hash_del(&cset->hlist);  in put_css_set_locked()
974 	css_set_count--;  in put_css_set_locked()
976 	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {  in put_css_set_locked()
977 		list_del(&link->cset_link);  in put_css_set_locked()
978 		list_del(&link->cgrp_link);  in put_css_set_locked()
979 		if (cgroup_parent(link->cgrp))  in put_css_set_locked()
980 			cgroup_put(link->cgrp);  in put_css_set_locked()
985 		list_del(&cset->threaded_csets_node);  in put_css_set_locked()
986 		put_css_set_locked(cset->dom_cset);  in put_css_set_locked()
993  * compare_css_sets - helper function for find_existing_css_set().
997  * @template: desired set of css pointers in css_set (pre-calculated)
1015 	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))  in compare_css_sets()
1023 		new_dfl_cgrp = old_cset->dfl_cgrp;  in compare_css_sets()
1025 	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)  in compare_css_sets()
1034 	l1 = &cset->cgrp_links;  in compare_css_sets()
1035 	l2 = &old_cset->cgrp_links;  in compare_css_sets()
1040 		l1 = l1->next;  in compare_css_sets()
1041 		l2 = l2->next;  in compare_css_sets()
1042 		/* See if we reached the end - both lists are equal length. */  in compare_css_sets()
1043 		if (l1 == &cset->cgrp_links) {  in compare_css_sets()
1044 			BUG_ON(l2 != &old_cset->cgrp_links);  in compare_css_sets()
1047 			BUG_ON(l2 == &old_cset->cgrp_links);  in compare_css_sets()
1052 		cgrp1 = link1->cgrp;  in compare_css_sets()
1053 		cgrp2 = link2->cgrp;  in compare_css_sets()
1055 		BUG_ON(cgrp1->root != cgrp2->root);  in compare_css_sets()
1064 		if (cgrp1->root == new_cgrp->root) {  in compare_css_sets()
1076  * find_existing_css_set - init css array and find the matching css_set
1085 	struct cgroup_root *root = cgrp->root;  in find_existing_css_set()
1097 		if (root->subsys_mask & (1UL << i)) {  in find_existing_css_set()
1108 			template[i] = old_cset->subsys[i];  in find_existing_css_set()
1130 		list_del(&link->cset_link);  in free_cgrp_cset_links()
1136  * allocate_cgrp_cset_links - allocate cgrp_cset_links
1141  * through ->cset_link.  Returns 0 on success or -errno.
1154 			return -ENOMEM;  in allocate_cgrp_cset_links()
1156 		list_add(&link->cset_link, tmp_links);  in allocate_cgrp_cset_links()
1162  * link_css_set - a helper function to link a css_set to a cgroup
1175 		cset->dfl_cgrp = cgrp;  in link_css_set()
1178 	link->cset = cset;  in link_css_set()
1179 	link->cgrp = cgrp;  in link_css_set()
1185 	list_move_tail(&link->cset_link, &cgrp->cset_links);  in link_css_set()
1186 	list_add_tail(&link->cgrp_link, &cset->cgrp_links);  in link_css_set()
1193  * find_css_set - return a new css_set with one cgroup updated
1234 	refcount_set(&cset->refcount, 1);  in find_css_set()
1235 	cset->dom_cset = cset;  in find_css_set()
1236 	INIT_LIST_HEAD(&cset->tasks);  in find_css_set()
1237 	INIT_LIST_HEAD(&cset->mg_tasks);  in find_css_set()
1238 	INIT_LIST_HEAD(&cset->dying_tasks);  in find_css_set()
1239 	INIT_LIST_HEAD(&cset->task_iters);  in find_css_set()
1240 	INIT_LIST_HEAD(&cset->threaded_csets);  in find_css_set()
1241 	INIT_HLIST_NODE(&cset->hlist);  in find_css_set()
1242 	INIT_LIST_HEAD(&cset->cgrp_links);  in find_css_set()
1243 	INIT_LIST_HEAD(&cset->mg_src_preload_node);  in find_css_set()
1244 	INIT_LIST_HEAD(&cset->mg_dst_preload_node);  in find_css_set()
1245 	INIT_LIST_HEAD(&cset->mg_node);  in find_css_set()
1249 	memcpy(cset->subsys, template, sizeof(cset->subsys));  in find_css_set()
1253 	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {  in find_css_set()
1254 		struct cgroup *c = link->cgrp;  in find_css_set()
1256 		if (c->root == cgrp->root)  in find_css_set()
1266 	key = css_set_hash(cset->subsys);  in find_css_set()
1267 	hash_add(css_set_table, &cset->hlist, key);  in find_css_set()
1270 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()
1272 		list_add_tail(&cset->e_cset_node[ssid],  in find_css_set()
1273 			      &css->cgroup->e_csets[ssid]);  in find_css_set()
1285 	if (cgroup_is_threaded(cset->dfl_cgrp)) {  in find_css_set()
1288 		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);  in find_css_set()
1295 		cset->dom_cset = dcset;  in find_css_set()
1296 		list_add_tail(&cset->threaded_csets_node,  in find_css_set()
1297 			      &dcset->threaded_csets);  in find_css_set()
1306 	struct cgroup *root_cgrp = kernfs_root_to_node(kf_root)->priv;  in cgroup_root_from_kf()
1308 	return root_cgrp->root;  in cgroup_root_from_kf()
1313 	bool favoring = root->flags & CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_favor_dynmods()
1318 		root->flags |= CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_favor_dynmods()
1321 		root->flags &= ~CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_favor_dynmods()
1335 	root->hierarchy_id = id;  in cgroup_init_root_id()
1343 	idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);  in cgroup_exit_root_id()
1353 	struct cgroup *cgrp = &root->cgrp;  in cgroup_destroy_root()
1360 	BUG_ON(atomic_read(&root->nr_cgrps));  in cgroup_destroy_root()
1361 	BUG_ON(!list_empty(&cgrp->self.children));  in cgroup_destroy_root()
1364 	WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask));  in cgroup_destroy_root()
1372 	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {  in cgroup_destroy_root()
1373 		list_del(&link->cset_link);  in cgroup_destroy_root()
1374 		list_del(&link->cgrp_link);  in cgroup_destroy_root()
1380 	if (!list_empty(&root->root_list)) {  in cgroup_destroy_root()
1381 		list_del(&root->root_list);  in cgroup_destroy_root()
1382 		cgroup_root_count--;  in cgroup_destroy_root()
1391 	kernfs_destroy_root(root->kf_root);  in cgroup_destroy_root()
1404 		res_cgroup = &root->cgrp;  in __cset_cgroup_from_root()
1406 		res_cgroup = cset->dfl_cgrp;  in __cset_cgroup_from_root()
1411 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in __cset_cgroup_from_root()
1412 			struct cgroup *c = link->cgrp;  in __cset_cgroup_from_root()
1414 			if (c->root == root) {  in __cset_cgroup_from_root()
1439 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_from_root()
1452  * - Internal rcu_read_lock is unnecessary because we don't dereference any rcu
1454  * - css_set_lock is not needed because we just read cset->dfl_cgrp.
1455  * - As a bonus returned cgrp is pinned with the current because it cannot
1462 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_dfl()
1484 	 * No need to lock the task - since we hold css_set_lock the  in task_cgroup_from_root()
1521 	struct cgroup_subsys *ss = cft->ss;  in cgroup_file_name()
1523 	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&  in cgroup_file_name()
1524 	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {  in cgroup_file_name()
1525 		const char *dbg = (cft->flags & CFTYPE_DEBUG) ? ".__DEBUG__." : "";  in cgroup_file_name()
1528 			 dbg, cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name,  in cgroup_file_name()
1529 			 cft->name);  in cgroup_file_name()
1531 		strscpy(buf, cft->name, CGROUP_FILE_NAME_MAX);  in cgroup_file_name()
1537  * cgroup_file_mode - deduce file mode of a control file
1546 	if (cft->read_u64 || cft->read_s64 || cft->seq_show)  in cgroup_file_mode()
1549 	if (cft->write_u64 || cft->write_s64 || cft->write) {  in cgroup_file_mode()
1550 		if (cft->flags & CFTYPE_WORLD_WRITABLE)  in cgroup_file_mode()
1560  * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
1565  * enabled together through its ->depends_on mask.  In such cases, more
1585 			new_ss_mask |= ss->depends_on;  in cgroup_calc_subtree_ss_mask()
1590 		 * happen only if some depended-upon subsystems were bound  in cgroup_calc_subtree_ss_mask()
1591 		 * to non-default hierarchies.  in cgroup_calc_subtree_ss_mask()
1604  * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1618 		cgrp = kn->priv;  in cgroup_kn_unlock()
1620 		cgrp = kn->parent->priv;  in cgroup_kn_unlock()
1629  * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1643  * including self-removal.
1650 		cgrp = kn->priv;  in cgroup_kn_lock_live()
1652 		cgrp = kn->parent->priv;  in cgroup_kn_lock_live()
1682 	if (cft->file_offset) {  in cgroup_rm_file()
1683 		struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);  in cgroup_rm_file()
1684 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_rm_file()
1687 		cfile->kn = NULL;  in cgroup_rm_file()
1690 		del_timer_sync(&cfile->notify_timer);  in cgroup_rm_file()
1693 	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));  in cgroup_rm_file()
1697  * css_clear_dir - remove subsys files in a cgroup directory
1702 	struct cgroup *cgrp = css->cgroup;  in css_clear_dir()
1705 	if (!(css->flags & CSS_VISIBLE))  in css_clear_dir()
1708 	css->flags &= ~CSS_VISIBLE;  in css_clear_dir()
1710 	if (!css->ss) {  in css_clear_dir()
1722 		list_for_each_entry(cfts, &css->ss->cfts, node)  in css_clear_dir()
1728  * css_populate_dir - create subsys files in a cgroup directory
1735 	struct cgroup *cgrp = css->cgroup;  in css_populate_dir()
1739 	if ((css->flags & CSS_VISIBLE) || !cgrp->kn)  in css_populate_dir()
1742 	if (!css->ss) {  in css_populate_dir()
1744 			ret = cgroup_addrm_files(&cgrp->self, cgrp,  in css_populate_dir()
1750 				ret = cgroup_addrm_files(&cgrp->self, cgrp,  in css_populate_dir()
1760 		list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1769 	css->flags |= CSS_VISIBLE;  in css_populate_dir()
1773 	list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1783 	struct cgroup *dcgrp = &dst_root->cgrp;  in rebind_subsystems()
1792 		 * If @ss has non-root csses attached to it, can't move.  in rebind_subsystems()
1796 		if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) &&  in rebind_subsystems()
1797 		    !ss->implicit_on_dfl)  in rebind_subsystems()
1798 			return -EBUSY;  in rebind_subsystems()
1800 		/* can't move between two non-dummy roots either */  in rebind_subsystems()
1801 		if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)  in rebind_subsystems()
1802 			return -EBUSY;  in rebind_subsystems()
1808 		if (ss->root == &cgrp_dfl_root)  in rebind_subsystems()
1826 		struct cgroup_root *src_root = ss->root;  in rebind_subsystems()
1827 		struct cgroup *scgrp = &src_root->cgrp;  in rebind_subsystems()
1835 			src_root->subsys_mask &= ~(1 << ssid);  in rebind_subsystems()
1841 		RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);  in rebind_subsystems()
1842 		rcu_assign_pointer(dcgrp->subsys[ssid], css);  in rebind_subsystems()
1843 		ss->root = dst_root;  in rebind_subsystems()
1844 		css->cgroup = dcgrp;  in rebind_subsystems()
1848 			list_move_tail(&cset->e_cset_node[ss->id],  in rebind_subsystems()
1849 				       &dcgrp->e_csets[ss->id]);  in rebind_subsystems()
1852 		if (ss->css_rstat_flush) {  in rebind_subsystems()
1853 			list_del_rcu(&css->rstat_css_node);  in rebind_subsystems()
1855 			list_add_rcu(&css->rstat_css_node,  in rebind_subsystems()
1856 				     &dcgrp->rstat_css_list);  in rebind_subsystems()
1860 		dst_root->subsys_mask |= 1 << ssid;  in rebind_subsystems()
1864 			dcgrp->subtree_control |= 1 << ssid;  in rebind_subsystems()
1871 				ss->name, ret);  in rebind_subsystems()
1873 		if (ss->bind)  in rebind_subsystems()
1874 			ss->bind(css);  in rebind_subsystems()
1877 	kernfs_activate(dcgrp->kn);  in rebind_subsystems()
1891 		return -ENOMEM;  in cgroup_show_path()
1895 	len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX);  in cgroup_show_path()
1899 		len = -ERANGE;  in cgroup_show_path()
1936 		ctx->flags |= CGRP_ROOT_NS_DELEGATE;  in cgroup2_parse_param()
1939 		ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS;  in cgroup2_parse_param()
1942 		ctx->flags |= CGRP_ROOT_MEMORY_LOCAL_EVENTS;  in cgroup2_parse_param()
1945 		ctx->flags |= CGRP_ROOT_MEMORY_RECURSIVE_PROT;  in cgroup2_parse_param()
1948 	return -EINVAL;  in cgroup2_parse_param()
1953 	if (current->nsproxy->cgroup_ns == &init_cgroup_ns) {  in apply_cgroup_root_flags()
1991 	apply_cgroup_root_flags(ctx->flags);  in cgroup_reconfigure()
2000 	INIT_LIST_HEAD(&cgrp->self.sibling);  in init_cgroup_housekeeping()
2001 	INIT_LIST_HEAD(&cgrp->self.children);  in init_cgroup_housekeeping()
2002 	INIT_LIST_HEAD(&cgrp->cset_links);  in init_cgroup_housekeeping()
2003 	INIT_LIST_HEAD(&cgrp->pidlists);  in init_cgroup_housekeeping()
2004 	mutex_init(&cgrp->pidlist_mutex);  in init_cgroup_housekeeping()
2005 	cgrp->self.cgroup = cgrp;  in init_cgroup_housekeeping()
2006 	cgrp->self.flags |= CSS_ONLINE;  in init_cgroup_housekeeping()
2007 	cgrp->dom_cgrp = cgrp;  in init_cgroup_housekeeping()
2008 	cgrp->max_descendants = INT_MAX;  in init_cgroup_housekeeping()
2009 	cgrp->max_depth = INT_MAX;  in init_cgroup_housekeeping()
2010 	INIT_LIST_HEAD(&cgrp->rstat_css_list);  in init_cgroup_housekeeping()
2011 	prev_cputime_init(&cgrp->prev_cputime);  in init_cgroup_housekeeping()
2014 		INIT_LIST_HEAD(&cgrp->e_csets[ssid]);  in init_cgroup_housekeeping()
2016 	init_waitqueue_head(&cgrp->offline_waitq);  in init_cgroup_housekeeping()
2017 	INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent);  in init_cgroup_housekeeping()
2022 	struct cgroup_root *root = ctx->root;  in init_cgroup_root()
2023 	struct cgroup *cgrp = &root->cgrp;  in init_cgroup_root()
2025 	INIT_LIST_HEAD(&root->root_list);  in init_cgroup_root()
2026 	atomic_set(&root->nr_cgrps, 1);  in init_cgroup_root()
2027 	cgrp->root = root;  in init_cgroup_root()
2031 	root->flags = ctx->flags & ~CGRP_ROOT_FAVOR_DYNMODS;  in init_cgroup_root()
2032 	if (ctx->release_agent)  in init_cgroup_root()
2033 		strscpy(root->release_agent_path, ctx->release_agent, PATH_MAX);  in init_cgroup_root()
2034 	if (ctx->name)  in init_cgroup_root()
2035 		strscpy(root->name, ctx->name, MAX_CGROUP_ROOT_NAMELEN);  in init_cgroup_root()
2036 	if (ctx->cpuset_clone_children)  in init_cgroup_root()
2037 		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);  in init_cgroup_root()
2043 	struct cgroup *root_cgrp = &root->cgrp;  in cgroup_setup_root()
2050 	ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release,  in cgroup_setup_root()
2057 	 * but that's OK - it can only be increased by someone holding  in cgroup_setup_root()
2073 	root->kf_root = kernfs_create_root(kf_sops,  in cgroup_setup_root()
2078 	if (IS_ERR(root->kf_root)) {  in cgroup_setup_root()
2079 		ret = PTR_ERR(root->kf_root);  in cgroup_setup_root()
2082 	root_cgrp->kn = kernfs_root_to_node(root->kf_root);  in cgroup_setup_root()
2084 	root_cgrp->ancestors[0] = root_cgrp;  in cgroup_setup_root()
2086 	ret = css_populate_dir(&root_cgrp->self);  in cgroup_setup_root()
2108 	list_add(&root->root_list, &cgroup_roots);  in cgroup_setup_root()
2123 	BUG_ON(!list_empty(&root_cgrp->self.children));  in cgroup_setup_root()
2124 	BUG_ON(atomic_read(&root->nr_cgrps) != 1);  in cgroup_setup_root()
2132 	kernfs_destroy_root(root->kf_root);  in cgroup_setup_root()
2133 	root->kf_root = NULL;  in cgroup_setup_root()
2137 	percpu_ref_exit(&root_cgrp->self.refcnt);  in cgroup_setup_root()
2148 	ctx->kfc.root = ctx->root->kf_root;  in cgroup_do_get_tree()
2149 	if (fc->fs_type == &cgroup2_fs_type)  in cgroup_do_get_tree()
2150 		ctx->kfc.magic = CGROUP2_SUPER_MAGIC;  in cgroup_do_get_tree()
2152 		ctx->kfc.magic = CGROUP_SUPER_MAGIC;  in cgroup_do_get_tree()
2156 	 * In non-init cgroup namespace, instead of root cgroup's dentry,  in cgroup_do_get_tree()
2157 	 * we return the dentry corresponding to the cgroupns->root_cgrp.  in cgroup_do_get_tree()
2159 	if (!ret && ctx->ns != &init_cgroup_ns) {  in cgroup_do_get_tree()
2161 		struct super_block *sb = fc->root->d_sb;  in cgroup_do_get_tree()
2167 		cgrp = cset_cgroup_from_root(ctx->ns->root_cset, ctx->root);  in cgroup_do_get_tree()
2172 		nsdentry = kernfs_node_dentry(cgrp->kn, sb);  in cgroup_do_get_tree()
2173 		dput(fc->root);  in cgroup_do_get_tree()
2179 		fc->root = nsdentry;  in cgroup_do_get_tree()
2182 	if (!ctx->kfc.new_sb_created)  in cgroup_do_get_tree()
2183 		cgroup_put(&ctx->root->cgrp);  in cgroup_do_get_tree()
2195 	kfree(ctx->name);  in cgroup_fs_context_free()
2196 	kfree(ctx->release_agent);  in cgroup_fs_context_free()
2197 	put_cgroup_ns(ctx->ns);  in cgroup_fs_context_free()
2209 	ctx->root = &cgrp_dfl_root;  in cgroup_get_tree()
2213 		apply_cgroup_root_flags(ctx->flags);  in cgroup_get_tree()
2241 		return -ENOMEM;  in cgroup_init_fs_context()
2243 	ctx->ns = current->nsproxy->cgroup_ns;  in cgroup_init_fs_context()
2244 	get_cgroup_ns(ctx->ns);  in cgroup_init_fs_context()
2245 	fc->fs_private = &ctx->kfc;  in cgroup_init_fs_context()
2246 	if (fc->fs_type == &cgroup2_fs_type)  in cgroup_init_fs_context()
2247 		fc->ops = &cgroup_fs_context_ops;  in cgroup_init_fs_context()
2249 		fc->ops = &cgroup1_fs_context_ops;  in cgroup_init_fs_context()
2250 	put_user_ns(fc->user_ns);  in cgroup_init_fs_context()
2251 	fc->user_ns = get_user_ns(ctx->ns->user_ns);  in cgroup_init_fs_context()
2252 	fc->global = true;  in cgroup_init_fs_context()
2255 	ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_init_fs_context()
2271 	if (list_empty(&root->cgrp.self.children) && root != &cgrp_dfl_root &&  in cgroup_kill_sb()
2272 	    !percpu_ref_is_dying(&root->cgrp.self.refcnt)) {  in cgroup_kill_sb()
2273 		cgroup_bpf_offline(&root->cgrp);  in cgroup_kill_sb()
2274 		percpu_ref_kill(&root->cgrp.self.refcnt);  in cgroup_kill_sb()
2276 	cgroup_put(&root->cgrp);  in cgroup_kill_sb()
2319 	fc->ops = &cpuset_fs_context_ops;  in cpuset_init_fs_context()
2322 	ctx->subsys_mask = 1 << cpuset_cgrp_id;  in cpuset_init_fs_context()
2323 	ctx->flags |= CGRP_ROOT_NOPREFIX;  in cpuset_init_fs_context()
2324 	ctx->release_agent = agent;  in cpuset_init_fs_context()
2327 	put_filesystem(fc->fs_type);  in cpuset_init_fs_context()
2328 	fc->fs_type = &cgroup_fs_type;  in cpuset_init_fs_context()
2343 	struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root);  in cgroup_path_ns_locked()
2345 	return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen);  in cgroup_path_ns_locked()
2366  * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
2371  * Determine @task's cgroup on the first (the one with the lowest non-zero
2405  * cgroup_attach_lock - Lock for ->attach()
2409  * exits by write-locking cgroup_threadgroup_rwsem. However, some ->attach()
2411  * Unfortunately, letting ->attach() operations acquire cpus_read_lock() can
2415  * read-locking threadgroup_rwsem, so threadgroup_rwsem nests inside
2416  * cpus_read_lock(). If we call an ->attach() which acquires the cpus lock while
2417  * write-locking threadgroup_rwsem, the locking order is reversed and we end up
2418  * waiting for an on-going CPU hotplug operation which in turn is waiting for
2424  * write-locking cgroup_threadgroup_rwsem. This allows ->attach() to assume that
2435  * cgroup_attach_unlock - Undo cgroup_attach_lock()
2446  * cgroup_migrate_add_task - add a migration target task to a migration context
2450  * Add @task, which is a migration target, to @mgctx->tset.  This function
2452  * should have been added as a migration source and @task->cg_list will be
2463 	if (task->flags & PF_EXITING)  in cgroup_migrate_add_task()
2467 	WARN_ON_ONCE(list_empty(&task->cg_list));  in cgroup_migrate_add_task()
2470 	if (!cset->mg_src_cgrp)  in cgroup_migrate_add_task()
2473 	mgctx->tset.nr_tasks++;  in cgroup_migrate_add_task()
2475 	list_move_tail(&task->cg_list, &cset->mg_tasks);  in cgroup_migrate_add_task()
2476 	if (list_empty(&cset->mg_node))  in cgroup_migrate_add_task()
2477 		list_add_tail(&cset->mg_node,  in cgroup_migrate_add_task()
2478 			      &mgctx->tset.src_csets);  in cgroup_migrate_add_task()
2479 	if (list_empty(&cset->mg_dst_cset->mg_node))  in cgroup_migrate_add_task()
2480 		list_add_tail(&cset->mg_dst_cset->mg_node,  in cgroup_migrate_add_task()
2481 			      &mgctx->tset.dst_csets);  in cgroup_migrate_add_task()
2485  * cgroup_taskset_first - reset taskset and return the first task
2494 	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);  in cgroup_taskset_first()
2495 	tset->cur_task = NULL;  in cgroup_taskset_first()
2501  * cgroup_taskset_next - iterate to the next task in taskset
2511 	struct css_set *cset = tset->cur_cset;  in cgroup_taskset_next()
2512 	struct task_struct *task = tset->cur_task;  in cgroup_taskset_next()
2514 	while (CGROUP_HAS_SUBSYS_CONFIG && &cset->mg_node != tset->csets) {  in cgroup_taskset_next()
2516 			task = list_first_entry(&cset->mg_tasks,  in cgroup_taskset_next()
2521 		if (&task->cg_list != &cset->mg_tasks) {  in cgroup_taskset_next()
2522 			tset->cur_cset = cset;  in cgroup_taskset_next()
2523 			tset->cur_task = task;  in cgroup_taskset_next()
2529 			 * has its ->mg_dst_cset set.  in cgroup_taskset_next()
2531 			if (cset->mg_dst_cset)  in cgroup_taskset_next()
2532 				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];  in cgroup_taskset_next()
2534 				*dst_cssp = cset->subsys[tset->ssid];  in cgroup_taskset_next()
2547  * cgroup_migrate_execute - migrate a taskset
2551  * This function fails iff one of the ->can_attach callbacks fails and
2557 	struct cgroup_taskset *tset = &mgctx->tset;  in cgroup_migrate_execute()
2564 	if (tset->nr_tasks) {  in cgroup_migrate_execute()
2565 		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {  in cgroup_migrate_execute()
2566 			if (ss->can_attach) {  in cgroup_migrate_execute()
2567 				tset->ssid = ssid;  in cgroup_migrate_execute()
2568 				ret = ss->can_attach(tset);  in cgroup_migrate_execute()
2583 	list_for_each_entry(cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2584 		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {  in cgroup_migrate_execute()
2586 			struct css_set *to_cset = cset->mg_dst_cset;  in cgroup_migrate_execute()
2589 			to_cset->nr_tasks++;  in cgroup_migrate_execute()
2591 			from_cset->nr_tasks--;  in cgroup_migrate_execute()
2596 			cgroup_freezer_migrate_task(task, from_cset->dfl_cgrp,  in cgroup_migrate_execute()
2597 						    to_cset->dfl_cgrp);  in cgroup_migrate_execute()
2609 	tset->csets = &tset->dst_csets;  in cgroup_migrate_execute()
2611 	if (tset->nr_tasks) {  in cgroup_migrate_execute()
2612 		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {  in cgroup_migrate_execute()
2613 			if (ss->attach) {  in cgroup_migrate_execute()
2614 				tset->ssid = ssid;  in cgroup_migrate_execute()
2615 				ss->attach(tset);  in cgroup_migrate_execute()
2624 	if (tset->nr_tasks) {  in cgroup_migrate_execute()
2625 		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {  in cgroup_migrate_execute()
2628 			if (ss->cancel_attach) {  in cgroup_migrate_execute()
2629 				tset->ssid = ssid;  in cgroup_migrate_execute()
2630 				ss->cancel_attach(tset);  in cgroup_migrate_execute()
2636 	list_splice_init(&tset->dst_csets, &tset->src_csets);  in cgroup_migrate_execute()
2637 	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2638 		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);  in cgroup_migrate_execute()
2639 		list_del_init(&cset->mg_node);  in cgroup_migrate_execute()
2644 	 * Re-initialize the cgroup_taskset structure in case it is reused  in cgroup_migrate_execute()
2648 	tset->nr_tasks = 0;  in cgroup_migrate_execute()
2649 	tset->csets    = &tset->src_csets;  in cgroup_migrate_execute()
2654  * cgroup_migrate_vet_dst - verify whether a cgroup can be migration destination
2669 	if (!cgroup_is_valid_domain(dst_cgrp->dom_cgrp))  in cgroup_migrate_vet_dst()
2670 		return -EOPNOTSUPP;  in cgroup_migrate_vet_dst()
2679 	/* apply no-internal-process constraint */  in cgroup_migrate_vet_dst()
2680 	if (dst_cgrp->subtree_control)  in cgroup_migrate_vet_dst()
2681 		return -EBUSY;  in cgroup_migrate_vet_dst()
2687  * cgroup_migrate_finish - cleanup after attach
2701 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_src_csets,  in cgroup_migrate_finish()
2703 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2704 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2705 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2706 		list_del_init(&cset->mg_src_preload_node);  in cgroup_migrate_finish()
2710 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_dst_csets,  in cgroup_migrate_finish()
2712 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2713 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2714 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2715 		list_del_init(&cset->mg_dst_preload_node);  in cgroup_migrate_finish()
2723  * cgroup_migrate_add_src - add a migration source css_set
2729  * @src_cset and add it to @mgctx->src_csets, which should later be cleaned
2748 	 * If ->dead, @src_set is associated with one or more dead cgroups  in cgroup_migrate_add_src()
2752 	if (src_cset->dead)  in cgroup_migrate_add_src()
2755 	if (!list_empty(&src_cset->mg_src_preload_node))  in cgroup_migrate_add_src()
2758 	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);  in cgroup_migrate_add_src()
2760 	WARN_ON(src_cset->mg_src_cgrp);  in cgroup_migrate_add_src()
2761 	WARN_ON(src_cset->mg_dst_cgrp);  in cgroup_migrate_add_src()
2762 	WARN_ON(!list_empty(&src_cset->mg_tasks));  in cgroup_migrate_add_src()
2763 	WARN_ON(!list_empty(&src_cset->mg_node));  in cgroup_migrate_add_src()
2765 	src_cset->mg_src_cgrp = src_cgrp;  in cgroup_migrate_add_src()
2766 	src_cset->mg_dst_cgrp = dst_cgrp;  in cgroup_migrate_add_src()
2768 	list_add_tail(&src_cset->mg_src_preload_node, &mgctx->preloaded_src_csets);  in cgroup_migrate_add_src()
2772  * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2776  * preloaded to @mgctx->preloaded_src_csets.  This function looks up and
2778  * to @mgctx->preloaded_dst_csets.
2792 	list_for_each_entry_safe(src_cset, tmp_cset, &mgctx->preloaded_src_csets,  in cgroup_migrate_prepare_dst()
2798 		dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp);  in cgroup_migrate_prepare_dst()
2800 			return -ENOMEM;  in cgroup_migrate_prepare_dst()
2802 		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);  in cgroup_migrate_prepare_dst()
2810 			src_cset->mg_src_cgrp = NULL;  in cgroup_migrate_prepare_dst()
2811 			src_cset->mg_dst_cgrp = NULL;  in cgroup_migrate_prepare_dst()
2812 			list_del_init(&src_cset->mg_src_preload_node);  in cgroup_migrate_prepare_dst()
2818 		src_cset->mg_dst_cset = dst_cset;  in cgroup_migrate_prepare_dst()
2820 		if (list_empty(&dst_cset->mg_dst_preload_node))  in cgroup_migrate_prepare_dst()
2821 			list_add_tail(&dst_cset->mg_dst_preload_node,  in cgroup_migrate_prepare_dst()
2822 				      &mgctx->preloaded_dst_csets);  in cgroup_migrate_prepare_dst()
2827 			if (src_cset->subsys[ssid] != dst_cset->subsys[ssid])  in cgroup_migrate_prepare_dst()
2828 				mgctx->ss_mask |= 1 << ssid;  in cgroup_migrate_prepare_dst()
2835  * cgroup_migrate - migrate a process or task to a cgroup
2846  * As long as a controller's ->can_attach() doesn't fail, this function is
2847  * guaranteed to succeed.  This means that, excluding ->can_attach()
2877  * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2923 		return ERR_PTR(-EINVAL);  in cgroup_procs_write_start()
2941 			tsk = ERR_PTR(-ESRCH);  in cgroup_procs_write_start()
2949 		tsk = tsk->group_leader;  in cgroup_procs_write_start()
2953 	 * If userland migrates such a kthread to a non-root cgroup, it can  in cgroup_procs_write_start()
2957 	if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) {  in cgroup_procs_write_start()
2958 		tsk = ERR_PTR(-EINVAL);  in cgroup_procs_write_start()
2984 		if (ss->post_attach)  in cgroup_procs_write_finish()
2985 			ss->post_attach();  in cgroup_procs_write_finish()
2997 		seq_puts(seq, ss->name);  in cgroup_print_ss_mask()
3007 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_controllers_show()
3016 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_subtree_control_show()
3018 	cgroup_print_ss_mask(seq, cgrp->subtree_control);  in cgroup_subtree_control_show()
3023  * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
3056 		list_for_each_entry(link, &dsct->cset_links, cset_link)  in cgroup_update_dfl_csses()
3057 			cgroup_migrate_add_src(link->cset, dsct, &mgctx);  in cgroup_update_dfl_csses()
3062 	 * We need to write-lock threadgroup_rwsem while migrating tasks.  in cgroup_update_dfl_csses()
3065 	 * write-locking can be skipped safely.  in cgroup_update_dfl_csses()
3081 		list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list)  in cgroup_update_dfl_csses()
3094  * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses
3097  * Because css offlining is asynchronous, userland may try to re-enable a
3117 			if (!css || !percpu_ref_is_dying(&css->refcnt))  in cgroup_lock_and_drain_offline()
3121 			prepare_to_wait(&dsct->offline_waitq, &wait,  in cgroup_lock_and_drain_offline()
3126 			finish_wait(&dsct->offline_waitq, &wait);  in cgroup_lock_and_drain_offline()
3135  * cgroup_save_control - save control masks and dom_cgrp of a subtree
3138  * Save ->subtree_control, ->subtree_ss_mask and ->dom_cgrp to the
3148 		dsct->old_subtree_control = dsct->subtree_control;  in cgroup_save_control()
3149 		dsct->old_subtree_ss_mask = dsct->subtree_ss_mask;  in cgroup_save_control()
3150 		dsct->old_dom_cgrp = dsct->dom_cgrp;  in cgroup_save_control()
3155  * cgroup_propagate_control - refresh control masks of a subtree
3158  * For @cgrp and its subtree, ensure ->subtree_ss_mask matches
3159  * ->subtree_control and propagate controller availability through the
3168 		dsct->subtree_control &= cgroup_control(dsct);  in cgroup_propagate_control()
3169 		dsct->subtree_ss_mask =  in cgroup_propagate_control()
3170 			cgroup_calc_subtree_ss_mask(dsct->subtree_control,  in cgroup_propagate_control()
3176  * cgroup_restore_control - restore control masks and dom_cgrp of a subtree
3179  * Restore ->subtree_control, ->subtree_ss_mask and ->dom_cgrp from the
3189 		dsct->subtree_control = dsct->old_subtree_control;  in cgroup_restore_control()
3190 		dsct->subtree_ss_mask = dsct->old_subtree_ss_mask;  in cgroup_restore_control()
3191 		dsct->dom_cgrp = dsct->old_dom_cgrp;  in cgroup_restore_control()
3197 	struct cgroup_subsys *ss = css->ss;  in css_visible()
3198 	struct cgroup *cgrp = css->cgroup;  in css_visible()
3200 	if (cgroup_control(cgrp) & (1 << ss->id))  in css_visible()
3202 	if (!(cgroup_ss_mask(cgrp) & (1 << ss->id)))  in css_visible()
3204 	return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl;  in css_visible()
3208  * cgroup_apply_control_enable - enable or show csses according to control
3216  * Returns 0 on success, -errno on failure.  On failure, csses which have
3231 			if (!(cgroup_ss_mask(dsct) & (1 << ss->id)))  in cgroup_apply_control_enable()
3240 			WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_enable()
3254  * cgroup_apply_control_disable - kill or hide csses according to control
3263  * Controllers which may be depended upon should provide ->css_reset() for
3280 			WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_disable()
3282 			if (css->parent &&  in cgroup_apply_control_disable()
3283 			    !(cgroup_ss_mask(dsct) & (1 << ss->id))) {  in cgroup_apply_control_disable()
3287 				if (ss->css_reset)  in cgroup_apply_control_disable()
3288 					ss->css_reset(css);  in cgroup_apply_control_disable()
3295  * cgroup_apply_control - apply control mask updates to the subtree
3302  * 2. Update ->subtree_control masks in the subtree as desired.
3330  * cgroup_finalize_control - finalize control mask update
3355 	if (!cgroup_is_valid_domain(cgrp->dom_cgrp))  in cgroup_vet_subtree_control_enable()
3356 		return -EOPNOTSUPP;  in cgroup_vet_subtree_control_enable()
3365 			return -EOPNOTSUPP;  in cgroup_vet_subtree_control_enable()
3381 		return -EBUSY;  in cgroup_vet_subtree_control_enable()
3398 	 * Parse input - space separated list of subsystem names prefixed  in cgroup_subtree_control_write()
3399 	 * with either + or -.  in cgroup_subtree_control_write()
3407 			    strcmp(tok + 1, ss->name))  in cgroup_subtree_control_write()
3413 			} else if (*tok == '-') {  in cgroup_subtree_control_write()
3417 				return -EINVAL;  in cgroup_subtree_control_write()
3422 			return -EINVAL;  in cgroup_subtree_control_write()
3425 	cgrp = cgroup_kn_lock_live(of->kn, true);  in cgroup_subtree_control_write()
3427 		return -ENODEV;  in cgroup_subtree_control_write()
3431 			if (cgrp->subtree_control & (1 << ssid)) {  in cgroup_subtree_control_write()
3437 				ret = -ENOENT;  in cgroup_subtree_control_write()
3441 			if (!(cgrp->subtree_control & (1 << ssid))) {  in cgroup_subtree_control_write()
3448 				if (child->subtree_control & (1 << ssid)) {  in cgroup_subtree_control_write()
3449 					ret = -EBUSY;  in cgroup_subtree_control_write()
3468 	cgrp->subtree_control |= enable;  in cgroup_subtree_control_write()
3469 	cgrp->subtree_control &= ~disable;  in cgroup_subtree_control_write()
3476 	kernfs_activate(cgrp->kn);  in cgroup_subtree_control_write()
3478 	cgroup_kn_unlock(of->kn);  in cgroup_subtree_control_write()
3483  * cgroup_enable_threaded - make @cgrp threaded
3494 	struct cgroup *dom_cgrp = parent->dom_cgrp;  in cgroup_enable_threaded()
3512 	    cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask)  in cgroup_enable_threaded()
3513 		return -EOPNOTSUPP;  in cgroup_enable_threaded()
3518 		return -EOPNOTSUPP;  in cgroup_enable_threaded()
3528 			dsct->dom_cgrp = dom_cgrp;  in cgroup_enable_threaded()
3532 		parent->nr_threaded_children++;  in cgroup_enable_threaded()
3540 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_type_show()
3562 		return -EINVAL;  in cgroup_type_write()
3564 	/* drain dying csses before we re-apply (threaded) subtree control */  in cgroup_type_write()
3565 	cgrp = cgroup_kn_lock_live(of->kn, true);  in cgroup_type_write()
3567 		return -ENOENT;  in cgroup_type_write()
3572 	cgroup_kn_unlock(of->kn);  in cgroup_type_write()
3578 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_max_descendants_show()
3579 	int descendants = READ_ONCE(cgrp->max_descendants);  in cgroup_max_descendants_show()
3606 		return -ERANGE;  in cgroup_max_descendants_write()
3608 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_max_descendants_write()
3610 		return -ENOENT;  in cgroup_max_descendants_write()
3612 	cgrp->max_descendants = descendants;  in cgroup_max_descendants_write()
3614 	cgroup_kn_unlock(of->kn);  in cgroup_max_descendants_write()
3621 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_max_depth_show()
3622 	int depth = READ_ONCE(cgrp->max_depth);  in cgroup_max_depth_show()
3649 		return -ERANGE;  in cgroup_max_depth_write()
3651 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_max_depth_write()
3653 		return -ENOENT;  in cgroup_max_depth_write()
3655 	cgrp->max_depth = depth;  in cgroup_max_depth_write()
3657 	cgroup_kn_unlock(of->kn);  in cgroup_max_depth_write()
3664 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_events_show()
3667 	seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags));  in cgroup_events_show()
3674 	struct cgroup *cgroup = seq_css(seq)->cgroup;  in cgroup_stat_show()
3677 		   cgroup->nr_descendants);  in cgroup_stat_show()
3679 		   cgroup->nr_dying_descendants);  in cgroup_stat_show()
3691 	if (!ss->css_extra_stat_show)  in cgroup_extra_stat_show()
3698 	ret = ss->css_extra_stat_show(seq, css);  in cgroup_extra_stat_show()
3705 	struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup;  in cpu_stat_show()
3718 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_io_pressure_show()
3719 	struct psi_group *psi = cgroup_psi(cgrp);  in cgroup_io_pressure_show()  local
3721 	return psi_show(seq, psi, PSI_IO);  in cgroup_io_pressure_show()
3725 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_memory_pressure_show()
3726 	struct psi_group *psi = cgroup_psi(cgrp);  in cgroup_memory_pressure_show()  local
3728 	return psi_show(seq, psi, PSI_MEM);  in cgroup_memory_pressure_show()
3732 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_cpu_pressure_show()
3733 	struct psi_group *psi = cgroup_psi(cgrp);  in cgroup_cpu_pressure_show()  local
3735 	return psi_show(seq, psi, PSI_CPU);  in cgroup_cpu_pressure_show()
3741 	struct cgroup_file_ctx *ctx = of->priv;  in pressure_write()
3744 	struct psi_group *psi;  in pressure_write()  local
3746 	cgrp = cgroup_kn_lock_live(of->kn, false);  in pressure_write()
3748 		return -ENODEV;  in pressure_write()
3751 	cgroup_kn_unlock(of->kn);  in pressure_write()
3754 	if (ctx->psi.trigger) {  in pressure_write()
3756 		return -EBUSY;  in pressure_write()
3759 	psi = cgroup_psi(cgrp);  in pressure_write()
3760 	new = psi_trigger_create(psi, buf, res);  in pressure_write()
3766 	smp_store_release(&ctx->psi.trigger, new);  in pressure_write()
3796 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_irq_pressure_show()
3797 	struct psi_group *psi = cgroup_psi(cgrp);  in cgroup_irq_pressure_show()  local
3799 	return psi_show(seq, psi, PSI_IRQ);  in cgroup_irq_pressure_show()
3812 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_pressure_show()
3813 	struct psi_group *psi = cgroup_psi(cgrp);  in cgroup_pressure_show()  local
3815 	seq_printf(seq, "%d\n", psi->enabled);  in cgroup_pressure_show()
3827 	struct psi_group *psi;  in cgroup_pressure_write()  local
3834 		return -ERANGE;  in cgroup_pressure_write()
3836 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_pressure_write()
3838 		return -ENOENT;  in cgroup_pressure_write()
3840 	psi = cgroup_psi(cgrp);  in cgroup_pressure_write()
3841 	if (psi->enabled != enable) {  in cgroup_pressure_write()
3846 			cgroup_file_show(&cgrp->psi_files[i], enable);  in cgroup_pressure_write()
3848 		psi->enabled = enable;  in cgroup_pressure_write()
3850 			psi_cgroup_restart(psi);  in cgroup_pressure_write()
3853 	cgroup_kn_unlock(of->kn);  in cgroup_pressure_write()
3861 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_pressure_poll()
3863 	return psi_trigger_poll(&ctx->psi.trigger, of->file, pt);  in cgroup_pressure_poll()
3868 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_pressure_release()
3870 	psi_trigger_destroy(ctx->psi.trigger);  in cgroup_pressure_release()
3891 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_freeze_show()
3893 	seq_printf(seq, "%d\n", cgrp->freezer.freeze);  in cgroup_freeze_show()
3910 		return -ERANGE;  in cgroup_freeze_write()
3912 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_freeze_write()
3914 		return -ENOENT;  in cgroup_freeze_write()
3918 	cgroup_kn_unlock(of->kn);  in cgroup_freeze_write()
3931 	set_bit(CGRP_KILL, &cgrp->flags);  in __cgroup_kill()
3934 	css_task_iter_start(&cgrp->self, CSS_TASK_ITER_PROCS | CSS_TASK_ITER_THREADED, &it);  in __cgroup_kill()
3937 		if (task->flags & PF_KTHREAD)  in __cgroup_kill()
3949 	clear_bit(CGRP_KILL, &cgrp->flags);  in __cgroup_kill()
3976 		return -ERANGE;  in cgroup_kill_write()
3978 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_kill_write()
3980 		return -ENOENT;  in cgroup_kill_write()
3983 	 * Killing is a process directed operation, i.e. the whole thread-group  in cgroup_kill_write()
3985 	 * writable in non-threaded cgroups.  in cgroup_kill_write()
3988 		ret = -EOPNOTSUPP;  in cgroup_kill_write()
3992 	cgroup_kn_unlock(of->kn);  in cgroup_kill_write()
4005 		return -ENOMEM;  in cgroup_file_open()
4007 	ctx->ns = current->nsproxy->cgroup_ns;  in cgroup_file_open()
4008 	get_cgroup_ns(ctx->ns);  in cgroup_file_open()
4009 	of->priv = ctx;  in cgroup_file_open()
4011 	if (!cft->open)  in cgroup_file_open()
4014 	ret = cft->open(of);  in cgroup_file_open()
4016 		put_cgroup_ns(ctx->ns);  in cgroup_file_open()
4025 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_file_release()
4027 	if (cft->release)  in cgroup_file_release()
4028 		cft->release(of);  in cgroup_file_release()
4029 	put_cgroup_ns(ctx->ns);  in cgroup_file_release()
4036 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_file_write()
4037 	struct cgroup *cgrp = of->kn->parent->priv;  in cgroup_file_write()
4047 	 * files in an non-init namespace root from inside the namespace  in cgroup_file_write()
4048 	 * except for the files explicitly marked delegatable -  in cgroup_file_write()
4051 	if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) &&  in cgroup_file_write()
4052 	    !(cft->flags & CFTYPE_NS_DELEGATABLE) &&  in cgroup_file_write()
4053 	    ctx->ns != &init_cgroup_ns && ctx->ns->root_cset->dfl_cgrp == cgrp)  in cgroup_file_write()
4054 		return -EPERM;  in cgroup_file_write()
4056 	if (cft->write)  in cgroup_file_write()
4057 		return cft->write(of, buf, nbytes, off);  in cgroup_file_write()
4066 	css = cgroup_css(cgrp, cft->ss);  in cgroup_file_write()
4069 	if (cft->write_u64) {  in cgroup_file_write()
4073 			ret = cft->write_u64(css, cft, v);  in cgroup_file_write()
4074 	} else if (cft->write_s64) {  in cgroup_file_write()
4078 			ret = cft->write_s64(css, cft, v);  in cgroup_file_write()
4080 		ret = -EINVAL;  in cgroup_file_write()
4090 	if (cft->poll)  in cgroup_file_poll()
4091 		return cft->poll(of, pt);  in cgroup_file_poll()
4098 	return seq_cft(seq)->seq_start(seq, ppos);  in cgroup_seqfile_start()
4103 	return seq_cft(seq)->seq_next(seq, v, ppos);  in cgroup_seqfile_next()
4108 	if (seq_cft(seq)->seq_stop)  in cgroup_seqfile_stop()
4109 		seq_cft(seq)->seq_stop(seq, v);  in cgroup_seqfile_stop()
4117 	if (cft->seq_show)  in cgroup_seqfile_show()
4118 		return cft->seq_show(m, arg);  in cgroup_seqfile_show()
4120 	if (cft->read_u64)  in cgroup_seqfile_show()
4121 		seq_printf(m, "%llu\n", cft->read_u64(css, cft));  in cgroup_seqfile_show()
4122 	else if (cft->read_s64)  in cgroup_seqfile_show()
4123 		seq_printf(m, "%lld\n", cft->read_s64(css, cft));  in cgroup_seqfile_show()
4125 		return -EINVAL;  in cgroup_seqfile_show()
4179 	key = &cft->lockdep_key;  in cgroup_add_file()
4181 	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),  in cgroup_add_file()
4184 				  0, cft->kf_ops, cft,  in cgroup_add_file()
4195 	if (cft->file_offset) {  in cgroup_add_file()
4196 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_add_file()
4198 		timer_setup(&cfile->notify_timer, cgroup_file_notify_timer, 0);  in cgroup_add_file()
4201 		cfile->kn = kn;  in cgroup_add_file()
4209  * cgroup_addrm_files - add or remove files to a cgroup directory
4211  * @cgrp: the target cgroup (usually css->cgroup)
4228 	for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) {  in cgroup_addrm_files()
4229 		/* does cft->flags tell us to skip this file on @cgrp? */  in cgroup_addrm_files()
4230 		if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))  in cgroup_addrm_files()
4232 		if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))  in cgroup_addrm_files()
4234 		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))  in cgroup_addrm_files()
4236 		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))  in cgroup_addrm_files()
4238 		if ((cft->flags & CFTYPE_DEBUG) && !cgroup_debug)  in cgroup_addrm_files()
4244 					__func__, cft->name, ret);  in cgroup_addrm_files()
4259 	struct cgroup *root = &ss->root->cgrp;  in cgroup_apply_cftypes()
4267 		struct cgroup *cgrp = css->cgroup;  in cgroup_apply_cftypes()
4269 		if (!(css->flags & CSS_VISIBLE))  in cgroup_apply_cftypes()
4278 		kernfs_activate(root->kn);  in cgroup_apply_cftypes()
4286 	for (cft = cfts; cft->name[0] != '\0'; cft++) {  in cgroup_exit_cftypes()
4288 		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)  in cgroup_exit_cftypes()
4289 			kfree(cft->kf_ops);  in cgroup_exit_cftypes()
4290 		cft->kf_ops = NULL;  in cgroup_exit_cftypes()
4291 		cft->ss = NULL;  in cgroup_exit_cftypes()
4294 		cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL |  in cgroup_exit_cftypes()
4304 	for (cft = cfts; cft->name[0] != '\0'; cft++) {  in cgroup_init_cftypes()
4307 		WARN_ON(cft->ss || cft->kf_ops);  in cgroup_init_cftypes()
4309 		if (cft->flags & __CFTYPE_ADDED) {  in cgroup_init_cftypes()
4310 			ret = -EBUSY;  in cgroup_init_cftypes()
4314 		if (cft->seq_start)  in cgroup_init_cftypes()
4323 		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {  in cgroup_init_cftypes()
4326 				ret = -ENOMEM;  in cgroup_init_cftypes()
4329 			kf_ops->atomic_write_len = cft->max_write_len;  in cgroup_init_cftypes()
4332 		cft->kf_ops = kf_ops;  in cgroup_init_cftypes()
4333 		cft->ss = ss;  in cgroup_init_cftypes()
4334 		cft->flags |= __CFTYPE_ADDED;  in cgroup_init_cftypes()
4346 	list_del(&cfts->node);  in cgroup_rm_cftypes_locked()
4353  * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
4354  * @cfts: zero-length name terminated array of cftypes
4360  * Returns 0 on successful unregistration, -ENOENT if @cfts is not
4371 		return -ENOENT;  in cgroup_rm_cftypes()
4380  * cgroup_add_cftypes - add an array of cftypes to a subsystem
4382  * @cfts: zero-length name terminated array of cftypes
4389  * Returns 0 on successful registration, -errno on failure.  Note that this
4397 	if (!cgroup_ssid_enabled(ss->id))  in cgroup_add_cftypes()
4409 	list_add_tail(&cfts->node, &ss->cfts);  in cgroup_add_cftypes()
4419  * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
4421  * @cfts: zero-length name terminated array of cftypes
4430 	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)  in cgroup_add_dfl_cftypes()
4431 		cft->flags |= __CFTYPE_ONLY_ON_DFL;  in cgroup_add_dfl_cftypes()
4436  * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
4438  * @cfts: zero-length name terminated array of cftypes
4447 	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)  in cgroup_add_legacy_cftypes()
4448 		cft->flags |= __CFTYPE_NOT_ON_DFL;  in cgroup_add_legacy_cftypes()
4453  * cgroup_file_notify - generate a file modified event for a cgroup_file
4456  * @cfile must have been obtained by setting cftype->file_offset.
4463 	if (cfile->kn) {  in cgroup_file_notify()
4464 		unsigned long last = cfile->notified_at;  in cgroup_file_notify()
4468 			timer_reduce(&cfile->notify_timer, next);  in cgroup_file_notify()
4470 			kernfs_notify(cfile->kn);  in cgroup_file_notify()
4471 			cfile->notified_at = jiffies;  in cgroup_file_notify()
4478  * cgroup_file_show - show or hide a hidden cgroup file
4479  * @cfile: target cgroup_file obtained by setting cftype->file_offset
4487 	kn = cfile->kn;  in cgroup_file_show()
4498  * css_next_child - find the next child of a given css
4507  * If a subsystem synchronizes ->css_online() and the start of iteration, a
4508  * css which finished ->css_online() is guaranteed to be visible in the
4510  * A css which hasn't finished ->css_online() or already finished
4511  * ->css_offline() may show up during traversal.  It's each subsystem's
4523 	 * Once a cgroup is removed, its ->sibling.next is no longer  in css_next_child()
4531 	 * have dropped rcu_read_lock() in-between iterations.  in css_next_child()
4542 		next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);  in css_next_child()
4543 	} else if (likely(!(pos->flags & CSS_RELEASED))) {  in css_next_child()
4544 		next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);  in css_next_child()
4546 		list_for_each_entry_rcu(next, &parent->children, sibling,  in css_next_child()
4548 			if (next->serial_nr > pos->serial_nr)  in css_next_child()
4556 	if (&next->sibling != &parent->children)  in css_next_child()
4562  * css_next_descendant_pre - find the next descendant for pre-order walk
4567  * to visit for pre-order traversal of @root's descendants.  @root is
4575  * If a subsystem synchronizes ->css_online() and the start of iteration, a
4576  * css which finished ->css_online() is guaranteed to be visible in the
4578  * A css which hasn't finished ->css_online() or already finished
4579  * ->css_offline() may show up during traversal.  It's each subsystem's
4601 		next = css_next_child(pos, pos->parent);  in css_next_descendant_pre()
4604 		pos = pos->parent;  in css_next_descendant_pre()
4612  * css_rightmost_descendant - return the rightmost descendant of a css
4616  * is returned.  This can be used during pre-order traversal to skip
4633 		/* ->prev isn't RCU safe, walk ->next till the end */  in css_rightmost_descendant()
4656  * css_next_descendant_post - find the next descendant for post-order walk
4661  * to visit for post-order traversal of @root's descendants.  @root is
4670  * If a subsystem synchronizes ->css_online() and the start of iteration, a
4671  * css which finished ->css_online() is guaranteed to be visible in the
4673  * A css which hasn't finished ->css_online() or already finished
4674  * ->css_offline() may show up during traversal.  It's each subsystem's
4694 	next = css_next_child(pos, pos->parent);  in css_next_descendant_post()
4699 	return pos->parent;  in css_next_descendant_post()
4703  * css_has_online_children - does a css have online children
4717 		if (child->flags & CSS_ONLINE) {  in css_has_online_children()
4728 	struct list_head *l;  in css_task_iter_next_css_set()  local
4735 	if (it->tcset_pos) {  in css_task_iter_next_css_set()
4736 		l = it->tcset_pos->next;  in css_task_iter_next_css_set()
4738 		if (l != it->tcset_head) {  in css_task_iter_next_css_set()
4739 			it->tcset_pos = l;  in css_task_iter_next_css_set()
4740 			return container_of(l, struct css_set,  in css_task_iter_next_css_set()
4744 		it->tcset_pos = NULL;  in css_task_iter_next_css_set()
4748 	l = it->cset_pos;  in css_task_iter_next_css_set()
4749 	l = l->next;  in css_task_iter_next_css_set()
4750 	if (l == it->cset_head) {  in css_task_iter_next_css_set()
4751 		it->cset_pos = NULL;  in css_task_iter_next_css_set()
4755 	if (it->ss) {  in css_task_iter_next_css_set()
4756 		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);  in css_task_iter_next_css_set()
4758 		link = list_entry(l, struct cgrp_cset_link, cset_link);  in css_task_iter_next_css_set()
4759 		cset = link->cset;  in css_task_iter_next_css_set()
4762 	it->cset_pos = l;  in css_task_iter_next_css_set()
4765 	if (it->flags & CSS_TASK_ITER_THREADED) {  in css_task_iter_next_css_set()
4766 		if (it->cur_dcset)  in css_task_iter_next_css_set()
4767 			put_css_set_locked(it->cur_dcset);  in css_task_iter_next_css_set()
4768 		it->cur_dcset = cset;  in css_task_iter_next_css_set()
4771 		it->tcset_head = &cset->threaded_csets;  in css_task_iter_next_css_set()
4772 		it->tcset_pos = &cset->threaded_csets;  in css_task_iter_next_css_set()
4779  * css_task_iter_advance_css_set - advance a task iterator to the next css_set
4790 	/* Advance to the next non-empty css_set and find first non-empty tasks list*/  in css_task_iter_advance_css_set()
4792 		if (!list_empty(&cset->tasks)) {  in css_task_iter_advance_css_set()
4793 			it->cur_tasks_head = &cset->tasks;  in css_task_iter_advance_css_set()
4795 		} else if (!list_empty(&cset->mg_tasks)) {  in css_task_iter_advance_css_set()
4796 			it->cur_tasks_head = &cset->mg_tasks;  in css_task_iter_advance_css_set()
4798 		} else if (!list_empty(&cset->dying_tasks)) {  in css_task_iter_advance_css_set()
4799 			it->cur_tasks_head = &cset->dying_tasks;  in css_task_iter_advance_css_set()
4804 		it->task_pos = NULL;  in css_task_iter_advance_css_set()
4807 	it->task_pos = it->cur_tasks_head->next;  in css_task_iter_advance_css_set()
4812 	 * the lock is re-acquired.  Iteration is performed at two levels -  in css_task_iter_advance_css_set()
4824 	if (it->cur_cset) {  in css_task_iter_advance_css_set()
4825 		list_del(&it->iters_node);  in css_task_iter_advance_css_set()
4826 		put_css_set_locked(it->cur_cset);  in css_task_iter_advance_css_set()
4829 	it->cur_cset = cset;  in css_task_iter_advance_css_set()
4830 	list_add(&it->iters_node, &cset->task_iters);  in css_task_iter_advance_css_set()
4838 	if (it->task_pos == &task->cg_list) {  in css_task_iter_skip()
4839 		it->task_pos = it->task_pos->next;  in css_task_iter_skip()
4840 		it->flags |= CSS_TASK_ITER_SKIPPED;  in css_task_iter_skip()
4850 	if (it->task_pos) {  in css_task_iter_advance()
4856 		if (it->flags & CSS_TASK_ITER_SKIPPED)  in css_task_iter_advance()
4857 			it->flags &= ~CSS_TASK_ITER_SKIPPED;  in css_task_iter_advance()
4859 			it->task_pos = it->task_pos->next;  in css_task_iter_advance()
4861 		if (it->task_pos == &it->cur_cset->tasks) {  in css_task_iter_advance()
4862 			it->cur_tasks_head = &it->cur_cset->mg_tasks;  in css_task_iter_advance()
4863 			it->task_pos = it->cur_tasks_head->next;  in css_task_iter_advance()
4865 		if (it->task_pos == &it->cur_cset->mg_tasks) {  in css_task_iter_advance()
4866 			it->cur_tasks_head = &it->cur_cset->dying_tasks;  in css_task_iter_advance()
4867 			it->task_pos = it->cur_tasks_head->next;  in css_task_iter_advance()
4869 		if (it->task_pos == &it->cur_cset->dying_tasks)  in css_task_iter_advance()
4876 	if (!it->task_pos)  in css_task_iter_advance()
4879 	task = list_entry(it->task_pos, struct task_struct, cg_list);  in css_task_iter_advance()
4881 	if (it->flags & CSS_TASK_ITER_PROCS) {  in css_task_iter_advance()
4887 		if (it->cur_tasks_head == &it->cur_cset->dying_tasks &&  in css_task_iter_advance()
4888 		    !atomic_read(&task->signal->live))  in css_task_iter_advance()
4892 		if (it->cur_tasks_head == &it->cur_cset->dying_tasks)  in css_task_iter_advance()
4898  * css_task_iter_start - initiate task iteration
4915 	it->ss = css->ss;  in css_task_iter_start()
4916 	it->flags = flags;  in css_task_iter_start()
4918 	if (CGROUP_HAS_SUBSYS_CONFIG && it->ss)  in css_task_iter_start()
4919 		it->cset_pos = &css->cgroup->e_csets[css->ss->id];  in css_task_iter_start()
4921 		it->cset_pos = &css->cgroup->cset_links;  in css_task_iter_start()
4923 	it->cset_head = it->cset_pos;  in css_task_iter_start()
4931  * css_task_iter_next - return the next task for the iterator
4940 	if (it->cur_task) {  in css_task_iter_next()
4941 		put_task_struct(it->cur_task);  in css_task_iter_next()
4942 		it->cur_task = NULL;  in css_task_iter_next()
4947 	/* @it may be half-advanced by skips, finish advancing */  in css_task_iter_next()
4948 	if (it->flags & CSS_TASK_ITER_SKIPPED)  in css_task_iter_next()
4951 	if (it->task_pos) {  in css_task_iter_next()
4952 		it->cur_task = list_entry(it->task_pos, struct task_struct,  in css_task_iter_next()
4954 		get_task_struct(it->cur_task);  in css_task_iter_next()
4960 	return it->cur_task;  in css_task_iter_next()
4964  * css_task_iter_end - finish task iteration
4971 	if (it->cur_cset) {  in css_task_iter_end()
4973 		list_del(&it->iters_node);  in css_task_iter_end()
4974 		put_css_set_locked(it->cur_cset);  in css_task_iter_end()
4978 	if (it->cur_dcset)  in css_task_iter_end()
4979 		put_css_set(it->cur_dcset);  in css_task_iter_end()
4981 	if (it->cur_task)  in css_task_iter_end()
4982 		put_task_struct(it->cur_task);  in css_task_iter_end()
4987 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_procs_release()
4989 	if (ctx->procs.started)  in cgroup_procs_release()
4990 		css_task_iter_end(&ctx->procs.iter);  in cgroup_procs_release()
4995 	struct kernfs_open_file *of = s->private;  in cgroup_procs_next()
4996 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_procs_next()
5001 	return css_task_iter_next(&ctx->procs.iter);  in cgroup_procs_next()
5007 	struct kernfs_open_file *of = s->private;  in __cgroup_procs_start()
5008 	struct cgroup *cgrp = seq_css(s)->cgroup;  in __cgroup_procs_start()
5009 	struct cgroup_file_ctx *ctx = of->priv;  in __cgroup_procs_start()
5010 	struct css_task_iter *it = &ctx->procs.iter;  in __cgroup_procs_start()
5016 	if (!ctx->procs.started) {  in __cgroup_procs_start()
5018 			return ERR_PTR(-EINVAL);  in __cgroup_procs_start()
5019 		css_task_iter_start(&cgrp->self, iter_flags, it);  in __cgroup_procs_start()
5020 		ctx->procs.started = true;  in __cgroup_procs_start()
5023 		css_task_iter_start(&cgrp->self, iter_flags, it);  in __cgroup_procs_start()
5025 		return it->cur_task;  in __cgroup_procs_start()
5032 	struct cgroup *cgrp = seq_css(s)->cgroup;  in cgroup_procs_start()
5041 		return ERR_PTR(-EOPNOTSUPP);  in cgroup_procs_start()
5060 	inode = kernfs_get_inode(sb, cgrp->procs_file.kn);  in cgroup_may_write()
5062 		return -ENOMEM;  in cgroup_may_write()
5093 	    (!cgroup_is_descendant(src_cgrp, ns->root_cset->dfl_cgrp) ||  in cgroup_procs_write_permission()
5094 	     !cgroup_is_descendant(dst_cgrp, ns->root_cset->dfl_cgrp)))  in cgroup_procs_write_permission()
5095 		return -ENOENT;  in cgroup_procs_write_permission()
5115 	if (!threadgroup && (src_cgrp->dom_cgrp != dst_cgrp->dom_cgrp))  in cgroup_attach_permissions()
5116 		ret = -EOPNOTSUPP;  in cgroup_attach_permissions()
5124 	struct cgroup_file_ctx *ctx = of->priv;  in __cgroup_procs_write()
5131 	dst_cgrp = cgroup_kn_lock_live(of->kn, false);  in __cgroup_procs_write()
5133 		return -ENODEV;  in __cgroup_procs_write()
5150 	saved_cred = override_creds(of->file->f_cred);  in __cgroup_procs_write()
5152 					of->file->f_path.dentry->d_sb,  in __cgroup_procs_write()
5153 					threadgroup, ctx->ns);  in __cgroup_procs_write()
5163 	cgroup_kn_unlock(of->kn);  in __cgroup_procs_write()
5306  * css destruction is four-stage process.
5324  * and thus involve punting to css->destroy_work adding two additional
5331 	struct cgroup_subsys *ss = css->ss;  in css_free_rwork_fn()
5332 	struct cgroup *cgrp = css->cgroup;  in css_free_rwork_fn()
5334 	percpu_ref_exit(&css->refcnt);  in css_free_rwork_fn()
5338 		struct cgroup_subsys_state *parent = css->parent;  in css_free_rwork_fn()
5339 		int id = css->id;  in css_free_rwork_fn()
5341 		ss->css_free(css);  in css_free_rwork_fn()
5342 		cgroup_idr_remove(&ss->css_idr, id);  in css_free_rwork_fn()
5349 		atomic_dec(&cgrp->root->nr_cgrps);  in css_free_rwork_fn()
5351 		cancel_work_sync(&cgrp->release_agent_work);  in css_free_rwork_fn()
5361 			kernfs_put(cgrp->kn);  in css_free_rwork_fn()
5371 			cgroup_destroy_root(cgrp->root);  in css_free_rwork_fn()
5380 	struct cgroup_subsys *ss = css->ss;  in css_release_work_fn()
5381 	struct cgroup *cgrp = css->cgroup;  in css_release_work_fn()
5385 	css->flags |= CSS_RELEASED;  in css_release_work_fn()
5386 	list_del_rcu(&css->sibling);  in css_release_work_fn()
5390 		if (!list_empty(&css->rstat_css_node)) {  in css_release_work_fn()
5392 			list_del_rcu(&css->rstat_css_node);  in css_release_work_fn()
5395 		cgroup_idr_replace(&ss->css_idr, NULL, css->id);  in css_release_work_fn()
5396 		if (ss->css_released)  in css_release_work_fn()
5397 			ss->css_released(css);  in css_release_work_fn()
5409 			tcgrp->nr_dying_descendants--;  in css_release_work_fn()
5414 		 * cgroup from dentry without going through kernfs -  in css_release_work_fn()
5417 		 * cgrp->kn->priv backpointer.  in css_release_work_fn()
5419 		if (cgrp->kn)  in css_release_work_fn()
5420 			RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv,  in css_release_work_fn()
5426 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_release_work_fn()
5427 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_release_work_fn()
5435 	INIT_WORK(&css->destroy_work, css_release_work_fn);  in css_release()
5436 	queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_release()
5447 	css->cgroup = cgrp;  in init_and_link_css()
5448 	css->ss = ss;  in init_and_link_css()
5449 	css->id = -1;  in init_and_link_css()
5450 	INIT_LIST_HEAD(&css->sibling);  in init_and_link_css()
5451 	INIT_LIST_HEAD(&css->children);  in init_and_link_css()
5452 	INIT_LIST_HEAD(&css->rstat_css_node);  in init_and_link_css()
5453 	css->serial_nr = css_serial_nr_next++;  in init_and_link_css()
5454 	atomic_set(&css->online_cnt, 0);  in init_and_link_css()
5457 		css->parent = cgroup_css(cgroup_parent(cgrp), ss);  in init_and_link_css()
5458 		css_get(css->parent);  in init_and_link_css()
5461 	if (ss->css_rstat_flush)  in init_and_link_css()
5462 		list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list);  in init_and_link_css()
5467 /* invoke ->css_online() on a new CSS and mark it online if successful */
5470 	struct cgroup_subsys *ss = css->ss;  in online_css()
5475 	if (ss->css_online)  in online_css()
5476 		ret = ss->css_online(css);  in online_css()
5478 		css->flags |= CSS_ONLINE;  in online_css()
5479 		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);  in online_css()
5481 		atomic_inc(&css->online_cnt);  in online_css()
5482 		if (css->parent)  in online_css()
5483 			atomic_inc(&css->parent->online_cnt);  in online_css()
5488 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */
5491 	struct cgroup_subsys *ss = css->ss;  in offline_css()
5495 	if (!(css->flags & CSS_ONLINE))  in offline_css()
5498 	if (ss->css_offline)  in offline_css()
5499 		ss->css_offline(css);  in offline_css()
5501 	css->flags &= ~CSS_ONLINE;  in offline_css()
5502 	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);  in offline_css()
5504 	wake_up_all(&css->cgroup->offline_waitq);  in offline_css()
5508  * css_create - create a cgroup_subsys_state
5512  * Create a new css associated with @cgrp - @ss pair.  On success, the new
5514  * interface files.  Returns 0 on success, -errno on failure.
5526 	css = ss->css_alloc(parent_css);  in css_create()
5528 		css = ERR_PTR(-ENOMEM);  in css_create()
5534 	err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);  in css_create()
5538 	err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL);  in css_create()
5541 	css->id = err;  in css_create()
5544 	list_add_tail_rcu(&css->sibling, &parent_css->children);  in css_create()
5545 	cgroup_idr_replace(&ss->css_idr, css, css->id);  in css_create()
5554 	list_del_rcu(&css->sibling);  in css_create()
5556 	list_del_rcu(&css->rstat_css_node);  in css_create()
5557 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_create()
5558 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_create()
5570 	struct cgroup_root *root = parent->root;  in cgroup_create()
5573 	int level = parent->level + 1;  in cgroup_create()
5579 		return ERR_PTR(-ENOMEM);  in cgroup_create()
5581 	ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);  in cgroup_create()
5590 	kn = kernfs_create_dir(parent->kn, name, mode, cgrp);  in cgroup_create()
5595 	cgrp->kn = kn;  in cgroup_create()
5599 	cgrp->self.parent = &parent->self;  in cgroup_create()
5600 	cgrp->root = root;  in cgroup_create()
5601 	cgrp->level = level;  in cgroup_create()
5615 	cgrp->freezer.e_freeze = parent->freezer.e_freeze;  in cgroup_create()
5616 	if (cgrp->freezer.e_freeze) {  in cgroup_create()
5623 		set_bit(CGRP_FREEZE, &cgrp->flags);  in cgroup_create()
5624 		set_bit(CGRP_FROZEN, &cgrp->flags);  in cgroup_create()
5629 		cgrp->ancestors[tcgrp->level] = tcgrp;  in cgroup_create()
5632 			tcgrp->nr_descendants++;  in cgroup_create()
5639 			if (cgrp->freezer.e_freeze)  in cgroup_create()
5640 				tcgrp->freezer.nr_frozen_descendants++;  in cgroup_create()
5646 		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);  in cgroup_create()
5648 	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))  in cgroup_create()
5649 		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);  in cgroup_create()
5651 	cgrp->self.serial_nr = css_serial_nr_next++;  in cgroup_create()
5654 	list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);  in cgroup_create()
5655 	atomic_inc(&root->nr_cgrps);  in cgroup_create()
5663 		cgrp->subtree_control = cgroup_control(cgrp);  in cgroup_create()
5672 	kernfs_remove(cgrp->kn);  in cgroup_create()
5676 	percpu_ref_exit(&cgrp->self.refcnt);  in cgroup_create()
5691 		if (cgroup->nr_descendants >= cgroup->max_descendants)  in cgroup_check_hierarchy_limits()
5694 		if (level > cgroup->max_depth)  in cgroup_check_hierarchy_limits()
5712 		return -EINVAL;  in cgroup_mkdir()
5716 		return -ENODEV;  in cgroup_mkdir()
5719 		ret = -EAGAIN;  in cgroup_mkdir()
5731 	 * that @cgrp->kn is always accessible.  in cgroup_mkdir()
5733 	kernfs_get(cgrp->kn);  in cgroup_mkdir()
5735 	ret = cgroup_kn_set_ugid(cgrp->kn);  in cgroup_mkdir()
5739 	ret = css_populate_dir(&cgrp->self);  in cgroup_mkdir()
5750 	kernfs_activate(cgrp->kn);  in cgroup_mkdir()
5778 		css = css->parent;  in css_killed_work_fn()
5779 	} while (css && atomic_dec_and_test(&css->online_cnt));  in css_killed_work_fn()
5790 	if (atomic_dec_and_test(&css->online_cnt)) {  in css_killed_ref_fn()
5791 		INIT_WORK(&css->destroy_work, css_killed_work_fn);  in css_killed_ref_fn()
5792 		queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_killed_ref_fn()
5797  * kill_css - destroy a css
5801  * files and putting its base reference.  ->css_offline() will be invoked
5809 	if (css->flags & CSS_DYING)  in kill_css()
5812 	css->flags |= CSS_DYING;  in kill_css()
5822 	 * until after ->css_offline().  in kill_css()
5827 	 * cgroup core guarantees that, by the time ->css_offline() is  in kill_css()
5836 	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);  in kill_css()
5840  * cgroup_destroy_locked - the first stage of cgroup destruction
5846  * ->css_offline() is invoked.  To satisfy all the requirements,
5854  * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
5856  *     cgroup is RCU-freed.
5878 		return -EBUSY;  in cgroup_destroy_locked()
5882 	 * ->self.children as dead children linger on it while being  in cgroup_destroy_locked()
5885 	if (css_has_online_children(&cgrp->self))  in cgroup_destroy_locked()
5886 		return -EBUSY;  in cgroup_destroy_locked()
5894 	cgrp->self.flags &= ~CSS_ONLINE;  in cgroup_destroy_locked()
5897 	list_for_each_entry(link, &cgrp->cset_links, cset_link)  in cgroup_destroy_locked()
5898 		link->cset->dead = true;  in cgroup_destroy_locked()
5906 	css_clear_dir(&cgrp->self);  in cgroup_destroy_locked()
5907 	kernfs_remove(cgrp->kn);  in cgroup_destroy_locked()
5910 		parent->nr_threaded_children--;  in cgroup_destroy_locked()
5914 		tcgrp->nr_descendants--;  in cgroup_destroy_locked()
5915 		tcgrp->nr_dying_descendants++;  in cgroup_destroy_locked()
5920 		if (test_bit(CGRP_FROZEN, &cgrp->flags))  in cgroup_destroy_locked()
5921 			tcgrp->freezer.nr_frozen_descendants--;  in cgroup_destroy_locked()
5930 	percpu_ref_kill(&cgrp->self.refcnt);  in cgroup_destroy_locked()
5963 	pr_debug("Initializing cgroup subsys %s\n", ss->name);  in cgroup_init_subsys()
5967 	idr_init(&ss->css_idr);  in cgroup_init_subsys()
5968 	INIT_LIST_HEAD(&ss->cfts);  in cgroup_init_subsys()
5971 	ss->root = &cgrp_dfl_root;  in cgroup_init_subsys()
5972 	css = ss->css_alloc(NULL);  in cgroup_init_subsys()
5981 	css->flags |= CSS_NO_REF;  in cgroup_init_subsys()
5985 		css->id = 1;  in cgroup_init_subsys()
5987 		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);  in cgroup_init_subsys()
5988 		BUG_ON(css->id < 0);  in cgroup_init_subsys()
5992 	 * pointer to this state - since the subsystem is  in cgroup_init_subsys()
5995 	init_css_set.subsys[ss->id] = css;  in cgroup_init_subsys()
5997 	have_fork_callback |= (bool)ss->fork << ss->id;  in cgroup_init_subsys()
5998 	have_exit_callback |= (bool)ss->exit << ss->id;  in cgroup_init_subsys()
5999 	have_release_callback |= (bool)ss->release << ss->id;  in cgroup_init_subsys()
6000 	have_canfork_callback |= (bool)ss->can_fork << ss->id;  in cgroup_init_subsys()
6013  * cgroup_init_early - cgroup initialization at system boot
6031 		WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,  in cgroup_init_early()
6033 		     i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,  in cgroup_init_early()
6034 		     ss->id, ss->name);  in cgroup_init_early()
6038 		ss->id = i;  in cgroup_init_early()
6039 		ss->name = cgroup_subsys_name[i];  in cgroup_init_early()
6040 		if (!ss->legacy_name)  in cgroup_init_early()
6041 			ss->legacy_name = cgroup_subsys_name[i];  in cgroup_init_early()
6043 		if (ss->early_init)  in cgroup_init_early()
6050  * cgroup_init - cgroup initialization
6083 		if (ss->early_init) {  in cgroup_init()
6085 				init_css_set.subsys[ss->id];  in cgroup_init()
6087 			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,  in cgroup_init()
6089 			BUG_ON(css->id < 0);  in cgroup_init()
6107 			       ss->name);  in cgroup_init()
6109 		cgrp_dfl_root.subsys_mask |= 1 << ss->id;  in cgroup_init()
6112 		WARN_ON(ss->implicit_on_dfl && !ss->threaded);  in cgroup_init()
6114 		if (ss->implicit_on_dfl)  in cgroup_init()
6115 			cgrp_dfl_implicit_ss_mask |= 1 << ss->id;  in cgroup_init()
6116 		else if (!ss->dfl_cftypes)  in cgroup_init()
6117 			cgrp_dfl_inhibit_ss_mask |= 1 << ss->id;  in cgroup_init()
6119 		if (ss->threaded)  in cgroup_init()
6120 			cgrp_dfl_threaded_ss_mask |= 1 << ss->id;  in cgroup_init()
6122 		if (ss->dfl_cftypes == ss->legacy_cftypes) {  in cgroup_init()
6123 			WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));  in cgroup_init()
6125 			WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));  in cgroup_init()
6126 			WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));  in cgroup_init()
6129 		if (ss->bind)  in cgroup_init()
6130 			ss->bind(init_css_set.subsys[ssid]);  in cgroup_init()
6137 	/* init_css_set.subsys[] has been updated, re-hash */  in cgroup_init()
6193 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6197 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6202 	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);  in cgroup_get_from_id()
6210 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6215 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6224  *  - Print task's cgroup paths into seq_file, one line for each hierarchy
6225  *  - Used for /proc/<pid>/cgroup.
6234 	retval = -ENOMEM;  in proc_cgroup_show()
6250 		seq_printf(m, "%d:", root->hierarchy_id);  in proc_cgroup_show()
6253 				if (root->subsys_mask & (1 << ssid))  in proc_cgroup_show()
6255 						   ss->legacy_name);  in proc_cgroup_show()
6256 		if (strlen(root->name))  in proc_cgroup_show()
6258 				   root->name);  in proc_cgroup_show()
6272 		if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) {  in proc_cgroup_show()
6274 						current->nsproxy->cgroup_ns);  in proc_cgroup_show()
6276 				retval = -ENAMETOOLONG;  in proc_cgroup_show()
6301  * cgroup_fork - initialize cgroup related fields during copy_process()
6309 	RCU_INIT_POINTER(child->cgroups, &init_css_set);  in cgroup_fork()
6310 	INIT_LIST_HEAD(&child->cg_list);  in cgroup_fork()
6314  * cgroup_v1v2_get_from_file - get a cgroup pointer from a file pointer
6325 	css = css_tryget_online_from_dir(f->f_path.dentry, NULL);  in cgroup_v1v2_get_from_file()
6329 	return css->cgroup;  in cgroup_v1v2_get_from_file()
6333  * cgroup_get_from_file - same as cgroup_v1v2_get_from_file, but only supports
6346 		return ERR_PTR(-EBADF);  in cgroup_get_from_file()
6353  * cgroup_css_set_fork - find or create a css_set for a child process
6377 	if (kargs->flags & CLONE_INTO_CGROUP)  in cgroup_css_set_fork()
6387 	if (!(kargs->flags & CLONE_INTO_CGROUP)) {  in cgroup_css_set_fork()
6388 		kargs->cset = cset;  in cgroup_css_set_fork()
6392 	f = fget_raw(kargs->cgroup);  in cgroup_css_set_fork()
6394 		ret = -EBADF;  in cgroup_css_set_fork()
6397 	sb = f->f_path.dentry->d_sb;  in cgroup_css_set_fork()
6407 		ret = -ENODEV;  in cgroup_css_set_fork()
6428 	 *   write(fd, <child-pid>, ...);  in cgroup_css_set_fork()
6434 	ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb,  in cgroup_css_set_fork()
6435 					!(kargs->flags & CLONE_THREAD),  in cgroup_css_set_fork()
6436 					current->nsproxy->cgroup_ns);  in cgroup_css_set_fork()
6440 	kargs->cset = find_css_set(cset, dst_cgrp);  in cgroup_css_set_fork()
6441 	if (!kargs->cset) {  in cgroup_css_set_fork()
6442 		ret = -ENOMEM;  in cgroup_css_set_fork()
6448 	kargs->cgrp = dst_cgrp;  in cgroup_css_set_fork()
6459 	if (kargs->cset)  in cgroup_css_set_fork()
6460 		put_css_set(kargs->cset);  in cgroup_css_set_fork()
6465  * cgroup_css_set_put_fork - drop references we took during fork
6476 	if (kargs->flags & CLONE_INTO_CGROUP) {  in cgroup_css_set_put_fork()
6477 		struct cgroup *cgrp = kargs->cgrp;  in cgroup_css_set_put_fork()
6478 		struct css_set *cset = kargs->cset;  in cgroup_css_set_put_fork()
6484 			kargs->cset = NULL;  in cgroup_css_set_put_fork()
6489 			kargs->cgrp = NULL;  in cgroup_css_set_put_fork()
6495  * cgroup_can_fork - called on a new task before the process is exposed
6515 		ret = ss->can_fork(child, kargs->cset);  in cgroup_can_fork()
6526 		if (ss->cancel_fork)  in cgroup_can_fork()
6527 			ss->cancel_fork(child, kargs->cset);  in cgroup_can_fork()
6536  * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
6551 		if (ss->cancel_fork)  in cgroup_cancel_fork()
6552 			ss->cancel_fork(child, kargs->cset);  in cgroup_cancel_fork()
6558  * cgroup_post_fork - finalize cgroup setup for the child process
6575 	cset = kargs->cset;  in cgroup_post_fork()
6576 	kargs->cset = NULL;  in cgroup_post_fork()
6581 	if (likely(child->pid)) {  in cgroup_post_fork()
6582 		if (kargs->cgrp)  in cgroup_post_fork()
6583 			cgrp_flags = kargs->cgrp->flags;  in cgroup_post_fork()
6585 			cgrp_flags = cset->dfl_cgrp->flags;  in cgroup_post_fork()
6587 		WARN_ON_ONCE(!list_empty(&child->cg_list));  in cgroup_post_fork()
6588 		cset->nr_tasks++;  in cgroup_post_fork()
6595 	if (!(child->flags & PF_KTHREAD)) {  in cgroup_post_fork()
6602 			spin_lock(&child->sighand->siglock);  in cgroup_post_fork()
6603 			WARN_ON_ONCE(child->frozen);  in cgroup_post_fork()
6604 			child->jobctl |= JOBCTL_TRAP_FREEZE;  in cgroup_post_fork()
6605 			spin_unlock(&child->sighand->siglock);  in cgroup_post_fork()
6626 	 * Call ss->fork().  This must happen after @child is linked on  in cgroup_post_fork()
6627 	 * css_set; otherwise, @child might change state between ->fork()  in cgroup_post_fork()
6631 		ss->fork(child);  in cgroup_post_fork()
6635 	if (kargs->flags & CLONE_NEWCGROUP) {  in cgroup_post_fork()
6636 		struct css_set *rcset = child->nsproxy->cgroup_ns->root_cset;  in cgroup_post_fork()
6639 		child->nsproxy->cgroup_ns->root_cset = cset;  in cgroup_post_fork()
6651  * cgroup_exit - detach cgroup from exiting task
6665 	WARN_ON_ONCE(list_empty(&tsk->cg_list));  in cgroup_exit()
6668 	list_add_tail(&tsk->cg_list, &cset->dying_tasks);  in cgroup_exit()
6669 	cset->nr_tasks--;  in cgroup_exit()
6672 	if (unlikely(!(tsk->flags & PF_KTHREAD) &&  in cgroup_exit()
6673 		     test_bit(CGRP_FREEZE, &task_dfl_cgroup(tsk)->flags)))  in cgroup_exit()
6680 		ss->exit(tsk);  in cgroup_exit()
6690 		ss->release(task);  in cgroup_release()
6695 	list_del_init(&task->cg_list);  in cgroup_release()
6716 			if (strcmp(token, ss->name) &&  in cgroup_disable()
6717 			    strcmp(token, ss->legacy_name))  in cgroup_disable()
6722 				ss->name);  in cgroup_disable()
6749  * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
6761 	struct file_system_type *s_type = dentry->d_sb->s_type;  in css_tryget_online_from_dir()
6768 		return ERR_PTR(-EBADF);  in css_tryget_online_from_dir()
6774 	 * have been or be removed at any point.  @kn->priv is RCU  in css_tryget_online_from_dir()
6777 	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);  in css_tryget_online_from_dir()
6782 		css = ERR_PTR(-ENOENT);  in css_tryget_online_from_dir()
6789  * css_from_id - lookup css by id
6799 	return idr_find(&ss->css_idr, id);  in css_from_id()
6803  * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
6808  * success, ERR_PTR(-ENOENT) if @path doesn't exist or if the cgroup has already
6809  * been released and ERR_PTR(-ENOTDIR) if @path points to a non-directory.
6814 	struct cgroup *cgrp = ERR_PTR(-ENOENT);  in cgroup_get_from_path()
6818 	kn = kernfs_walk_and_get(root_cgrp->kn, path);  in cgroup_get_from_path()
6823 		cgrp = ERR_PTR(-ENOTDIR);  in cgroup_get_from_path()
6829 	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);  in cgroup_get_from_path()
6831 		cgrp = ERR_PTR(-ENOENT);  in cgroup_get_from_path()
6843  * cgroup_v1v2_get_from_fd - get a cgroup pointer from a fd
6858 		return ERR_PTR(-EBADF);  in cgroup_v1v2_get_from_fd()
6866  * cgroup_get_from_fd - same as cgroup_v1v2_get_from_fd, but only supports
6879 		return ERR_PTR(-EBADF);  in cgroup_get_from_fd()
6888 	while (power--)  in power_of_ten()
6894  * cgroup_parse_float - parse a floating number
6902  * Returns 0 on success, -errno otherwise.
6913 		return -EINVAL;  in cgroup_parse_float()
6915 		return -EINVAL;  in cgroup_parse_float()
6917 	flen = fend > fstart ? fend - fstart : 0;  in cgroup_parse_float()
6919 		frac *= power_of_ten(dec_shift - flen);  in cgroup_parse_float()
6921 		frac = DIV_ROUND_CLOSEST_ULL(frac, power_of_ten(flen - dec_shift));  in cgroup_parse_float()
6928  * sock->sk_cgrp_data handling.  For more info, see sock_cgroup_data
6929  * definition in cgroup-defs.h.
6949 		if (likely(cgroup_tryget(cset->dfl_cgrp))) {  in cgroup_sk_alloc()
6950 			cgroup = cset->dfl_cgrp;  in cgroup_sk_alloc()
6956 	skcd->cgroup = cgroup;  in cgroup_sk_alloc()
6991 	for (cft = files; cft && cft->name[0] != '\0'; cft++) {  in show_delegatable_files()
6992 		if (!(cft->flags & CFTYPE_NS_DELEGATABLE))  in show_delegatable_files()
6996 			ret += snprintf(buf + ret, size - ret, "%s.", prefix);  in show_delegatable_files()
6998 		ret += snprintf(buf + ret, size - ret, "%s\n", cft->name);  in show_delegatable_files()
7015 				     PAGE_SIZE - ret, NULL);  in delegate_show()
7018 					      PAGE_SIZE - ret, NULL);  in delegate_show()
7021 		ret += show_delegatable_files(ss->dfl_cftypes, buf + ret,  in delegate_show()
7022 					      PAGE_SIZE - ret,  in delegate_show()