220 static void kill_css(struct cgroup_subsys_state *css);
221 static int cgroup_addrm_files(struct cgroup_subsys_state *css,
480 	struct cgroup_subsys_state *css;  in cgroup_tryget_css()  local
483 	css = cgroup_css(cgrp, ss);  in cgroup_tryget_css()
484 	if (css && !css_tryget_online(css))  in cgroup_tryget_css()
485 		css = NULL;  in cgroup_tryget_css()
488 	return css;  in cgroup_tryget_css()
538 	struct cgroup_subsys_state *css;  in cgroup_e_css()  local
541 		css = cgroup_css(cgrp, ss);  in cgroup_e_css()
543 		if (css)  in cgroup_e_css()
544 			return css;  in cgroup_e_css()
565 	struct cgroup_subsys_state *css;  in cgroup_get_e_css()  local
570 		css = cgroup_css(cgrp, ss);  in cgroup_get_e_css()
572 		if (css && css_tryget_online(css))  in cgroup_get_e_css()
577 	css = init_css_set.subsys[ss->id];  in cgroup_get_e_css()
578 	css_get(css);  in cgroup_get_e_css()
581 	return css;  in cgroup_get_e_css()
651 #define for_each_css(css, ssid, cgrp)					\  argument
653 		if (!((css) = rcu_dereference_check(			\
666 #define for_each_e_css(css, ssid, cgrp)					    \  argument
668 		if (!((css) = cgroup_e_css_by_mask(cgrp,		    \
913 static unsigned long css_set_hash(struct cgroup_subsys_state *css[])  in css_set_hash()  argument
920 		key += (unsigned long)css[i];  in css_set_hash()
1240 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()  local
1243 			      &css->cgroup->e_csets[ssid]);  in find_css_set()
1244 		css_get(css);  in find_css_set()
1623 		struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);  in cgroup_rm_file()  local
1624 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_rm_file()
1640 static void css_clear_dir(struct cgroup_subsys_state *css)  in css_clear_dir()  argument
1642 	struct cgroup *cgrp = css->cgroup;  in css_clear_dir()
1645 	if (!(css->flags & CSS_VISIBLE))  in css_clear_dir()
1648 	css->flags &= ~CSS_VISIBLE;  in css_clear_dir()
1650 	if (!css->ss) {  in css_clear_dir()
1656 		cgroup_addrm_files(css, cgrp, cfts, false);  in css_clear_dir()
1658 		list_for_each_entry(cfts, &css->ss->cfts, node)  in css_clear_dir()
1659 			cgroup_addrm_files(css, cgrp, cfts, false);  in css_clear_dir()
1669 static int css_populate_dir(struct cgroup_subsys_state *css)  in css_populate_dir()  argument
1671 	struct cgroup *cgrp = css->cgroup;  in css_populate_dir()
1675 	if ((css->flags & CSS_VISIBLE) || !cgrp->kn)  in css_populate_dir()
1678 	if (!css->ss) {  in css_populate_dir()
1688 		list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1689 			ret = cgroup_addrm_files(css, cgrp, cfts, true);  in css_populate_dir()
1697 	css->flags |= CSS_VISIBLE;  in css_populate_dir()
1701 	list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1704 		cgroup_addrm_files(css, cgrp, cfts, false);  in css_populate_dir()
1735 		struct cgroup_subsys_state *css = cgroup_css(scgrp, ss);  in rebind_subsystems()  local
1738 		WARN_ON(!css || cgroup_css(dcgrp, ss));  in rebind_subsystems()
1747 		rcu_assign_pointer(dcgrp->subsys[ssid], css);  in rebind_subsystems()
1749 		css->cgroup = dcgrp;  in rebind_subsystems()
1772 			ss->bind(css);  in rebind_subsystems()
2937 			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);  in cgroup_lock_and_drain_offline()  local
2940 			if (!css || !percpu_ref_is_dying(&css->refcnt))  in cgroup_lock_and_drain_offline()
3018 static bool css_visible(struct cgroup_subsys_state *css)  in css_visible()  argument
3020 	struct cgroup_subsys *ss = css->ss;  in css_visible()
3021 	struct cgroup *cgrp = css->cgroup;  in css_visible()
3052 			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);  in cgroup_apply_control_enable()  local
3057 			if (!css) {  in cgroup_apply_control_enable()
3058 				css = css_create(dsct, ss);  in cgroup_apply_control_enable()
3059 				if (IS_ERR(css))  in cgroup_apply_control_enable()
3060 					return PTR_ERR(css);  in cgroup_apply_control_enable()
3063 			WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_enable()
3065 			if (css_visible(css)) {  in cgroup_apply_control_enable()
3066 				ret = css_populate_dir(css);  in cgroup_apply_control_enable()
3098 			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);  in cgroup_apply_control_disable()  local
3100 			if (!css)  in cgroup_apply_control_disable()
3103 			WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_disable()
3105 			if (css->parent &&  in cgroup_apply_control_disable()
3107 				kill_css(css);  in cgroup_apply_control_disable()
3108 			} else if (!css_visible(css)) {  in cgroup_apply_control_disable()
3109 				css_clear_dir(css);  in cgroup_apply_control_disable()
3111 					ss->css_reset(css);  in cgroup_apply_control_disable()
3515 	struct cgroup_subsys_state *css;  in cgroup_extra_stat_show()  local
3521 	css = cgroup_tryget_css(cgrp, ss);  in cgroup_extra_stat_show()
3522 	if (!css)  in cgroup_extra_stat_show()
3525 	ret = ss->css_extra_stat_show(seq, css);  in cgroup_extra_stat_show()
3526 	css_put(css);  in cgroup_extra_stat_show()
3681 	struct cgroup_subsys_state *css;  in cgroup_file_write()  local
3708 	css = cgroup_css(cgrp, cft->ss);  in cgroup_file_write()
3715 			ret = cft->write_u64(css, cft, v);  in cgroup_file_write()
3720 			ret = cft->write_s64(css, cft, v);  in cgroup_file_write()
3757 	struct cgroup_subsys_state *css = seq_css(m);  in cgroup_seqfile_show()  local
3763 		seq_printf(m, "%llu\n", cft->read_u64(css, cft));  in cgroup_seqfile_show()
3765 		seq_printf(m, "%lld\n", cft->read_s64(css, cft));  in cgroup_seqfile_show()
3812 static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,  in cgroup_add_file()  argument
3838 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_add_file()
3860 static int cgroup_addrm_files(struct cgroup_subsys_state *css,  in cgroup_addrm_files()  argument
3883 			ret = cgroup_add_file(css, cgrp, cft);  in cgroup_addrm_files()
3902 	struct cgroup_subsys_state *css;  in cgroup_apply_cftypes()  local
3908 	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {  in cgroup_apply_cftypes()
3909 		struct cgroup *cgrp = css->cgroup;  in cgroup_apply_cftypes()
3911 		if (!(css->flags & CSS_VISIBLE))  in cgroup_apply_cftypes()
3914 		ret = cgroup_addrm_files(css, cgrp, cfts, is_add);  in cgroup_apply_cftypes()
4319 bool css_has_online_children(struct cgroup_subsys_state *css)  in css_has_online_children()  argument
4325 	css_for_each_child(child, css) {  in css_has_online_children()
4517 void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,  in css_task_iter_start()  argument
4524 	it->ss = css->ss;  in css_task_iter_start()
4528 		it->cset_pos = &css->cgroup->e_csets[css->ss->id];  in css_task_iter_start()
4530 		it->cset_pos = &css->cgroup->cset_links;  in css_task_iter_start()
4931 	struct cgroup_subsys_state *css = container_of(to_rcu_work(work),  in css_free_rwork_fn()  local
4933 	struct cgroup_subsys *ss = css->ss;  in css_free_rwork_fn()
4934 	struct cgroup *cgrp = css->cgroup;  in css_free_rwork_fn()
4936 	percpu_ref_exit(&css->refcnt);  in css_free_rwork_fn()
4940 		struct cgroup_subsys_state *parent = css->parent;  in css_free_rwork_fn()
4941 		int id = css->id;  in css_free_rwork_fn()
4943 		ss->css_free(css);  in css_free_rwork_fn()
4981 	struct cgroup_subsys_state *css =  in css_release_work_fn()  local
4983 	struct cgroup_subsys *ss = css->ss;  in css_release_work_fn()
4984 	struct cgroup *cgrp = css->cgroup;  in css_release_work_fn()
4988 	css->flags |= CSS_RELEASED;  in css_release_work_fn()
4989 	list_del_rcu(&css->sibling);  in css_release_work_fn()
4993 		if (!list_empty(&css->rstat_css_node)) {  in css_release_work_fn()
4995 			list_del_rcu(&css->rstat_css_node);  in css_release_work_fn()
4998 		cgroup_idr_replace(&ss->css_idr, NULL, css->id);  in css_release_work_fn()
5000 			ss->css_released(css);  in css_release_work_fn()
5030 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_release_work_fn()
5031 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_release_work_fn()
5036 	struct cgroup_subsys_state *css =  in css_release()  local
5039 	INIT_WORK(&css->destroy_work, css_release_work_fn);  in css_release()
5040 	queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_release()
5043 static void init_and_link_css(struct cgroup_subsys_state *css,  in init_and_link_css()  argument
5050 	memset(css, 0, sizeof(*css));  in init_and_link_css()
5051 	css->cgroup = cgrp;  in init_and_link_css()
5052 	css->ss = ss;  in init_and_link_css()
5053 	css->id = -1;  in init_and_link_css()
5054 	INIT_LIST_HEAD(&css->sibling);  in init_and_link_css()
5055 	INIT_LIST_HEAD(&css->children);  in init_and_link_css()
5056 	INIT_LIST_HEAD(&css->rstat_css_node);  in init_and_link_css()
5057 	css->serial_nr = css_serial_nr_next++;  in init_and_link_css()
5058 	atomic_set(&css->online_cnt, 0);  in init_and_link_css()
5061 		css->parent = cgroup_css(cgroup_parent(cgrp), ss);  in init_and_link_css()
5062 		css_get(css->parent);  in init_and_link_css()
5066 		list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list);  in init_and_link_css()
5072 static int online_css(struct cgroup_subsys_state *css)  in online_css()  argument
5074 	struct cgroup_subsys *ss = css->ss;  in online_css()
5080 		ret = ss->css_online(css);  in online_css()
5082 		css->flags |= CSS_ONLINE;  in online_css()
5083 		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);  in online_css()
5085 		atomic_inc(&css->online_cnt);  in online_css()
5086 		if (css->parent)  in online_css()
5087 			atomic_inc(&css->parent->online_cnt);  in online_css()
5093 static void offline_css(struct cgroup_subsys_state *css)  in offline_css()  argument
5095 	struct cgroup_subsys *ss = css->ss;  in offline_css()
5099 	if (!(css->flags & CSS_ONLINE))  in offline_css()
5103 		ss->css_offline(css);  in offline_css()
5105 	css->flags &= ~CSS_ONLINE;  in offline_css()
5106 	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);  in offline_css()
5108 	wake_up_all(&css->cgroup->offline_waitq);  in offline_css()
5125 	struct cgroup_subsys_state *css;  in css_create()  local
5130 	css = ss->css_alloc(parent_css);  in css_create()
5131 	if (!css)  in css_create()
5132 		css = ERR_PTR(-ENOMEM);  in css_create()
5133 	if (IS_ERR(css))  in css_create()
5134 		return css;  in css_create()
5136 	init_and_link_css(css, ss, cgrp);  in css_create()
5138 	err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);  in css_create()
5145 	css->id = err;  in css_create()
5148 	list_add_tail_rcu(&css->sibling, &parent_css->children);  in css_create()
5149 	cgroup_idr_replace(&ss->css_idr, css, css->id);  in css_create()
5151 	err = online_css(css);  in css_create()
5164 	return css;  in css_create()
5167 	list_del_rcu(&css->sibling);  in css_create()
5169 	list_del_rcu(&css->rstat_css_node);  in css_create()
5170 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_create()
5171 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_create()
5386 	struct cgroup_subsys_state *css =  in css_killed_work_fn()  local
5392 		offline_css(css);  in css_killed_work_fn()
5393 		css_put(css);  in css_killed_work_fn()
5395 		css = css->parent;  in css_killed_work_fn()
5396 	} while (css && atomic_dec_and_test(&css->online_cnt));  in css_killed_work_fn()
5404 	struct cgroup_subsys_state *css =  in css_killed_ref_fn()  local
5407 	if (atomic_dec_and_test(&css->online_cnt)) {  in css_killed_ref_fn()
5408 		INIT_WORK(&css->destroy_work, css_killed_work_fn);  in css_killed_ref_fn()
5409 		queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_killed_ref_fn()
5422 static void kill_css(struct cgroup_subsys_state *css)  in kill_css()  argument
5426 	if (css->flags & CSS_DYING)  in kill_css()
5429 	css->flags |= CSS_DYING;  in kill_css()
5435 	css_clear_dir(css);  in kill_css()
5441 	css_get(css);  in kill_css()
5453 	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);  in kill_css()
5484 	struct cgroup_subsys_state *css;  in cgroup_destroy_locked()  local
5519 	for_each_css(css, ssid, cgrp)  in cgroup_destroy_locked()
5520 		kill_css(css);  in cgroup_destroy_locked()
5578 	struct cgroup_subsys_state *css;  in cgroup_init_subsys()  local
5589 	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));  in cgroup_init_subsys()
5591 	BUG_ON(IS_ERR(css));  in cgroup_init_subsys()
5592 	init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);  in cgroup_init_subsys()
5598 	css->flags |= CSS_NO_REF;  in cgroup_init_subsys()
5602 		css->id = 1;  in cgroup_init_subsys()
5604 		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);  in cgroup_init_subsys()
5605 		BUG_ON(css->id < 0);  in cgroup_init_subsys()
5612 	init_css_set.subsys[ss->id] = css;  in cgroup_init_subsys()
5624 	BUG_ON(online_css(css));  in cgroup_init_subsys()
5708 			struct cgroup_subsys_state *css =  in cgroup_init()  local
5711 			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,  in cgroup_init()
5713 			BUG_ON(css->id < 0);  in cgroup_init()
5901 	struct cgroup_subsys_state *css;  in cgroup_get_from_file()  local
5904 	css = css_tryget_online_from_dir(f->f_path.dentry, NULL);  in cgroup_get_from_file()
5905 	if (IS_ERR(css))  in cgroup_get_from_file()
5906 		return ERR_CAST(css);  in cgroup_get_from_file()
5908 	cgrp = css->cgroup;  in cgroup_get_from_file()
6277 	struct cgroup_subsys_state *css = NULL;  in css_tryget_online_from_dir()  local
6294 		css = cgroup_css(cgrp, ss);  in css_tryget_online_from_dir()
6296 	if (!css || !css_tryget_online(css))  in css_tryget_online_from_dir()
6297 		css = ERR_PTR(-ENOENT);  in css_tryget_online_from_dir()
6300 	return css;  in css_tryget_online_from_dir()