220 static void kill_css(struct cgroup_subsys_state *css);
221 static int cgroup_addrm_files(struct cgroup_subsys_state *css,
483 	struct cgroup_subsys_state *css;  in cgroup_tryget_css()  local
486 	css = cgroup_css(cgrp, ss);  in cgroup_tryget_css()
487 	if (!css || !css_tryget_online(css))  in cgroup_tryget_css()
488 		css = NULL;  in cgroup_tryget_css()
491 	return css;  in cgroup_tryget_css()
539 	struct cgroup_subsys_state *css;  in cgroup_get_e_css()  local
544 		css = cgroup_css(cgrp, ss);  in cgroup_get_e_css()
546 		if (css && css_tryget_online(css))  in cgroup_get_e_css()
551 	css = init_css_set.subsys[ss->id];  in cgroup_get_e_css()
552 	css_get(css);  in cgroup_get_e_css()
555 	return css;  in cgroup_get_e_css()
592 #define for_each_css(css, ssid, cgrp)					\  argument
594 		if (!((css) = rcu_dereference_check(			\
607 #define for_each_e_css(css, ssid, cgrp)					\  argument
609 		if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
849 static unsigned long css_set_hash(struct cgroup_subsys_state *css[])  in css_set_hash()  argument
856 		key += (unsigned long)css[i];  in css_set_hash()
1175 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()  local
1178 			      &css->cgroup->e_csets[ssid]);  in find_css_set()
1179 		css_get(css);  in find_css_set()
1557 		struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);  in cgroup_rm_file()  local
1558 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_rm_file()
1574 static void css_clear_dir(struct cgroup_subsys_state *css)  in css_clear_dir()  argument
1576 	struct cgroup *cgrp = css->cgroup;  in css_clear_dir()
1579 	if (!(css->flags & CSS_VISIBLE))  in css_clear_dir()
1582 	css->flags &= ~CSS_VISIBLE;  in css_clear_dir()
1584 	if (!css->ss) {  in css_clear_dir()
1590 		cgroup_addrm_files(css, cgrp, cfts, false);  in css_clear_dir()
1592 		list_for_each_entry(cfts, &css->ss->cfts, node)  in css_clear_dir()
1593 			cgroup_addrm_files(css, cgrp, cfts, false);  in css_clear_dir()
1603 static int css_populate_dir(struct cgroup_subsys_state *css)  in css_populate_dir()  argument
1605 	struct cgroup *cgrp = css->cgroup;  in css_populate_dir()
1609 	if ((css->flags & CSS_VISIBLE) || !cgrp->kn)  in css_populate_dir()
1612 	if (!css->ss) {  in css_populate_dir()
1622 		list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1623 			ret = cgroup_addrm_files(css, cgrp, cfts, true);  in css_populate_dir()
1631 	css->flags |= CSS_VISIBLE;  in css_populate_dir()
1635 	list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1638 		cgroup_addrm_files(css, cgrp, cfts, false);  in css_populate_dir()
1669 		struct cgroup_subsys_state *css = cgroup_css(scgrp, ss);  in rebind_subsystems()  local
1672 		WARN_ON(!css || cgroup_css(dcgrp, ss));  in rebind_subsystems()
1681 		rcu_assign_pointer(dcgrp->subsys[ssid], css);  in rebind_subsystems()
1683 		css->cgroup = dcgrp;  in rebind_subsystems()
1706 			ss->bind(css);  in rebind_subsystems()
2818 			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);  in cgroup_lock_and_drain_offline()  local
2821 			if (!css || !percpu_ref_is_dying(&css->refcnt))  in cgroup_lock_and_drain_offline()
2899 static bool css_visible(struct cgroup_subsys_state *css)  in css_visible()  argument
2901 	struct cgroup_subsys *ss = css->ss;  in css_visible()
2902 	struct cgroup *cgrp = css->cgroup;  in css_visible()
2933 			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);  in cgroup_apply_control_enable()  local
2935 			WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_enable()
2940 			if (!css) {  in cgroup_apply_control_enable()
2941 				css = css_create(dsct, ss);  in cgroup_apply_control_enable()
2942 				if (IS_ERR(css))  in cgroup_apply_control_enable()
2943 					return PTR_ERR(css);  in cgroup_apply_control_enable()
2946 			if (css_visible(css)) {  in cgroup_apply_control_enable()
2947 				ret = css_populate_dir(css);  in cgroup_apply_control_enable()
2979 			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);  in cgroup_apply_control_disable()  local
2981 			WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_disable()
2983 			if (!css)  in cgroup_apply_control_disable()
2986 			if (css->parent &&  in cgroup_apply_control_disable()
2988 				kill_css(css);  in cgroup_apply_control_disable()
2989 			} else if (!css_visible(css)) {  in cgroup_apply_control_disable()
2990 				css_clear_dir(css);  in cgroup_apply_control_disable()
2992 					ss->css_reset(css);  in cgroup_apply_control_disable()
3392 	struct cgroup_subsys_state *css;  in cgroup_extra_stat_show()  local
3398 	css = cgroup_tryget_css(cgrp, ss);  in cgroup_extra_stat_show()
3399 	if (!css)  in cgroup_extra_stat_show()
3402 	ret = ss->css_extra_stat_show(seq, css);  in cgroup_extra_stat_show()
3403 	css_put(css);  in cgroup_extra_stat_show()
3442 	struct cgroup_subsys_state *css;  in cgroup_file_write()  local
3466 	css = cgroup_css(cgrp, cft->ss);  in cgroup_file_write()
3473 			ret = cft->write_u64(css, cft, v);  in cgroup_file_write()
3478 			ret = cft->write_s64(css, cft, v);  in cgroup_file_write()
3505 	struct cgroup_subsys_state *css = seq_css(m);  in cgroup_seqfile_show()  local
3511 		seq_printf(m, "%llu\n", cft->read_u64(css, cft));  in cgroup_seqfile_show()
3513 		seq_printf(m, "%lld\n", cft->read_s64(css, cft));  in cgroup_seqfile_show()
3558 static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,  in cgroup_add_file()  argument
3584 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_add_file()
3606 static int cgroup_addrm_files(struct cgroup_subsys_state *css,  in cgroup_addrm_files()  argument
3628 			ret = cgroup_add_file(css, cgrp, cft);  in cgroup_addrm_files()
3647 	struct cgroup_subsys_state *css;  in cgroup_apply_cftypes()  local
3653 	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {  in cgroup_apply_cftypes()
3654 		struct cgroup *cgrp = css->cgroup;  in cgroup_apply_cftypes()
3656 		if (!(css->flags & CSS_VISIBLE))  in cgroup_apply_cftypes()
3659 		ret = cgroup_addrm_files(css, cgrp, cfts, is_add);  in cgroup_apply_cftypes()
4062 bool css_has_online_children(struct cgroup_subsys_state *css)  in css_has_online_children()  argument
4068 	css_for_each_child(child, css) {  in css_has_online_children()
4222 void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,  in css_task_iter_start()  argument
4232 	it->ss = css->ss;  in css_task_iter_start()
4236 		it->cset_pos = &css->cgroup->e_csets[css->ss->id];  in css_task_iter_start()
4238 		it->cset_pos = &css->cgroup->cset_links;  in css_task_iter_start()
4576 	struct cgroup_subsys_state *css = container_of(to_rcu_work(work),  in css_free_rwork_fn()  local
4578 	struct cgroup_subsys *ss = css->ss;  in css_free_rwork_fn()
4579 	struct cgroup *cgrp = css->cgroup;  in css_free_rwork_fn()
4581 	percpu_ref_exit(&css->refcnt);  in css_free_rwork_fn()
4585 		struct cgroup_subsys_state *parent = css->parent;  in css_free_rwork_fn()
4586 		int id = css->id;  in css_free_rwork_fn()
4588 		ss->css_free(css);  in css_free_rwork_fn()
4625 	struct cgroup_subsys_state *css =  in css_release_work_fn()  local
4627 	struct cgroup_subsys *ss = css->ss;  in css_release_work_fn()
4628 	struct cgroup *cgrp = css->cgroup;  in css_release_work_fn()
4632 	css->flags |= CSS_RELEASED;  in css_release_work_fn()
4633 	list_del_rcu(&css->sibling);  in css_release_work_fn()
4637 		if (!list_empty(&css->rstat_css_node)) {  in css_release_work_fn()
4639 			list_del_rcu(&css->rstat_css_node);  in css_release_work_fn()
4642 		cgroup_idr_replace(&ss->css_idr, NULL, css->id);  in css_release_work_fn()
4644 			ss->css_released(css);  in css_release_work_fn()
4677 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_release_work_fn()
4678 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_release_work_fn()
4683 	struct cgroup_subsys_state *css =  in css_release()  local
4686 	INIT_WORK(&css->destroy_work, css_release_work_fn);  in css_release()
4687 	queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_release()
4690 static void init_and_link_css(struct cgroup_subsys_state *css,  in init_and_link_css()  argument
4697 	memset(css, 0, sizeof(*css));  in init_and_link_css()
4698 	css->cgroup = cgrp;  in init_and_link_css()
4699 	css->ss = ss;  in init_and_link_css()
4700 	css->id = -1;  in init_and_link_css()
4701 	INIT_LIST_HEAD(&css->sibling);  in init_and_link_css()
4702 	INIT_LIST_HEAD(&css->children);  in init_and_link_css()
4703 	INIT_LIST_HEAD(&css->rstat_css_node);  in init_and_link_css()
4704 	css->serial_nr = css_serial_nr_next++;  in init_and_link_css()
4705 	atomic_set(&css->online_cnt, 0);  in init_and_link_css()
4708 		css->parent = cgroup_css(cgroup_parent(cgrp), ss);  in init_and_link_css()
4709 		css_get(css->parent);  in init_and_link_css()
4713 		list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list);  in init_and_link_css()
4719 static int online_css(struct cgroup_subsys_state *css)  in online_css()  argument
4721 	struct cgroup_subsys *ss = css->ss;  in online_css()
4727 		ret = ss->css_online(css);  in online_css()
4729 		css->flags |= CSS_ONLINE;  in online_css()
4730 		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);  in online_css()
4732 		atomic_inc(&css->online_cnt);  in online_css()
4733 		if (css->parent)  in online_css()
4734 			atomic_inc(&css->parent->online_cnt);  in online_css()
4740 static void offline_css(struct cgroup_subsys_state *css)  in offline_css()  argument
4742 	struct cgroup_subsys *ss = css->ss;  in offline_css()
4746 	if (!(css->flags & CSS_ONLINE))  in offline_css()
4750 		ss->css_offline(css);  in offline_css()
4752 	css->flags &= ~CSS_ONLINE;  in offline_css()
4753 	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);  in offline_css()
4755 	wake_up_all(&css->cgroup->offline_waitq);  in offline_css()
4772 	struct cgroup_subsys_state *css;  in css_create()  local
4777 	css = ss->css_alloc(parent_css);  in css_create()
4778 	if (!css)  in css_create()
4779 		css = ERR_PTR(-ENOMEM);  in css_create()
4780 	if (IS_ERR(css))  in css_create()
4781 		return css;  in css_create()
4783 	init_and_link_css(css, ss, cgrp);  in css_create()
4785 	err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);  in css_create()
4792 	css->id = err;  in css_create()
4795 	list_add_tail_rcu(&css->sibling, &parent_css->children);  in css_create()
4796 	cgroup_idr_replace(&ss->css_idr, css, css->id);  in css_create()
4798 	err = online_css(css);  in css_create()
4811 	return css;  in css_create()
4814 	list_del_rcu(&css->sibling);  in css_create()
4816 	list_del_rcu(&css->rstat_css_node);  in css_create()
4817 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_create()
4818 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_create()
5014 	struct cgroup_subsys_state *css =  in css_killed_work_fn()  local
5020 		offline_css(css);  in css_killed_work_fn()
5021 		css_put(css);  in css_killed_work_fn()
5023 		css = css->parent;  in css_killed_work_fn()
5024 	} while (css && atomic_dec_and_test(&css->online_cnt));  in css_killed_work_fn()
5032 	struct cgroup_subsys_state *css =  in css_killed_ref_fn()  local
5035 	if (atomic_dec_and_test(&css->online_cnt)) {  in css_killed_ref_fn()
5036 		INIT_WORK(&css->destroy_work, css_killed_work_fn);  in css_killed_ref_fn()
5037 		queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_killed_ref_fn()
5050 static void kill_css(struct cgroup_subsys_state *css)  in kill_css()  argument
5054 	if (css->flags & CSS_DYING)  in kill_css()
5057 	css->flags |= CSS_DYING;  in kill_css()
5063 	css_clear_dir(css);  in kill_css()
5069 	css_get(css);  in kill_css()
5081 	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);  in kill_css()
5112 	struct cgroup_subsys_state *css;  in cgroup_destroy_locked()  local
5147 	for_each_css(css, ssid, cgrp)  in cgroup_destroy_locked()
5148 		kill_css(css);  in cgroup_destroy_locked()
5197 	struct cgroup_subsys_state *css;  in cgroup_init_subsys()  local
5208 	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));  in cgroup_init_subsys()
5210 	BUG_ON(IS_ERR(css));  in cgroup_init_subsys()
5211 	init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);  in cgroup_init_subsys()
5217 	css->flags |= CSS_NO_REF;  in cgroup_init_subsys()
5221 		css->id = 1;  in cgroup_init_subsys()
5223 		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);  in cgroup_init_subsys()
5224 		BUG_ON(css->id < 0);  in cgroup_init_subsys()
5231 	init_css_set.subsys[ss->id] = css;  in cgroup_init_subsys()
5243 	BUG_ON(online_css(css));  in cgroup_init_subsys()
5327 			struct cgroup_subsys_state *css =  in cgroup_init()  local
5330 			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,  in cgroup_init()
5332 			BUG_ON(css->id < 0);  in cgroup_init()
5719 	struct cgroup_subsys_state *css = NULL;  in css_tryget_online_from_dir()  local
5736 		css = cgroup_css(cgrp, ss);  in css_tryget_online_from_dir()
5738 	if (!css || !css_tryget_online(css))  in css_tryget_online_from_dir()
5739 		css = ERR_PTR(-ENOENT);  in css_tryget_online_from_dir()
5742 	return css;  in css_tryget_online_from_dir()
5804 	struct cgroup_subsys_state *css;  in cgroup_get_from_fd()  local
5812 	css = css_tryget_online_from_dir(f->f_path.dentry, NULL);  in cgroup_get_from_fd()
5814 	if (IS_ERR(css))  in cgroup_get_from_fd()
5815 		return ERR_CAST(css);  in cgroup_get_from_fd()
5817 	cgrp = css->cgroup;  in cgroup_get_from_fd()