631 		count += link->cset->nr_tasks;  in __cgroup_task_count()
781 static bool css_set_threaded(struct css_set *cset)  in css_set_threaded()  argument
783 	return cset->dom_cset != cset;  in css_set_threaded()
795 static bool css_set_populated(struct css_set *cset)  in css_set_populated()  argument
799 	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);  in css_set_populated()
859 static void css_set_update_populated(struct css_set *cset, bool populated)  in css_set_update_populated()  argument
865 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)  in css_set_update_populated()
875 static void css_set_skip_task_iters(struct css_set *cset,  in css_set_skip_task_iters()  argument
880 	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)  in css_set_skip_task_iters()
954 void put_css_set_locked(struct css_set *cset)  in put_css_set_locked()  argument
962 	if (!refcount_dec_and_test(&cset->refcount))  in put_css_set_locked()
965 	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));  in put_css_set_locked()
969 		list_del(&cset->e_cset_node[ssid]);  in put_css_set_locked()
970 		css_put(cset->subsys[ssid]);  in put_css_set_locked()
972 	hash_del(&cset->hlist);  in put_css_set_locked()
975 	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {  in put_css_set_locked()
983 	if (css_set_threaded(cset)) {  in put_css_set_locked()
984 		list_del(&cset->threaded_csets_node);  in put_css_set_locked()
985 		put_css_set_locked(cset->dom_cset);  in put_css_set_locked()
988 	kfree_rcu(cset, rcu_head);  in put_css_set_locked()
1001 static bool compare_css_sets(struct css_set *cset,  in compare_css_sets()  argument
1014 	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))  in compare_css_sets()
1024 	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)  in compare_css_sets()
1033 	l1 = &cset->cgrp_links;  in compare_css_sets()
1042 		if (l1 == &cset->cgrp_links) {  in compare_css_sets()
1086 	struct css_set *cset;  in find_existing_css_set()  local
1112 	hash_for_each_possible(css_set_table, cset, hlist, key) {  in find_existing_css_set()
1113 		if (!compare_css_sets(cset, old_cset, cgrp, template))  in find_existing_css_set()
1117 		return cset;  in find_existing_css_set()
1166 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,  in link_css_set()  argument
1174 		cset->dfl_cgrp = cgrp;  in link_css_set()
1177 	link->cset = cset;  in link_css_set()
1185 	list_add_tail(&link->cgrp_link, &cset->cgrp_links);  in link_css_set()
1203 	struct css_set *cset;  in find_css_set()  local
1215 	cset = find_existing_css_set(old_cset, cgrp, template);  in find_css_set()
1216 	if (cset)  in find_css_set()
1217 		get_css_set(cset);  in find_css_set()
1220 	if (cset)  in find_css_set()
1221 		return cset;  in find_css_set()
1223 	cset = kzalloc(sizeof(*cset), GFP_KERNEL);  in find_css_set()
1224 	if (!cset)  in find_css_set()
1229 		kfree(cset);  in find_css_set()
1233 	refcount_set(&cset->refcount, 1);  in find_css_set()
1234 	cset->dom_cset = cset;  in find_css_set()
1235 	INIT_LIST_HEAD(&cset->tasks);  in find_css_set()
1236 	INIT_LIST_HEAD(&cset->mg_tasks);  in find_css_set()
1237 	INIT_LIST_HEAD(&cset->dying_tasks);  in find_css_set()
1238 	INIT_LIST_HEAD(&cset->task_iters);  in find_css_set()
1239 	INIT_LIST_HEAD(&cset->threaded_csets);  in find_css_set()
1240 	INIT_HLIST_NODE(&cset->hlist);  in find_css_set()
1241 	INIT_LIST_HEAD(&cset->cgrp_links);  in find_css_set()
1242 	INIT_LIST_HEAD(&cset->mg_preload_node);  in find_css_set()
1243 	INIT_LIST_HEAD(&cset->mg_node);  in find_css_set()
1247 	memcpy(cset->subsys, template, sizeof(cset->subsys));  in find_css_set()
1256 		link_css_set(&tmp_links, cset, c);  in find_css_set()
1264 	key = css_set_hash(cset->subsys);  in find_css_set()
1265 	hash_add(css_set_table, &cset->hlist, key);  in find_css_set()
1268 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()
1270 		list_add_tail(&cset->e_cset_node[ssid],  in find_css_set()
1283 	if (cgroup_is_threaded(cset->dfl_cgrp)) {  in find_css_set()
1286 		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);  in find_css_set()
1288 			put_css_set(cset);  in find_css_set()
1293 		cset->dom_cset = dcset;  in find_css_set()
1294 		list_add_tail(&cset->threaded_csets_node,  in find_css_set()
1299 	return cset;  in find_css_set()
1386 	struct css_set *cset;  in current_cgns_cgroup_from_root()  local
1392 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_from_root()
1393 	if (cset == &init_css_set) {  in current_cgns_cgroup_from_root()
1396 		res = cset->dfl_cgrp;  in current_cgns_cgroup_from_root()
1400 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in current_cgns_cgroup_from_root()
1416 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,  in cset_cgroup_from_root()  argument
1424 	if (cset == &init_css_set) {  in cset_cgroup_from_root()
1427 		res = cset->dfl_cgrp;  in cset_cgroup_from_root()
1431 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in cset_cgroup_from_root()
1765 		struct css_set *cset;  in rebind_subsystems()  local
1781 		hash_for_each(css_set_table, i, cset, hlist)  in rebind_subsystems()
1782 			list_move_tail(&cset->e_cset_node[ss->id],  in rebind_subsystems()
1967 	struct css_set *cset;  in cgroup_setup_root()  local
2038 	hash_for_each(css_set_table, i, cset, hlist) {  in cgroup_setup_root()
2039 		link_css_set(&tmp_links, cset, root_cgrp);  in cgroup_setup_root()
2040 		if (css_set_populated(cset))  in cgroup_setup_root()
2335 	struct css_set *cset;  in cgroup_migrate_add_task()  local
2346 	cset = task_css_set(task);  in cgroup_migrate_add_task()
2347 	if (!cset->mg_src_cgrp)  in cgroup_migrate_add_task()
2352 	list_move_tail(&task->cg_list, &cset->mg_tasks);  in cgroup_migrate_add_task()
2353 	if (list_empty(&cset->mg_node))  in cgroup_migrate_add_task()
2354 		list_add_tail(&cset->mg_node,  in cgroup_migrate_add_task()
2356 	if (list_empty(&cset->mg_dst_cset->mg_node))  in cgroup_migrate_add_task()
2357 		list_add_tail(&cset->mg_dst_cset->mg_node,  in cgroup_migrate_add_task()
2388 	struct css_set *cset = tset->cur_cset;  in cgroup_taskset_next()  local
2391 	while (CGROUP_HAS_SUBSYS_CONFIG && &cset->mg_node != tset->csets) {  in cgroup_taskset_next()
2393 			task = list_first_entry(&cset->mg_tasks,  in cgroup_taskset_next()
2398 		if (&task->cg_list != &cset->mg_tasks) {  in cgroup_taskset_next()
2399 			tset->cur_cset = cset;  in cgroup_taskset_next()
2408 			if (cset->mg_dst_cset)  in cgroup_taskset_next()
2409 				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];  in cgroup_taskset_next()
2411 				*dst_cssp = cset->subsys[tset->ssid];  in cgroup_taskset_next()
2416 		cset = list_next_entry(cset, mg_node);  in cgroup_taskset_next()
2437 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_execute()  local
2460 	list_for_each_entry(cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2461 		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {  in cgroup_migrate_execute()
2463 			struct css_set *to_cset = cset->mg_dst_cset;  in cgroup_migrate_execute()
2514 	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2515 		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);  in cgroup_migrate_execute()
2516 		list_del_init(&cset->mg_node);  in cgroup_migrate_execute()
2577 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_finish()  local
2586 	list_for_each_entry_safe(cset, tmp_cset, &preloaded, mg_preload_node) {  in cgroup_migrate_finish()
2587 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2588 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2589 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2590 		list_del_init(&cset->mg_preload_node);  in cgroup_migrate_finish()
2591 		put_css_set_locked(cset);  in cgroup_migrate_finish()
2932 			cgroup_migrate_add_src(link->cset, dsct, &mgctx);  in cgroup_update_dfl_csses()
4474 	struct css_set *cset;  in css_task_iter_next_css_set()  local
4500 		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);  in css_task_iter_next_css_set()
4503 		cset = link->cset;  in css_task_iter_next_css_set()
4512 		it->cur_dcset = cset;  in css_task_iter_next_css_set()
4513 		get_css_set(cset);  in css_task_iter_next_css_set()
4515 		it->tcset_head = &cset->threaded_csets;  in css_task_iter_next_css_set()
4516 		it->tcset_pos = &cset->threaded_csets;  in css_task_iter_next_css_set()
4519 	return cset;  in css_task_iter_next_css_set()
4530 	struct css_set *cset;  in css_task_iter_advance_css_set()  local
4535 	while ((cset = css_task_iter_next_css_set(it))) {  in css_task_iter_advance_css_set()
4536 		if (!list_empty(&cset->tasks)) {  in css_task_iter_advance_css_set()
4537 			it->cur_tasks_head = &cset->tasks;  in css_task_iter_advance_css_set()
4539 		} else if (!list_empty(&cset->mg_tasks)) {  in css_task_iter_advance_css_set()
4540 			it->cur_tasks_head = &cset->mg_tasks;  in css_task_iter_advance_css_set()
4542 		} else if (!list_empty(&cset->dying_tasks)) {  in css_task_iter_advance_css_set()
4543 			it->cur_tasks_head = &cset->dying_tasks;  in css_task_iter_advance_css_set()
4547 	if (!cset) {  in css_task_iter_advance_css_set()
4572 	get_css_set(cset);  in css_task_iter_advance_css_set()
4573 	it->cur_cset = cset;  in css_task_iter_advance_css_set()
4574 	list_add(&it->iters_node, &cset->task_iters);  in css_task_iter_advance_css_set()
5617 		link->cset->dead = true;  in cgroup_destroy_locked()
6059 	struct css_set *cset;  in cgroup_css_set_fork()  local
6069 	cset = task_css_set(current);  in cgroup_css_set_fork()
6070 	get_css_set(cset);  in cgroup_css_set_fork()
6074 		kargs->cset = cset;  in cgroup_css_set_fork()
6106 	ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb,  in cgroup_css_set_fork()
6111 	kargs->cset = find_css_set(cset, dst_cgrp);  in cgroup_css_set_fork()
6112 	if (!kargs->cset) {  in cgroup_css_set_fork()
6117 	put_css_set(cset);  in cgroup_css_set_fork()
6129 	put_css_set(cset);  in cgroup_css_set_fork()
6130 	if (kargs->cset)  in cgroup_css_set_fork()
6131 		put_css_set(kargs->cset);  in cgroup_css_set_fork()
6149 		struct css_set *cset = kargs->cset;  in cgroup_css_set_put_fork()  local
6153 		if (cset) {  in cgroup_css_set_put_fork()
6154 			put_css_set(cset);  in cgroup_css_set_put_fork()
6155 			kargs->cset = NULL;  in cgroup_css_set_put_fork()
6185 		ret = ss->can_fork(child, kargs->cset);  in cgroup_can_fork()
6197 			ss->cancel_fork(child, kargs->cset);  in cgroup_can_fork()
6222 			ss->cancel_fork(child, kargs->cset);  in cgroup_cancel_fork()
6241 	struct css_set *cset;  in cgroup_post_fork()  local
6244 	cset = kargs->cset;  in cgroup_post_fork()
6245 	kargs->cset = NULL;  in cgroup_post_fork()
6254 			cgrp_flags = cset->dfl_cgrp->flags;  in cgroup_post_fork()
6257 		cset->nr_tasks++;  in cgroup_post_fork()
6258 		css_set_move_task(child, NULL, cset, false);  in cgroup_post_fork()
6260 		put_css_set(cset);  in cgroup_post_fork()
6261 		cset = NULL;  in cgroup_post_fork()
6307 		get_css_set(cset);  in cgroup_post_fork()
6308 		child->nsproxy->cgroup_ns->root_cset = cset;  in cgroup_post_fork()
6329 	struct css_set *cset;  in cgroup_exit()  local
6335 	cset = task_css_set(tsk);  in cgroup_exit()
6336 	css_set_move_task(tsk, cset, NULL, false);  in cgroup_exit()
6337 	list_add_tail(&tsk->cg_list, &cset->dying_tasks);  in cgroup_exit()
6338 	cset->nr_tasks--;  in cgroup_exit()
6370 	struct css_set *cset = task_css_set(task);  in cgroup_free()  local
6371 	put_css_set(cset);  in cgroup_free()
6590 		struct css_set *cset;  in cgroup_sk_alloc()  local
6592 		cset = task_css_set(current);  in cgroup_sk_alloc()
6593 		if (likely(cgroup_tryget(cset->dfl_cgrp))) {  in cgroup_sk_alloc()
6594 			cgroup = cset->dfl_cgrp;  in cgroup_sk_alloc()