603 		count += link->cset->nr_tasks;  in __cgroup_task_count()
753 static bool css_set_threaded(struct css_set *cset)  in css_set_threaded()  argument
755 	return cset->dom_cset != cset;  in css_set_threaded()
767 static bool css_set_populated(struct css_set *cset)  in css_set_populated()  argument
771 	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);  in css_set_populated()
831 static void css_set_update_populated(struct css_set *cset, bool populated)  in css_set_update_populated()  argument
837 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)  in css_set_update_populated()
847 static void css_set_skip_task_iters(struct css_set *cset,  in css_set_skip_task_iters()  argument
852 	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)  in css_set_skip_task_iters()
926 void put_css_set_locked(struct css_set *cset)  in put_css_set_locked()  argument
934 	if (!refcount_dec_and_test(&cset->refcount))  in put_css_set_locked()
937 	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));  in put_css_set_locked()
941 		list_del(&cset->e_cset_node[ssid]);  in put_css_set_locked()
942 		css_put(cset->subsys[ssid]);  in put_css_set_locked()
944 	hash_del(&cset->hlist);  in put_css_set_locked()
947 	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {  in put_css_set_locked()
955 	if (css_set_threaded(cset)) {  in put_css_set_locked()
956 		list_del(&cset->threaded_csets_node);  in put_css_set_locked()
957 		put_css_set_locked(cset->dom_cset);  in put_css_set_locked()
960 	kfree_rcu(cset, rcu_head);  in put_css_set_locked()
973 static bool compare_css_sets(struct css_set *cset,  in compare_css_sets()  argument
986 	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))  in compare_css_sets()
996 	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)  in compare_css_sets()
1005 	l1 = &cset->cgrp_links;  in compare_css_sets()
1014 		if (l1 == &cset->cgrp_links) {  in compare_css_sets()
1058 	struct css_set *cset;  in find_existing_css_set()  local
1084 	hash_for_each_possible(css_set_table, cset, hlist, key) {  in find_existing_css_set()
1085 		if (!compare_css_sets(cset, old_cset, cgrp, template))  in find_existing_css_set()
1089 		return cset;  in find_existing_css_set()
1138 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,  in link_css_set()  argument
1146 		cset->dfl_cgrp = cgrp;  in link_css_set()
1149 	link->cset = cset;  in link_css_set()
1157 	list_add_tail(&link->cgrp_link, &cset->cgrp_links);  in link_css_set()
1175 	struct css_set *cset;  in find_css_set()  local
1187 	cset = find_existing_css_set(old_cset, cgrp, template);  in find_css_set()
1188 	if (cset)  in find_css_set()
1189 		get_css_set(cset);  in find_css_set()
1192 	if (cset)  in find_css_set()
1193 		return cset;  in find_css_set()
1195 	cset = kzalloc(sizeof(*cset), GFP_KERNEL);  in find_css_set()
1196 	if (!cset)  in find_css_set()
1201 		kfree(cset);  in find_css_set()
1205 	refcount_set(&cset->refcount, 1);  in find_css_set()
1206 	cset->dom_cset = cset;  in find_css_set()
1207 	INIT_LIST_HEAD(&cset->tasks);  in find_css_set()
1208 	INIT_LIST_HEAD(&cset->mg_tasks);  in find_css_set()
1209 	INIT_LIST_HEAD(&cset->dying_tasks);  in find_css_set()
1210 	INIT_LIST_HEAD(&cset->task_iters);  in find_css_set()
1211 	INIT_LIST_HEAD(&cset->threaded_csets);  in find_css_set()
1212 	INIT_HLIST_NODE(&cset->hlist);  in find_css_set()
1213 	INIT_LIST_HEAD(&cset->cgrp_links);  in find_css_set()
1214 	INIT_LIST_HEAD(&cset->mg_preload_node);  in find_css_set()
1215 	INIT_LIST_HEAD(&cset->mg_node);  in find_css_set()
1219 	memcpy(cset->subsys, template, sizeof(cset->subsys));  in find_css_set()
1228 		link_css_set(&tmp_links, cset, c);  in find_css_set()
1236 	key = css_set_hash(cset->subsys);  in find_css_set()
1237 	hash_add(css_set_table, &cset->hlist, key);  in find_css_set()
1240 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()
1242 		list_add_tail(&cset->e_cset_node[ssid],  in find_css_set()
1255 	if (cgroup_is_threaded(cset->dfl_cgrp)) {  in find_css_set()
1258 		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);  in find_css_set()
1260 			put_css_set(cset);  in find_css_set()
1265 		cset->dom_cset = dcset;  in find_css_set()
1266 		list_add_tail(&cset->threaded_csets_node,  in find_css_set()
1271 	return cset;  in find_css_set()
1357 	struct css_set *cset;  in current_cgns_cgroup_from_root()  local
1363 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_from_root()
1364 	if (cset == &init_css_set) {  in current_cgns_cgroup_from_root()
1367 		res = cset->dfl_cgrp;  in current_cgns_cgroup_from_root()
1371 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in current_cgns_cgroup_from_root()
1387 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,  in cset_cgroup_from_root()  argument
1395 	if (cset == &init_css_set) {  in cset_cgroup_from_root()
1398 		res = cset->dfl_cgrp;  in cset_cgroup_from_root()
1402 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in cset_cgroup_from_root()
1736 		struct css_set *cset;  in rebind_subsystems()  local
1752 		hash_for_each(css_set_table, i, cset, hlist)  in rebind_subsystems()
1753 			list_move_tail(&cset->e_cset_node[ss->id],  in rebind_subsystems()
1932 	struct css_set *cset;  in cgroup_setup_root()  local
1999 	hash_for_each(css_set_table, i, cset, hlist) {  in cgroup_setup_root()
2000 		link_css_set(&tmp_links, cset, root_cgrp);  in cgroup_setup_root()
2001 		if (css_set_populated(cset))  in cgroup_setup_root()
2293 	struct css_set *cset;  in cgroup_migrate_add_task()  local
2304 	cset = task_css_set(task);  in cgroup_migrate_add_task()
2305 	if (!cset->mg_src_cgrp)  in cgroup_migrate_add_task()
2310 	list_move_tail(&task->cg_list, &cset->mg_tasks);  in cgroup_migrate_add_task()
2311 	if (list_empty(&cset->mg_node))  in cgroup_migrate_add_task()
2312 		list_add_tail(&cset->mg_node,  in cgroup_migrate_add_task()
2314 	if (list_empty(&cset->mg_dst_cset->mg_node))  in cgroup_migrate_add_task()
2315 		list_add_tail(&cset->mg_dst_cset->mg_node,  in cgroup_migrate_add_task()
2346 	struct css_set *cset = tset->cur_cset;  in cgroup_taskset_next()  local
2349 	while (&cset->mg_node != tset->csets) {  in cgroup_taskset_next()
2351 			task = list_first_entry(&cset->mg_tasks,  in cgroup_taskset_next()
2356 		if (&task->cg_list != &cset->mg_tasks) {  in cgroup_taskset_next()
2357 			tset->cur_cset = cset;  in cgroup_taskset_next()
2366 			if (cset->mg_dst_cset)  in cgroup_taskset_next()
2367 				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];  in cgroup_taskset_next()
2369 				*dst_cssp = cset->subsys[tset->ssid];  in cgroup_taskset_next()
2374 		cset = list_next_entry(cset, mg_node);  in cgroup_taskset_next()
2395 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_execute()  local
2418 	list_for_each_entry(cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2419 		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {  in cgroup_migrate_execute()
2421 			struct css_set *to_cset = cset->mg_dst_cset;  in cgroup_migrate_execute()
2472 	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2473 		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);  in cgroup_migrate_execute()
2474 		list_del_init(&cset->mg_node);  in cgroup_migrate_execute()
2535 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_finish()  local
2544 	list_for_each_entry_safe(cset, tmp_cset, &preloaded, mg_preload_node) {  in cgroup_migrate_finish()
2545 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2546 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2547 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2548 		list_del_init(&cset->mg_preload_node);  in cgroup_migrate_finish()
2549 		put_css_set_locked(cset);  in cgroup_migrate_finish()
2890 			cgroup_migrate_add_src(link->cset, dsct, &mgctx);  in cgroup_update_dfl_csses()
4339 	struct css_set *cset;  in css_task_iter_next_css_set()  local
4365 		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);  in css_task_iter_next_css_set()
4368 		cset = link->cset;  in css_task_iter_next_css_set()
4377 		it->cur_dcset = cset;  in css_task_iter_next_css_set()
4378 		get_css_set(cset);  in css_task_iter_next_css_set()
4380 		it->tcset_head = &cset->threaded_csets;  in css_task_iter_next_css_set()
4381 		it->tcset_pos = &cset->threaded_csets;  in css_task_iter_next_css_set()
4384 	return cset;  in css_task_iter_next_css_set()
4395 	struct css_set *cset;  in css_task_iter_advance_css_set()  local
4400 	while ((cset = css_task_iter_next_css_set(it))) {  in css_task_iter_advance_css_set()
4401 		if (!list_empty(&cset->tasks)) {  in css_task_iter_advance_css_set()
4402 			it->cur_tasks_head = &cset->tasks;  in css_task_iter_advance_css_set()
4404 		} else if (!list_empty(&cset->mg_tasks)) {  in css_task_iter_advance_css_set()
4405 			it->cur_tasks_head = &cset->mg_tasks;  in css_task_iter_advance_css_set()
4407 		} else if (!list_empty(&cset->dying_tasks)) {  in css_task_iter_advance_css_set()
4408 			it->cur_tasks_head = &cset->dying_tasks;  in css_task_iter_advance_css_set()
4412 	if (!cset) {  in css_task_iter_advance_css_set()
4437 	get_css_set(cset);  in css_task_iter_advance_css_set()
4438 	it->cur_cset = cset;  in css_task_iter_advance_css_set()
4439 	list_add(&it->iters_node, &cset->task_iters);  in css_task_iter_advance_css_set()
5515 		link->cset->dead = true;  in cgroup_destroy_locked()
5938 	struct css_set *cset;  in cgroup_css_set_fork()  local
5948 	cset = task_css_set(current);  in cgroup_css_set_fork()
5949 	get_css_set(cset);  in cgroup_css_set_fork()
5953 		kargs->cset = cset;  in cgroup_css_set_fork()
5985 	ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb,  in cgroup_css_set_fork()
5990 	kargs->cset = find_css_set(cset, dst_cgrp);  in cgroup_css_set_fork()
5991 	if (!kargs->cset) {  in cgroup_css_set_fork()
5996 	put_css_set(cset);  in cgroup_css_set_fork()
6008 	put_css_set(cset);  in cgroup_css_set_fork()
6009 	if (kargs->cset)  in cgroup_css_set_fork()
6010 		put_css_set(kargs->cset);  in cgroup_css_set_fork()
6028 		struct css_set *cset = kargs->cset;  in cgroup_css_set_put_fork()  local
6032 		if (cset) {  in cgroup_css_set_put_fork()
6033 			put_css_set(cset);  in cgroup_css_set_put_fork()
6034 			kargs->cset = NULL;  in cgroup_css_set_put_fork()
6064 		ret = ss->can_fork(child, kargs->cset);  in cgroup_can_fork()
6076 			ss->cancel_fork(child, kargs->cset);  in cgroup_can_fork()
6101 			ss->cancel_fork(child, kargs->cset);  in cgroup_cancel_fork()
6118 	struct css_set *cset;  in cgroup_post_fork()  local
6121 	cset = kargs->cset;  in cgroup_post_fork()
6122 	kargs->cset = NULL;  in cgroup_post_fork()
6129 		cset->nr_tasks++;  in cgroup_post_fork()
6130 		css_set_move_task(child, NULL, cset, false);  in cgroup_post_fork()
6132 		put_css_set(cset);  in cgroup_post_fork()
6133 		cset = NULL;  in cgroup_post_fork()
6170 		get_css_set(cset);  in cgroup_post_fork()
6171 		child->nsproxy->cgroup_ns->root_cset = cset;  in cgroup_post_fork()
6188 	struct css_set *cset;  in cgroup_exit()  local
6194 	cset = task_css_set(tsk);  in cgroup_exit()
6195 	css_set_move_task(tsk, cset, NULL, false);  in cgroup_exit()
6196 	list_add_tail(&tsk->cg_list, &cset->dying_tasks);  in cgroup_exit()
6197 	cset->nr_tasks--;  in cgroup_exit()
6228 	struct css_set *cset = task_css_set(task);  in cgroup_free()  local
6229 	put_css_set(cset);  in cgroup_free()
6456 		struct css_set *cset;  in cgroup_sk_alloc()  local
6458 		cset = task_css_set(current);  in cgroup_sk_alloc()
6459 		if (likely(cgroup_tryget(cset->dfl_cgrp))) {  in cgroup_sk_alloc()
6460 			skcd->val = (unsigned long)cset->dfl_cgrp;  in cgroup_sk_alloc()
6461 			cgroup_bpf_get(cset->dfl_cgrp);  in cgroup_sk_alloc()