610 		count += link->cset->nr_tasks;  in __cgroup_task_count()
760 static bool css_set_threaded(struct css_set *cset)  in css_set_threaded()  argument
762 	return cset->dom_cset != cset;  in css_set_threaded()
774 static bool css_set_populated(struct css_set *cset)  in css_set_populated()  argument
778 	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);  in css_set_populated()
838 static void css_set_update_populated(struct css_set *cset, bool populated)  in css_set_update_populated()  argument
844 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)  in css_set_update_populated()
854 static void css_set_skip_task_iters(struct css_set *cset,  in css_set_skip_task_iters()  argument
859 	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)  in css_set_skip_task_iters()
934 void put_css_set_locked(struct css_set *cset)  in put_css_set_locked()  argument
942 	if (!refcount_dec_and_test(&cset->refcount))  in put_css_set_locked()
945 	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));  in put_css_set_locked()
949 		list_del(&cset->e_cset_node[ssid]);  in put_css_set_locked()
950 		css_put(cset->subsys[ssid]);  in put_css_set_locked()
952 	hash_del(&cset->hlist);  in put_css_set_locked()
955 	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {  in put_css_set_locked()
963 	if (css_set_threaded(cset)) {  in put_css_set_locked()
964 		list_del(&cset->threaded_csets_node);  in put_css_set_locked()
965 		put_css_set_locked(cset->dom_cset);  in put_css_set_locked()
968 	kfree_rcu(cset, rcu_head);  in put_css_set_locked()
981 static bool compare_css_sets(struct css_set *cset,  in compare_css_sets()  argument
994 	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))  in compare_css_sets()
1004 	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)  in compare_css_sets()
1013 	l1 = &cset->cgrp_links;  in compare_css_sets()
1022 		if (l1 == &cset->cgrp_links) {  in compare_css_sets()
1066 	struct css_set *cset;  in find_existing_css_set()  local
1092 	hash_for_each_possible(css_set_table, cset, hlist, key) {  in find_existing_css_set()
1093 		if (!compare_css_sets(cset, old_cset, cgrp, template))  in find_existing_css_set()
1097 		return cset;  in find_existing_css_set()
1146 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,  in link_css_set()  argument
1154 		cset->dfl_cgrp = cgrp;  in link_css_set()
1157 	link->cset = cset;  in link_css_set()
1165 	list_add_tail(&link->cgrp_link, &cset->cgrp_links);  in link_css_set()
1183 	struct css_set *cset;  in find_css_set()  local
1195 	cset = find_existing_css_set(old_cset, cgrp, template);  in find_css_set()
1196 	if (cset)  in find_css_set()
1197 		get_css_set(cset);  in find_css_set()
1200 	if (cset)  in find_css_set()
1201 		return cset;  in find_css_set()
1203 	cset = kzalloc(sizeof(*cset), GFP_KERNEL);  in find_css_set()
1204 	if (!cset)  in find_css_set()
1209 		kfree(cset);  in find_css_set()
1213 	refcount_set(&cset->refcount, 1);  in find_css_set()
1214 	cset->dom_cset = cset;  in find_css_set()
1215 	INIT_LIST_HEAD(&cset->tasks);  in find_css_set()
1216 	INIT_LIST_HEAD(&cset->mg_tasks);  in find_css_set()
1217 	INIT_LIST_HEAD(&cset->dying_tasks);  in find_css_set()
1218 	INIT_LIST_HEAD(&cset->task_iters);  in find_css_set()
1219 	INIT_LIST_HEAD(&cset->threaded_csets);  in find_css_set()
1220 	INIT_HLIST_NODE(&cset->hlist);  in find_css_set()
1221 	INIT_LIST_HEAD(&cset->cgrp_links);  in find_css_set()
1222 	INIT_LIST_HEAD(&cset->mg_preload_node);  in find_css_set()
1223 	INIT_LIST_HEAD(&cset->mg_node);  in find_css_set()
1227 	memcpy(cset->subsys, template, sizeof(cset->subsys));  in find_css_set()
1236 		link_css_set(&tmp_links, cset, c);  in find_css_set()
1244 	key = css_set_hash(cset->subsys);  in find_css_set()
1245 	hash_add(css_set_table, &cset->hlist, key);  in find_css_set()
1248 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()
1250 		list_add_tail(&cset->e_cset_node[ssid],  in find_css_set()
1263 	if (cgroup_is_threaded(cset->dfl_cgrp)) {  in find_css_set()
1266 		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);  in find_css_set()
1268 			put_css_set(cset);  in find_css_set()
1273 		cset->dom_cset = dcset;  in find_css_set()
1274 		list_add_tail(&cset->threaded_csets_node,  in find_css_set()
1279 	return cset;  in find_css_set()
1368 	struct css_set *cset;  in current_cgns_cgroup_from_root()  local
1374 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_from_root()
1375 	if (cset == &init_css_set) {  in current_cgns_cgroup_from_root()
1380 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in current_cgns_cgroup_from_root()
1396 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,  in cset_cgroup_from_root()  argument
1404 	if (cset == &init_css_set) {  in cset_cgroup_from_root()
1407 		res = cset->dfl_cgrp;  in cset_cgroup_from_root()
1411 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in cset_cgroup_from_root()
1746 		struct css_set *cset;  in rebind_subsystems()  local
1762 		hash_for_each(css_set_table, i, cset, hlist)  in rebind_subsystems()
1763 			list_move_tail(&cset->e_cset_node[ss->id],  in rebind_subsystems()
1930 			struct css_set *cset = task_css_set(p);  in cgroup_enable_task_cg_lists()  local
1932 			if (!css_set_populated(cset))  in cgroup_enable_task_cg_lists()
1933 				css_set_update_populated(cset, true);  in cgroup_enable_task_cg_lists()
1934 			list_add_tail(&p->cg_list, &cset->tasks);  in cgroup_enable_task_cg_lists()
1935 			get_css_set(cset);  in cgroup_enable_task_cg_lists()
1936 			cset->nr_tasks++;  in cgroup_enable_task_cg_lists()
1995 	struct css_set *cset;  in cgroup_setup_root()  local
2065 	hash_for_each(css_set_table, i, cset, hlist) {  in cgroup_setup_root()
2066 		link_css_set(&tmp_links, cset, root_cgrp);  in cgroup_setup_root()
2067 		if (css_set_populated(cset))  in cgroup_setup_root()
2367 	struct css_set *cset;  in cgroup_migrate_add_task()  local
2379 	cset = task_css_set(task);  in cgroup_migrate_add_task()
2380 	if (!cset->mg_src_cgrp)  in cgroup_migrate_add_task()
2385 	list_move_tail(&task->cg_list, &cset->mg_tasks);  in cgroup_migrate_add_task()
2386 	if (list_empty(&cset->mg_node))  in cgroup_migrate_add_task()
2387 		list_add_tail(&cset->mg_node,  in cgroup_migrate_add_task()
2389 	if (list_empty(&cset->mg_dst_cset->mg_node))  in cgroup_migrate_add_task()
2390 		list_add_tail(&cset->mg_dst_cset->mg_node,  in cgroup_migrate_add_task()
2421 	struct css_set *cset = tset->cur_cset;  in cgroup_taskset_next()  local
2424 	while (&cset->mg_node != tset->csets) {  in cgroup_taskset_next()
2426 			task = list_first_entry(&cset->mg_tasks,  in cgroup_taskset_next()
2431 		if (&task->cg_list != &cset->mg_tasks) {  in cgroup_taskset_next()
2432 			tset->cur_cset = cset;  in cgroup_taskset_next()
2441 			if (cset->mg_dst_cset)  in cgroup_taskset_next()
2442 				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];  in cgroup_taskset_next()
2444 				*dst_cssp = cset->subsys[tset->ssid];  in cgroup_taskset_next()
2449 		cset = list_next_entry(cset, mg_node);  in cgroup_taskset_next()
2470 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_execute()  local
2493 	list_for_each_entry(cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2494 		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {  in cgroup_migrate_execute()
2496 			struct css_set *to_cset = cset->mg_dst_cset;  in cgroup_migrate_execute()
2547 	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2548 		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);  in cgroup_migrate_execute()
2549 		list_del_init(&cset->mg_node);  in cgroup_migrate_execute()
2610 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_finish()  local
2619 	list_for_each_entry_safe(cset, tmp_cset, &preloaded, mg_preload_node) {  in cgroup_migrate_finish()
2620 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2621 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2622 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2623 		list_del_init(&cset->mg_preload_node);  in cgroup_migrate_finish()
2624 		put_css_set_locked(cset);  in cgroup_migrate_finish()
2950 			cgroup_migrate_add_src(link->cset, dsct, &mgctx);  in cgroup_update_dfl_csses()
4394 	struct css_set *cset;  in css_task_iter_next_css_set()  local
4420 		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);  in css_task_iter_next_css_set()
4423 		cset = link->cset;  in css_task_iter_next_css_set()
4432 		it->cur_dcset = cset;  in css_task_iter_next_css_set()
4433 		get_css_set(cset);  in css_task_iter_next_css_set()
4435 		it->tcset_head = &cset->threaded_csets;  in css_task_iter_next_css_set()
4436 		it->tcset_pos = &cset->threaded_csets;  in css_task_iter_next_css_set()
4439 	return cset;  in css_task_iter_next_css_set()
4450 	struct css_set *cset;  in css_task_iter_advance_css_set()  local
4456 		cset = css_task_iter_next_css_set(it);  in css_task_iter_advance_css_set()
4457 		if (!cset) {  in css_task_iter_advance_css_set()
4461 	} while (!css_set_populated(cset) && list_empty(&cset->dying_tasks));  in css_task_iter_advance_css_set()
4463 	if (!list_empty(&cset->tasks))  in css_task_iter_advance_css_set()
4464 		it->task_pos = cset->tasks.next;  in css_task_iter_advance_css_set()
4465 	else if (!list_empty(&cset->mg_tasks))  in css_task_iter_advance_css_set()
4466 		it->task_pos = cset->mg_tasks.next;  in css_task_iter_advance_css_set()
4468 		it->task_pos = cset->dying_tasks.next;  in css_task_iter_advance_css_set()
4470 	it->tasks_head = &cset->tasks;  in css_task_iter_advance_css_set()
4471 	it->mg_tasks_head = &cset->mg_tasks;  in css_task_iter_advance_css_set()
4472 	it->dying_tasks_head = &cset->dying_tasks;  in css_task_iter_advance_css_set()
4493 	get_css_set(cset);  in css_task_iter_advance_css_set()
4494 	it->cur_cset = cset;  in css_task_iter_advance_css_set()
4495 	list_add(&it->iters_node, &cset->task_iters);  in css_task_iter_advance_css_set()
5557 		link->cset->dead = true;  in cgroup_destroy_locked()
6029 		struct css_set *cset;  in cgroup_post_fork()  local
6032 		cset = task_css_set(current);  in cgroup_post_fork()
6034 			get_css_set(cset);  in cgroup_post_fork()
6035 			cset->nr_tasks++;  in cgroup_post_fork()
6036 			css_set_move_task(child, NULL, cset, false);  in cgroup_post_fork()
6093 	struct css_set *cset;  in cgroup_exit()  local
6100 	cset = task_css_set(tsk);  in cgroup_exit()
6104 		css_set_move_task(tsk, cset, NULL, false);  in cgroup_exit()
6105 		list_add_tail(&tsk->cg_list, &cset->dying_tasks);  in cgroup_exit()
6106 		cset->nr_tasks--;  in cgroup_exit()
6114 		get_css_set(cset);  in cgroup_exit()
6142 	struct css_set *cset = task_css_set(task);  in cgroup_free()  local
6143 	put_css_set(cset);  in cgroup_free()
6386 		struct css_set *cset;  in cgroup_sk_alloc()  local
6388 		cset = task_css_set(current);  in cgroup_sk_alloc()
6389 		if (likely(cgroup_tryget(cset->dfl_cgrp))) {  in cgroup_sk_alloc()
6390 			skcd->val = (unsigned long)cset->dfl_cgrp;  in cgroup_sk_alloc()
6391 			cgroup_bpf_get(cset->dfl_cgrp);  in cgroup_sk_alloc()