Lines Matching +full:non +full:- +full:empty
7 * Copyright (C) 2004-2007 Silicon Graphics, Inc.
11 * sysfs is Copyright (c) 2001-3 Patrick Mochel
13 * 2003-10-10 Written by Simon Derr.
14 * 2003-10-22 Updates by Stephen Hemminger.
15 * 2004 May-July Rework by Paul Jackson.
59 #include <linux/backing-dev.h>
89 * The user-configured masks can only be changed by writing to
98 * and if it ends up empty, it will inherit the parent's mask.
103 * The user-configured masks are always the same with effective masks.
106 /* user-configured CPUs and Memory Nodes allow to tasks */
115 * CPUs allocated to child sub-partitions (default hierarchy only)
116 * - CPUs granted by the parent = effective_cpus U subparts_cpus
117 * - effective_cpus and subparts_cpus are mutually exclusive.
127 * - top_cpuset.old_mems_allowed is initialized to mems_allowed.
128 * - A new cpuset's old_mems_allowed is initialized when some
130 * - old_mems_allowed is used in cpuset_migrate_mm() when we change
140 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
158 * use_parent_ecpus - set if using parent's effective_cpus
159 * child_ecpus_count - # of children with use_parent_ecpus set
171 * 0 - not a partition root
173 * 1 - partition root
175 * -1 - invalid partition root
184 #define PRS_ERROR -1
208 return css_cs(cs->css.parent); in parent_cs()
226 return test_bit(CS_ONLINE, &cs->flags) && !css_is_dying(&cs->css); in is_cpuset_online()
231 return test_bit(CS_CPU_EXCLUSIVE, &cs->flags); in is_cpu_exclusive()
236 return test_bit(CS_MEM_EXCLUSIVE, &cs->flags); in is_mem_exclusive()
241 return test_bit(CS_MEM_HARDWALL, &cs->flags); in is_mem_hardwall()
246 return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in is_sched_load_balance()
251 return test_bit(CS_MEMORY_MIGRATE, &cs->flags); in is_memory_migrate()
256 return test_bit(CS_SPREAD_PAGE, &cs->flags); in is_spread_page()
261 return test_bit(CS_SPREAD_SLAB, &cs->flags); in is_spread_slab()
266 return cs->partition_root_state > 0; in is_partition_root()
276 cgroup_file_notify(&cs->partition_file); in notify_partition_change()
286 * cpuset_for_each_child - traverse online children of a cpuset
295 css_for_each_child((pos_css), &(parent_cs)->css) \
299 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
310 css_for_each_descendant_pre((pos_css), &(root_cs)->css) \
314 * There are two global locks guarding cpuset structures - cpuset_rwsem and
336 * If a task is only holding callback_lock, then it has read-only
344 * small pieces of code, such as when reading out possibly multi-word
386 (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); in is_in_v2_mode()
395 * One way or another, we guarantee to return some non-empty subset
412 while (!cpumask_intersects(cs->effective_cpus, pmask)) { in guarantee_online_cpus()
425 cpumask_and(pmask, pmask, cs->effective_cpus); in guarantee_online_cpus()
437 * One way or another, we guarantee to return some non-empty subset
444 while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) in guarantee_online_mems()
446 nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); in guarantee_online_mems()
469 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
478 return cpumask_subset(p->cpus_allowed, q->cpus_allowed) && in is_cpuset_subset()
479 nodes_subset(p->mems_allowed, q->mems_allowed) && in is_cpuset_subset()
485 * alloc_cpumasks - allocate three cpumasks for cpuset
488 * Return: 0 if successful, -ENOMEM otherwise.
490 * Only one of the two input arguments should be non-NULL.
497 pmask1 = &cs->cpus_allowed; in alloc_cpumasks()
498 pmask2 = &cs->effective_cpus; in alloc_cpumasks()
499 pmask3 = &cs->subparts_cpus; in alloc_cpumasks()
501 pmask1 = &tmp->new_cpus; in alloc_cpumasks()
502 pmask2 = &tmp->addmask; in alloc_cpumasks()
503 pmask3 = &tmp->delmask; in alloc_cpumasks()
507 return -ENOMEM; in alloc_cpumasks()
521 return -ENOMEM; in alloc_cpumasks()
525 * free_cpumasks - free cpumasks in a tmpmasks structure
532 free_cpumask_var(cs->cpus_allowed); in free_cpumasks()
533 free_cpumask_var(cs->effective_cpus); in free_cpumasks()
534 free_cpumask_var(cs->subparts_cpus); in free_cpumasks()
537 free_cpumask_var(tmp->new_cpus); in free_cpumasks()
538 free_cpumask_var(tmp->addmask); in free_cpumasks()
539 free_cpumask_var(tmp->delmask); in free_cpumasks()
544 * alloc_trial_cpuset - allocate a trial cpuset
560 cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); in alloc_trial_cpuset()
561 cpumask_copy(trial->effective_cpus, cs->effective_cpus); in alloc_trial_cpuset()
566 * free_cpuset - free the cpuset
576 * validate_change() - Used to validate that any proposed cpuset change
584 * 'cur' is the address of an actual, in-use cpuset. Operations
592 * Return 0 if valid, -errno if not.
604 ret = -EBUSY; in validate_change()
617 ret = -EACCES; in validate_change()
625 ret = -EINVAL; in validate_change()
629 cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) in validate_change()
633 nodes_intersects(trial->mems_allowed, c->mems_allowed)) in validate_change()
638 * Cpusets with tasks - existing or newly being attached - can't in validate_change()
639 * be changed to have empty cpus_allowed or mems_allowed. in validate_change()
641 ret = -ENOSPC; in validate_change()
642 if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) { in validate_change()
643 if (!cpumask_empty(cur->cpus_allowed) && in validate_change()
644 cpumask_empty(trial->cpus_allowed)) in validate_change()
646 if (!nodes_empty(cur->mems_allowed) && in validate_change()
647 nodes_empty(trial->mems_allowed)) in validate_change()
655 ret = -EBUSY; in validate_change()
657 !cpuset_cpumask_can_shrink(cur->cpus_allowed, in validate_change()
658 trial->cpus_allowed)) in validate_change()
674 return cpumask_intersects(a->effective_cpus, b->effective_cpus); in cpusets_overlap()
680 if (dattr->relax_domain_level < c->relax_domain_level) in update_domain_attr()
681 dattr->relax_domain_level = c->relax_domain_level; in update_domain_attr()
694 if (cpumask_empty(cp->cpus_allowed)) { in update_domain_attr_tree()
708 /* jump label reference count + the top-level cpuset */ in nr_cpusets()
716 * A 'partial partition' is a set of non-overlapping subsets whose
723 * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
734 * cp - cpuset pointer, used (together with pos_css) to perform a
735 * top-down scan of all cpusets. For our purposes, rebuilding
738 * csa - (for CpuSet Array) Array of pointers to all the cpusets
745 * doms - Conversion of 'csa' to an array of cpumasks, for passing to
768 struct cpuset *cp; /* top-down scan of cpusets */ in generate_sched_domains()
775 int nslot; /* next empty doms[] struct cpumask slot */ in generate_sched_domains()
820 * If root is load-balancing, we can skip @cp if it in generate_sched_domains()
823 if (!cpumask_empty(cp->cpus_allowed) && in generate_sched_domains()
825 cpumask_intersects(cp->cpus_allowed, in generate_sched_domains()
830 cpumask_subset(cp->cpus_allowed, top_cpuset.effective_cpus)) in generate_sched_domains()
834 !cpumask_empty(cp->effective_cpus)) in generate_sched_domains()
844 csa[i]->pn = i; in generate_sched_domains()
851 int apn = a->pn; in generate_sched_domains()
855 int bpn = b->pn; in generate_sched_domains()
861 if (c->pn == bpn) in generate_sched_domains()
862 c->pn = apn; in generate_sched_domains()
864 ndoms--; /* one less element */ in generate_sched_domains()
888 int apn = a->pn; in generate_sched_domains()
902 warnings--; in generate_sched_domains()
913 if (apn == b->pn) { in generate_sched_domains()
914 cpumask_or(dp, dp, b->effective_cpus); in generate_sched_domains()
920 b->pn = -1; in generate_sched_domains()
947 css_task_iter_start(&cs->css, 0, &it); in update_tasks_root_domain()
974 if (cpumask_empty(cs->effective_cpus)) { in rebuild_root_domains()
979 css_get(&cs->css); in rebuild_root_domains()
986 css_put(&cs->css); in rebuild_root_domains()
1004 * If the flag 'sched_load_balance' of any cpuset with non-empty
1006 * which has that flag enabled, or if any cpuset with a non-empty
1048 if (!cpumask_subset(cs->effective_cpus, in rebuild_sched_domains_locked()
1079 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
1091 css_task_iter_start(&cs->css, 0, &it); in update_tasks_cpumask()
1093 set_cpus_allowed_ptr(task, cs->effective_cpus); in update_tasks_cpumask()
1098 * compute_effective_cpumask - Compute the effective cpumask of the cpuset
1111 if (parent->nr_subparts_cpus) { in compute_effective_cpumask()
1112 cpumask_or(new_cpus, parent->effective_cpus, in compute_effective_cpumask()
1113 parent->subparts_cpus); in compute_effective_cpumask()
1114 cpumask_and(new_cpus, new_cpus, cs->cpus_allowed); in compute_effective_cpumask()
1117 cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); in compute_effective_cpumask()
1131 * update_parent_subparts_cpumask - update subparts_cpus mask of parent cpuset
1138 * For partcmd_enable, the cpuset is being transformed from a non-partition
1145 * root back to a non-partition root. Any CPUs in cpus_allowed that are in
1156 * Error code should only be returned when newmask is non-NULL.
1187 * not be empty. in update_parent_subparts_cpumask()
1191 (!newmask && cpumask_empty(cpuset->cpus_allowed))) in update_parent_subparts_cpumask()
1192 return -EINVAL; in update_parent_subparts_cpumask()
1198 if ((cmd != partcmd_update) && css_has_online_children(&cpuset->css)) in update_parent_subparts_cpumask()
1199 return -EBUSY; in update_parent_subparts_cpumask()
1207 (!cpumask_subset(cpuset->cpus_allowed, parent->effective_cpus) || in update_parent_subparts_cpumask()
1208 cpumask_equal(cpuset->cpus_allowed, parent->effective_cpus))) in update_parent_subparts_cpumask()
1209 return -EINVAL; in update_parent_subparts_cpumask()
1212 * A cpumask update cannot make parent's effective_cpus become empty. in update_parent_subparts_cpumask()
1215 old_prs = new_prs = cpuset->partition_root_state; in update_parent_subparts_cpumask()
1217 cpumask_copy(tmp->addmask, cpuset->cpus_allowed); in update_parent_subparts_cpumask()
1220 deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed, in update_parent_subparts_cpumask()
1221 parent->subparts_cpus); in update_parent_subparts_cpumask()
1226 * delmask = cpus_allowed & ~newmask & parent->subparts_cpus in update_parent_subparts_cpumask()
1227 * addmask = newmask & parent->effective_cpus in update_parent_subparts_cpumask()
1228 * & ~parent->subparts_cpus in update_parent_subparts_cpumask()
1230 cpumask_andnot(tmp->delmask, cpuset->cpus_allowed, newmask); in update_parent_subparts_cpumask()
1231 deleting = cpumask_and(tmp->delmask, tmp->delmask, in update_parent_subparts_cpumask()
1232 parent->subparts_cpus); in update_parent_subparts_cpumask()
1234 cpumask_and(tmp->addmask, newmask, parent->effective_cpus); in update_parent_subparts_cpumask()
1235 adding = cpumask_andnot(tmp->addmask, tmp->addmask, in update_parent_subparts_cpumask()
1236 parent->subparts_cpus); in update_parent_subparts_cpumask()
1238 * Return error if the new effective_cpus could become empty. in update_parent_subparts_cpumask()
1241 cpumask_equal(parent->effective_cpus, tmp->addmask)) { in update_parent_subparts_cpumask()
1243 return -EINVAL; in update_parent_subparts_cpumask()
1249 if (!cpumask_and(tmp->addmask, tmp->delmask, in update_parent_subparts_cpumask()
1251 return -EINVAL; in update_parent_subparts_cpumask()
1252 cpumask_copy(tmp->addmask, parent->effective_cpus); in update_parent_subparts_cpumask()
1258 * addmask = cpus_allowed & parent->effective_cpus in update_parent_subparts_cpumask()
1261 * pre-shrunk in case there is a change in the cpu list. in update_parent_subparts_cpumask()
1264 adding = cpumask_and(tmp->addmask, cpuset->cpus_allowed, in update_parent_subparts_cpumask()
1265 parent->effective_cpus); in update_parent_subparts_cpumask()
1266 part_error = cpumask_equal(tmp->addmask, in update_parent_subparts_cpumask()
1267 parent->effective_cpus); in update_parent_subparts_cpumask()
1271 int prev_prs = cpuset->partition_root_state; in update_parent_subparts_cpumask()
1277 switch (cpuset->partition_root_state) { in update_parent_subparts_cpumask()
1301 deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed, in update_parent_subparts_cpumask()
1302 parent->subparts_cpus); in update_parent_subparts_cpumask()
1315 cpumask_or(parent->subparts_cpus, in update_parent_subparts_cpumask()
1316 parent->subparts_cpus, tmp->addmask); in update_parent_subparts_cpumask()
1317 cpumask_andnot(parent->effective_cpus, in update_parent_subparts_cpumask()
1318 parent->effective_cpus, tmp->addmask); in update_parent_subparts_cpumask()
1321 cpumask_andnot(parent->subparts_cpus, in update_parent_subparts_cpumask()
1322 parent->subparts_cpus, tmp->delmask); in update_parent_subparts_cpumask()
1326 cpumask_and(tmp->delmask, tmp->delmask, cpu_active_mask); in update_parent_subparts_cpumask()
1327 cpumask_or(parent->effective_cpus, in update_parent_subparts_cpumask()
1328 parent->effective_cpus, tmp->delmask); in update_parent_subparts_cpumask()
1331 parent->nr_subparts_cpus = cpumask_weight(parent->subparts_cpus); in update_parent_subparts_cpumask()
1334 cpuset->partition_root_state = new_prs; in update_parent_subparts_cpumask()
1343 * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
1365 compute_effective_cpumask(tmp->new_cpus, cp, parent); in update_cpumasks_hier()
1368 * If it becomes empty, inherit the effective mask of the in update_cpumasks_hier()
1371 if (is_in_v2_mode() && cpumask_empty(tmp->new_cpus)) { in update_cpumasks_hier()
1372 cpumask_copy(tmp->new_cpus, parent->effective_cpus); in update_cpumasks_hier()
1373 if (!cp->use_parent_ecpus) { in update_cpumasks_hier()
1374 cp->use_parent_ecpus = true; in update_cpumasks_hier()
1375 parent->child_ecpus_count++; in update_cpumasks_hier()
1377 } else if (cp->use_parent_ecpus) { in update_cpumasks_hier()
1378 cp->use_parent_ecpus = false; in update_cpumasks_hier()
1379 WARN_ON_ONCE(!parent->child_ecpus_count); in update_cpumasks_hier()
1380 parent->child_ecpus_count--; in update_cpumasks_hier()
1387 if (!cp->partition_root_state && in update_cpumasks_hier()
1388 cpumask_equal(tmp->new_cpus, cp->effective_cpus)) { in update_cpumasks_hier()
1399 old_prs = new_prs = cp->partition_root_state; in update_cpumasks_hier()
1401 switch (parent->partition_root_state) { in update_cpumasks_hier()
1408 WARN_ON_ONCE(cp->partition_root_state in update_cpumasks_hier()
1419 clear_bit(CS_CPU_EXCLUSIVE, &cp->flags); in update_cpumasks_hier()
1436 if (!css_tryget_online(&cp->css)) in update_cpumasks_hier()
1442 cpumask_copy(cp->effective_cpus, tmp->new_cpus); in update_cpumasks_hier()
1443 if (cp->nr_subparts_cpus && (new_prs != PRS_ENABLED)) { in update_cpumasks_hier()
1444 cp->nr_subparts_cpus = 0; in update_cpumasks_hier()
1445 cpumask_clear(cp->subparts_cpus); in update_cpumasks_hier()
1446 } else if (cp->nr_subparts_cpus) { in update_cpumasks_hier()
1452 * becomes empty. we clear cp->nr_subparts_cpus and in update_cpumasks_hier()
1456 cpumask_andnot(cp->effective_cpus, cp->effective_cpus, in update_cpumasks_hier()
1457 cp->subparts_cpus); in update_cpumasks_hier()
1458 if (cpumask_empty(cp->effective_cpus)) { in update_cpumasks_hier()
1459 cpumask_copy(cp->effective_cpus, tmp->new_cpus); in update_cpumasks_hier()
1460 cpumask_clear(cp->subparts_cpus); in update_cpumasks_hier()
1461 cp->nr_subparts_cpus = 0; in update_cpumasks_hier()
1462 } else if (!cpumask_subset(cp->subparts_cpus, in update_cpumasks_hier()
1463 tmp->new_cpus)) { in update_cpumasks_hier()
1464 cpumask_andnot(cp->subparts_cpus, in update_cpumasks_hier()
1465 cp->subparts_cpus, tmp->new_cpus); in update_cpumasks_hier()
1466 cp->nr_subparts_cpus in update_cpumasks_hier()
1467 = cpumask_weight(cp->subparts_cpus); in update_cpumasks_hier()
1472 cp->partition_root_state = new_prs; in update_cpumasks_hier()
1478 !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); in update_cpumasks_hier()
1483 * On legacy hierarchy, if the effective cpumask of any non- in update_cpumasks_hier()
1484 * empty cpuset is changed, we need to rebuild sched domains. in update_cpumasks_hier()
1488 if (!cpumask_empty(cp->cpus_allowed) && in update_cpumasks_hier()
1495 css_put(&cp->css); in update_cpumasks_hier()
1504 * update_sibling_cpumasks - Update siblings cpumasks
1524 if (!sibling->use_parent_ecpus) in update_sibling_cpumasks()
1533 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
1544 /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */ in update_cpumask()
1546 return -EACCES; in update_cpumask()
1549 * An empty cpus_allowed is ok only if the cpuset has no tasks. in update_cpumask()
1550 * Since cpulist_parse() fails on an empty mask, we special case in update_cpumask()
1555 cpumask_clear(trialcs->cpus_allowed); in update_cpumask()
1557 retval = cpulist_parse(buf, trialcs->cpus_allowed); in update_cpumask()
1561 if (!cpumask_subset(trialcs->cpus_allowed, in update_cpumask()
1563 return -EINVAL; in update_cpumask()
1567 if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) in update_cpumask()
1579 tmp.addmask = trialcs->subparts_cpus; in update_cpumask()
1580 tmp.delmask = trialcs->effective_cpus; in update_cpumask()
1581 tmp.new_cpus = trialcs->cpus_allowed; in update_cpumask()
1584 if (cs->partition_root_state) { in update_cpumask()
1585 /* Cpumask of a partition root cannot be empty */ in update_cpumask()
1586 if (cpumask_empty(trialcs->cpus_allowed)) in update_cpumask()
1587 return -EINVAL; in update_cpumask()
1589 trialcs->cpus_allowed, &tmp) < 0) in update_cpumask()
1590 return -EINVAL; in update_cpumask()
1594 cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); in update_cpumask()
1599 if (cs->nr_subparts_cpus) { in update_cpumask()
1600 cpumask_andnot(cs->subparts_cpus, cs->subparts_cpus, in update_cpumask()
1601 cs->cpus_allowed); in update_cpumask()
1602 cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus); in update_cpumask()
1608 if (cs->partition_root_state) { in update_cpumask()
1615 if (parent->child_ecpus_count) in update_cpumask()
1642 do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL); in cpuset_migrate_mm_workfn()
1643 mmput(mwork->mm); in cpuset_migrate_mm_workfn()
1659 mwork->mm = mm; in cpuset_migrate_mm()
1660 mwork->from = *from; in cpuset_migrate_mm()
1661 mwork->to = *to; in cpuset_migrate_mm()
1662 INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn); in cpuset_migrate_mm()
1663 queue_work(cpuset_migrate_mm_wq, &mwork->work); in cpuset_migrate_mm()
1675 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
1679 * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
1681 * parallel, it might temporarily see an empty intersection, which results in
1690 write_seqcount_begin(&tsk->mems_allowed_seq); in cpuset_change_task_nodemask()
1692 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); in cpuset_change_task_nodemask()
1694 tsk->mems_allowed = *newmems; in cpuset_change_task_nodemask()
1696 write_seqcount_end(&tsk->mems_allowed_seq); in cpuset_change_task_nodemask()
1705 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
1724 * take while holding tasklist_lock. Forks can happen - the in update_tasks_nodemask()
1732 css_task_iter_start(&cs->css, 0, &it); in update_tasks_nodemask()
1745 mpol_rebind_mm(mm, &cs->mems_allowed); in update_tasks_nodemask()
1747 cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); in update_tasks_nodemask()
1755 * cs->old_mems_allowed. in update_tasks_nodemask()
1757 cs->old_mems_allowed = newmems; in update_tasks_nodemask()
1764 * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
1784 nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); in update_nodemasks_hier()
1787 * If it becomes empty, inherit the effective mask of the in update_nodemasks_hier()
1791 *new_mems = parent->effective_mems; in update_nodemasks_hier()
1794 if (nodes_equal(*new_mems, cp->effective_mems)) { in update_nodemasks_hier()
1799 if (!css_tryget_online(&cp->css)) in update_nodemasks_hier()
1804 cp->effective_mems = *new_mems; in update_nodemasks_hier()
1808 !nodes_equal(cp->mems_allowed, cp->effective_mems)); in update_nodemasks_hier()
1813 css_put(&cp->css); in update_nodemasks_hier()
1828 * lock each such tasks mm->mmap_lock, scan its vma's and rebind
1838 * it's read-only in update_nodemask()
1841 retval = -EACCES; in update_nodemask()
1846 * An empty mems_allowed is ok iff there are no tasks in the cpuset. in update_nodemask()
1847 * Since nodelist_parse() fails on an empty mask, we special case in update_nodemask()
1852 nodes_clear(trialcs->mems_allowed); in update_nodemask()
1854 retval = nodelist_parse(buf, trialcs->mems_allowed); in update_nodemask()
1858 if (!nodes_subset(trialcs->mems_allowed, in update_nodemask()
1860 retval = -EINVAL; in update_nodemask()
1865 if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) { in update_nodemask()
1866 retval = 0; /* Too easy - nothing to do */ in update_nodemask()
1874 cs->mems_allowed = trialcs->mems_allowed; in update_nodemask()
1877 /* use trialcs->mems_allowed as a temp variable */ in update_nodemask()
1878 update_nodemasks_hier(cs, &trialcs->mems_allowed); in update_nodemask()
1897 if (val < -1 || val >= sched_domain_level_max) in update_relax_domain_level()
1898 return -EINVAL; in update_relax_domain_level()
1901 if (val != cs->relax_domain_level) { in update_relax_domain_level()
1902 cs->relax_domain_level = val; in update_relax_domain_level()
1903 if (!cpumask_empty(cs->cpus_allowed) && in update_relax_domain_level()
1912 * update_tasks_flags - update the spread flags of tasks in the cpuset.
1924 css_task_iter_start(&cs->css, 0, &it); in update_tasks_flags()
1931 * update_flag - read a 0 or a 1 in a file and update associated flag
1949 return -ENOMEM; in update_flag()
1952 set_bit(bit, &trialcs->flags); in update_flag()
1954 clear_bit(bit, &trialcs->flags); in update_flag()
1967 cs->flags = trialcs->flags; in update_flag()
1970 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) in update_flag()
1981 * update_prstate - update partititon_root_state
1989 int err, old_prs = cs->partition_root_state; in update_prstate()
2001 return -EINVAL; in update_prstate()
2004 return -ENOMEM; in update_prstate()
2006 err = -EINVAL; in update_prstate()
2013 if (cpumask_empty(cs->cpus_allowed)) in update_prstate()
2053 if (parent->child_ecpus_count) in update_prstate()
2060 cs->partition_root_state = new_prs; in update_prstate()
2070 * Frequency meter - How fast is some event occurring?
2074 * fmeter_init() - initialize a frequency meter.
2075 * fmeter_markevent() - called each time the event happens.
2076 * fmeter_getrate() - returns the recent rate of such events.
2077 * fmeter_update() - internal routine used to update fmeter.
2084 * The filter is single-pole low-pass recursive (IIR). The time unit
2085 * is 1 second. Arithmetic is done using 32-bit integers scaled to
2089 * has a half-life of 10 seconds, meaning that if the events quit
2114 #define FM_COEF 933 /* coefficient for half-life of 10 secs */
2122 fmp->cnt = 0; in fmeter_init()
2123 fmp->val = 0; in fmeter_init()
2124 fmp->time = 0; in fmeter_init()
2125 spin_lock_init(&fmp->lock); in fmeter_init()
2128 /* Internal meter update - process cnt events and update value */
2135 ticks = now - fmp->time; in fmeter_update()
2141 while (ticks-- > 0) in fmeter_update()
2142 fmp->val = (FM_COEF * fmp->val) / FM_SCALE; in fmeter_update()
2143 fmp->time = now; in fmeter_update()
2145 fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE; in fmeter_update()
2146 fmp->cnt = 0; in fmeter_update()
2152 spin_lock(&fmp->lock); in fmeter_markevent()
2154 fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE); in fmeter_markevent()
2155 spin_unlock(&fmp->lock); in fmeter_markevent()
2163 spin_lock(&fmp->lock); in fmeter_getrate()
2165 val = fmp->val; in fmeter_getrate()
2166 spin_unlock(&fmp->lock); in fmeter_getrate()
2186 /* allow moving tasks into an empty cpuset if on default hierarchy */ in cpuset_can_attach()
2187 ret = -ENOSPC; in cpuset_can_attach()
2189 (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))) in cpuset_can_attach()
2193 ret = task_can_attach(task, cs->cpus_allowed); in cpuset_can_attach()
2205 cs->attach_in_progress++; in cpuset_can_attach()
2219 css_cs(css)->attach_in_progress--; in cpuset_cancel_attach()
2266 cpuset_attach_nodemask_to = cs->effective_mems; in cpuset_attach()
2277 * @mems_allowed has been updated and is empty, so in cpuset_attach()
2282 cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, in cpuset_attach()
2289 cs->old_mems_allowed = cpuset_attach_nodemask_to; in cpuset_attach()
2291 cs->attach_in_progress--; in cpuset_attach()
2292 if (!cs->attach_in_progress) in cpuset_attach()
2323 cpuset_filetype_t type = cft->private; in cpuset_write_u64()
2329 retval = -ENODEV; in cpuset_write_u64()
2359 retval = -EINVAL; in cpuset_write_u64()
2372 cpuset_filetype_t type = cft->private; in cpuset_write_s64()
2373 int retval = -ENODEV; in cpuset_write_s64()
2385 retval = -EINVAL; in cpuset_write_s64()
2402 int retval = -ENODEV; in cpuset_write_resmask()
2425 css_get(&cs->css); in cpuset_write_resmask()
2426 kernfs_break_active_protection(of->kn); in cpuset_write_resmask()
2436 retval = -ENOMEM; in cpuset_write_resmask()
2440 switch (of_cft(of)->private) { in cpuset_write_resmask()
2448 retval = -EINVAL; in cpuset_write_resmask()
2456 kernfs_unbreak_active_protection(of->kn); in cpuset_write_resmask()
2457 css_put(&cs->css); in cpuset_write_resmask()
2473 cpuset_filetype_t type = seq_cft(sf)->private; in cpuset_common_seq_show()
2480 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); in cpuset_common_seq_show()
2483 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); in cpuset_common_seq_show()
2486 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); in cpuset_common_seq_show()
2489 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); in cpuset_common_seq_show()
2492 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->subparts_cpus)); in cpuset_common_seq_show()
2495 ret = -EINVAL; in cpuset_common_seq_show()
2505 cpuset_filetype_t type = cft->private; in cpuset_read_u64()
2520 return fmeter_getrate(&cs->fmeter); in cpuset_read_u64()
2536 cpuset_filetype_t type = cft->private; in cpuset_read_s64()
2539 return cs->relax_domain_level; in cpuset_read_s64()
2552 switch (cs->partition_root_state) { in sched_partition_show()
2571 int retval = -ENODEV; in sched_partition_write()
2583 return -EINVAL; in sched_partition_write()
2585 css_get(&cs->css); in sched_partition_write()
2595 css_put(&cs->css); in sched_partition_write()
2761 * cpuset_css_alloc - allocate a cpuset css
2775 return ERR_PTR(-ENOMEM); in cpuset_css_alloc()
2779 return ERR_PTR(-ENOMEM); in cpuset_css_alloc()
2782 __set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in cpuset_css_alloc()
2783 nodes_clear(cs->mems_allowed); in cpuset_css_alloc()
2784 nodes_clear(cs->effective_mems); in cpuset_css_alloc()
2785 fmeter_init(&cs->fmeter); in cpuset_css_alloc()
2786 cs->relax_domain_level = -1; in cpuset_css_alloc()
2790 __set_bit(CS_MEMORY_MIGRATE, &cs->flags); in cpuset_css_alloc()
2792 return &cs->css; in cpuset_css_alloc()
2808 set_bit(CS_ONLINE, &cs->flags); in cpuset_css_online()
2810 set_bit(CS_SPREAD_PAGE, &cs->flags); in cpuset_css_online()
2812 set_bit(CS_SPREAD_SLAB, &cs->flags); in cpuset_css_online()
2818 cpumask_copy(cs->effective_cpus, parent->effective_cpus); in cpuset_css_online()
2819 cs->effective_mems = parent->effective_mems; in cpuset_css_online()
2820 cs->use_parent_ecpus = true; in cpuset_css_online()
2821 parent->child_ecpus_count++; in cpuset_css_online()
2825 if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags)) in cpuset_css_online()
2831 * histrical reasons - the flag may be specified during mount. in cpuset_css_online()
2834 * refuse to clone the configuration - thereby refusing the task to in cpuset_css_online()
2838 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive in cpuset_css_online()
2851 cs->mems_allowed = parent->mems_allowed; in cpuset_css_online()
2852 cs->effective_mems = parent->mems_allowed; in cpuset_css_online()
2853 cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); in cpuset_css_online()
2854 cpumask_copy(cs->effective_cpus, parent->cpus_allowed); in cpuset_css_online()
2887 if (cs->use_parent_ecpus) { in cpuset_css_offline()
2890 cs->use_parent_ecpus = false; in cpuset_css_offline()
2891 parent->child_ecpus_count--; in cpuset_css_offline()
2895 clear_bit(CS_ONLINE, &cs->flags); in cpuset_css_offline()
2936 set_cpus_allowed_ptr(task, current->cpus_ptr); in cpuset_fork()
2937 task->mems_allowed = current->mems_allowed; in cpuset_fork()
2958 * cpuset_init - initialize cpusets at system boot
2978 top_cpuset.relax_domain_level = -1; in cpuset_init()
2989 * last CPU or node from a cpuset, then move the tasks in the empty
2990 * cpuset to its next-highest non-empty parent.
2997 * Find its next-highest non-empty parent, (top cpuset in remove_tasks_in_empty_cpuset()
2998 * has online cpus, so can't be empty). in remove_tasks_in_empty_cpuset()
3001 while (cpumask_empty(parent->cpus_allowed) || in remove_tasks_in_empty_cpuset()
3002 nodes_empty(parent->mems_allowed)) in remove_tasks_in_empty_cpuset()
3005 if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) { in remove_tasks_in_empty_cpuset()
3006 pr_err("cpuset: failed to transfer tasks out of empty cpuset "); in remove_tasks_in_empty_cpuset()
3007 pr_cont_cgroup_name(cs->css.cgroup); in remove_tasks_in_empty_cpuset()
3020 cpumask_copy(cs->cpus_allowed, new_cpus); in hotplug_update_tasks_legacy()
3021 cpumask_copy(cs->effective_cpus, new_cpus); in hotplug_update_tasks_legacy()
3022 cs->mems_allowed = *new_mems; in hotplug_update_tasks_legacy()
3023 cs->effective_mems = *new_mems; in hotplug_update_tasks_legacy()
3027 * Don't call update_tasks_cpumask() if the cpuset becomes empty, in hotplug_update_tasks_legacy()
3030 if (cpus_updated && !cpumask_empty(cs->cpus_allowed)) in hotplug_update_tasks_legacy()
3032 if (mems_updated && !nodes_empty(cs->mems_allowed)) in hotplug_update_tasks_legacy()
3035 is_empty = cpumask_empty(cs->cpus_allowed) || in hotplug_update_tasks_legacy()
3036 nodes_empty(cs->mems_allowed); in hotplug_update_tasks_legacy()
3057 cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); in hotplug_update_tasks()
3059 *new_mems = parent_cs(cs)->effective_mems; in hotplug_update_tasks()
3062 cpumask_copy(cs->effective_cpus, new_cpus); in hotplug_update_tasks()
3063 cs->effective_mems = *new_mems; in hotplug_update_tasks()
3080 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
3096 wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); in cpuset_hotplug_update_tasks()
3102 * is finished, so we won't attach a task to an empty cpuset. in cpuset_hotplug_update_tasks()
3104 if (cs->attach_in_progress) { in cpuset_hotplug_update_tasks()
3111 nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); in cpuset_hotplug_update_tasks()
3113 if (cs->nr_subparts_cpus) in cpuset_hotplug_update_tasks()
3118 cpumask_andnot(&new_cpus, &new_cpus, cs->subparts_cpus); in cpuset_hotplug_update_tasks()
3120 if (!tmp || !cs->partition_root_state) in cpuset_hotplug_update_tasks()
3124 * In the unlikely event that a partition root has empty in cpuset_hotplug_update_tasks()
3129 (parent->partition_root_state == PRS_ERROR))) { in cpuset_hotplug_update_tasks()
3130 if (cs->nr_subparts_cpus) { in cpuset_hotplug_update_tasks()
3132 cs->nr_subparts_cpus = 0; in cpuset_hotplug_update_tasks()
3133 cpumask_clear(cs->subparts_cpus); in cpuset_hotplug_update_tasks()
3139 * If the effective_cpus is empty because the child in cpuset_hotplug_update_tasks()
3144 if ((parent->partition_root_state == PRS_ERROR) || in cpuset_hotplug_update_tasks()
3150 old_prs = cs->partition_root_state; in cpuset_hotplug_update_tasks()
3153 cs->partition_root_state = PRS_ERROR; in cpuset_hotplug_update_tasks()
3167 ((cs->partition_root_state == PRS_ERROR) || in cpuset_hotplug_update_tasks()
3168 !cpumask_intersects(&new_cpus, parent->subparts_cpus)) && in cpuset_hotplug_update_tasks()
3173 cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); in cpuset_hotplug_update_tasks()
3174 mems_updated = !nodes_equal(new_mems, cs->effective_mems); in cpuset_hotplug_update_tasks()
3187 * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
3195 * Non-root cpusets are only affected by offlining. If any CPUs or memory
3279 if (cs == &top_cpuset || !css_tryget_online(&cs->css)) in cpuset_hotplug_workfn()
3286 css_put(&cs->css); in cpuset_hotplug_workfn()
3333 * cpuset_init_smp - initialize cpus_allowed
3353 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
3354 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
3358 * attached to the specified @tsk. Guaranteed to return some non-empty
3373 * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
3377 * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
3378 * mode however, this value is the same as task_cs(tsk)->effective_cpus,
3393 cs_mask = task_cs(tsk)->cpus_allowed; in cpuset_cpus_allowed_fallback()
3401 * We own tsk->cpus_allowed, nobody can change it under us. in cpuset_cpus_allowed_fallback()
3403 * But we used cs && cs->cpus_allowed lockless and thus can in cpuset_cpus_allowed_fallback()
3405 * the wrong tsk->cpus_allowed. However, both cases imply the in cpuset_cpus_allowed_fallback()
3406 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr() in cpuset_cpus_allowed_fallback()
3410 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary in cpuset_cpus_allowed_fallback()
3422 nodes_setall(current->mems_allowed); in cpuset_init_current_mems_allowed()
3426 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
3427 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
3430 * attached to the specified @tsk. Guaranteed to return some non-empty
3450 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed
3453 * Are any of the nodes in the nodemask allowed in current->mems_allowed?
3457 return nodes_intersects(*nodemask, current->mems_allowed); in cpuset_nodemask_valid_mems_allowed()
3461 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
3474 * cpuset_node_allowed - Can we allocate on a memory node?
3493 * current tasks mems_allowed came up empty on the first pass over
3507 * in_interrupt - any node ok (current task context irrelevant)
3508 * GFP_ATOMIC - any node ok
3509 * tsk_is_oom_victim - any node ok
3510 * GFP_KERNEL - any node in enclosing hardwalled cpuset ok
3511 * GFP_USER - only nodes in current tasks mems allowed ok.
3521 if (node_isset(node, current->mems_allowed)) in __cpuset_node_allowed()
3532 if (current->flags & PF_EXITING) /* Let dying task have memory */ in __cpuset_node_allowed()
3540 allowed = node_isset(node, cs->mems_allowed); in __cpuset_node_allowed()
3548 * cpuset_mem_spread_node() - On which node to begin search for a file page
3549 * cpuset_slab_spread_node() - On which node to begin search for a slab page
3564 * only set nodes in task->mems_allowed that are online. So it
3576 return *rotor = next_node_in(*rotor, current->mems_allowed); in cpuset_spread_node()
3581 if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) in cpuset_mem_spread_node()
3582 current->cpuset_mem_spread_rotor = in cpuset_mem_spread_node()
3583 node_random(¤t->mems_allowed); in cpuset_mem_spread_node()
3585 return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); in cpuset_mem_spread_node()
3590 if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE) in cpuset_slab_spread_node()
3591 current->cpuset_slab_spread_rotor = in cpuset_slab_spread_node()
3592 node_random(¤t->mems_allowed); in cpuset_slab_spread_node()
3594 return cpuset_spread_node(¤t->cpuset_slab_spread_rotor); in cpuset_slab_spread_node()
3600 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
3613 return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); in cpuset_mems_allowed_intersects()
3617 * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
3628 cgrp = task_cs(current)->css.cgroup; in cpuset_print_current_mems_allowed()
3632 nodemask_pr_args(¤t->mems_allowed)); in cpuset_print_current_mems_allowed()
3646 * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
3657 * Display to user space in the per-cpuset read-only file
3666 fmeter_markevent(&task_cs(current)->fmeter); in __cpuset_memory_pressure_bump()
3673 * - Print tasks cpuset path into seq_file.
3674 * - Used for /proc/<pid>/cpuset.
3675 * - No need to task_lock(tsk) on this tsk->cpuset reference, as it
3676 * doesn't really matter if tsk->cpuset changes after we read it,
3687 retval = -ENOMEM; in proc_cpuset_show()
3693 retval = cgroup_path_ns(css->cgroup, buf, PATH_MAX, in proc_cpuset_show()
3694 current->nsproxy->cgroup_ns); in proc_cpuset_show()
3697 retval = -ENAMETOOLONG; in proc_cpuset_show()
3714 nodemask_pr_args(&task->mems_allowed)); in cpuset_task_status_allowed()
3716 nodemask_pr_args(&task->mems_allowed)); in cpuset_task_status_allowed()