Lines Matching +full:current +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0-only
123 * run-time system-wide default policy => local allocation
127 .mode = MPOL_LOCAL,
133 * numa_map_to_online_node - Find closest online node
160 struct mempolicy *pol = p->mempolicy; in get_task_policy()
170 if (pol->mode) in get_task_policy()
184 return pol->flags & MPOL_MODE_FLAGS; in mpol_store_user_nodemask()
198 return -EINVAL; in mpol_new_nodemask()
199 pol->nodes = *nodes; in mpol_new_nodemask()
206 return -EINVAL; in mpol_new_preferred()
208 nodes_clear(pol->nodes); in mpol_new_preferred()
209 node_set(first_node(*nodes), pol->nodes); in mpol_new_preferred()
216 * parameter with respect to the policy mode and flags.
231 if (!pol || pol->mode == MPOL_LOCAL) in mpol_set_nodemask()
235 nodes_and(nsc->mask1, in mpol_set_nodemask()
240 if (pol->flags & MPOL_F_RELATIVE_NODES) in mpol_set_nodemask()
241 mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1); in mpol_set_nodemask()
243 nodes_and(nsc->mask2, *nodes, nsc->mask1); in mpol_set_nodemask()
246 pol->w.user_nodemask = *nodes; in mpol_set_nodemask()
248 pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed; in mpol_set_nodemask()
250 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2); in mpol_set_nodemask()
258 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags, in mpol_new() argument
263 pr_debug("setting mode %d flags %d nodes[0] %lx\n", in mpol_new()
264 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE); in mpol_new()
266 if (mode == MPOL_DEFAULT) { in mpol_new()
268 return ERR_PTR(-EINVAL); in mpol_new()
276 * All other modes require a valid pointer to a non-empty nodemask. in mpol_new()
278 if (mode == MPOL_PREFERRED) { in mpol_new()
282 return ERR_PTR(-EINVAL); in mpol_new()
284 mode = MPOL_LOCAL; in mpol_new()
286 } else if (mode == MPOL_LOCAL) { in mpol_new()
290 return ERR_PTR(-EINVAL); in mpol_new()
292 return ERR_PTR(-EINVAL); in mpol_new()
295 return ERR_PTR(-ENOMEM); in mpol_new()
296 atomic_set(&policy->refcnt, 1); in mpol_new()
297 policy->mode = mode; in mpol_new()
298 policy->flags = flags; in mpol_new()
306 if (!atomic_dec_and_test(&p->refcnt)) in __mpol_put()
319 if (pol->flags & MPOL_F_STATIC_NODES) in mpol_rebind_nodemask()
320 nodes_and(tmp, pol->w.user_nodemask, *nodes); in mpol_rebind_nodemask()
321 else if (pol->flags & MPOL_F_RELATIVE_NODES) in mpol_rebind_nodemask()
322 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); in mpol_rebind_nodemask()
324 nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed, in mpol_rebind_nodemask()
326 pol->w.cpuset_mems_allowed = *nodes; in mpol_rebind_nodemask()
332 pol->nodes = tmp; in mpol_rebind_nodemask()
338 pol->w.cpuset_mems_allowed = *nodes; in mpol_rebind_preferred()
342 * mpol_rebind_policy - Migrate a policy to a different set of nodes
344 * Per-vma policies are protected by mmap_lock. Allocations using per-task
345 * policies are protected by task->mems_allowed_seq to prevent a premature
353 nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) in mpol_rebind_policy()
356 mpol_ops[pol->mode].rebind(pol, newmask); in mpol_rebind_policy()
368 mpol_rebind_policy(tsk->mempolicy, new); in mpol_rebind_task()
374 * Call holding a reference to mm. Takes mm->mmap_lock during call.
382 for (vma = mm->mmap; vma; vma = vma->vm_next) in mpol_rebind_mm()
383 mpol_rebind_policy(vma->vm_policy, new); in mpol_rebind_mm()
425 * Check if the page's nid is in qp->nmask.
427 * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
428 * in the invert of qp->nmask.
434 unsigned long flags = qp->flags; in queue_pages_required()
436 return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT); in queue_pages_required()
441 * 0 - pages are placed on the right node or queued successfully, or
443 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
445 * 2 - THP was split.
446 * -EIO - is migration entry or only MPOL_MF_STRICT was specified and an
456 struct queue_pages *qp = walk->private; in queue_pages_pmd()
460 ret = -EIO; in queue_pages_pmd()
466 walk->action = ACTION_CONTINUE; in queue_pages_pmd()
472 flags = qp->flags; in queue_pages_pmd()
475 if (!vma_migratable(walk->vma) || in queue_pages_pmd()
476 migrate_page_add(page, qp->pagelist, flags)) { in queue_pages_pmd()
481 ret = -EIO; in queue_pages_pmd()
493 * 0 - pages are placed on the right node or queued successfully, or
495 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
497 * -EIO - only MPOL_MF_STRICT was specified and an existing page was already
503 struct vm_area_struct *vma = walk->vma; in queue_pages_pte_range()
505 struct queue_pages *qp = walk->private; in queue_pages_pte_range()
506 unsigned long flags = qp->flags; in queue_pages_pte_range()
523 mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); in queue_pages_pte_range()
550 if (migrate_page_add(page, qp->pagelist, flags)) in queue_pages_pte_range()
561 return addr != end ? -EIO : 0; in queue_pages_pte_range()
570 struct queue_pages *qp = walk->private; in queue_pages_hugetlb()
571 unsigned long flags = (qp->flags & MPOL_MF_VALID); in queue_pages_hugetlb()
576 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); in queue_pages_hugetlb()
589 ret = -EIO; in queue_pages_hugetlb()
593 if (!vma_migratable(walk->vma)) { in queue_pages_hugetlb()
596 * stopped walking current vma. in queue_pages_hugetlb()
607 if (!isolate_huge_page(page, qp->pagelist) && in queue_pages_hugetlb()
655 struct vm_area_struct *vma = walk->vma; in queue_pages_test_walk()
656 struct queue_pages *qp = walk->private; in queue_pages_test_walk()
657 unsigned long endvma = vma->vm_end; in queue_pages_test_walk()
658 unsigned long flags = qp->flags; in queue_pages_test_walk()
663 if (!qp->first) { in queue_pages_test_walk()
664 qp->first = vma; in queue_pages_test_walk()
666 (qp->start < vma->vm_start)) in queue_pages_test_walk()
668 return -EFAULT; in queue_pages_test_walk()
671 ((vma->vm_end < qp->end) && in queue_pages_test_walk()
672 (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) in queue_pages_test_walk()
674 return -EFAULT; in queue_pages_test_walk()
677 * Need check MPOL_MF_STRICT to return -EIO if possible in queue_pages_test_walk()
690 !(vma->vm_flags & VM_MIXEDMAP)) in queue_pages_test_walk()
695 /* queue pages from current vma */ in queue_pages_test_walk()
715 * 1 - there is unmovable page, but MPOL_MF_MOVE* & MPOL_MF_STRICT were
717 * 0 - queue pages successfully or no misplaced page.
718 * errno - i.e. misplaced pages with MPOL_MF_STRICT specified (-EIO) or
720 * your accessible address space (-EFAULT)
741 err = -EFAULT; in queue_pages_range()
757 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n", in vma_replace_policy()
758 vma->vm_start, vma->vm_end, vma->vm_pgoff, in vma_replace_policy()
759 vma->vm_ops, vma->vm_file, in vma_replace_policy()
760 vma->vm_ops ? vma->vm_ops->set_policy : NULL); in vma_replace_policy()
766 if (vma->vm_ops && vma->vm_ops->set_policy) { in vma_replace_policy()
767 err = vma->vm_ops->set_policy(vma, new); in vma_replace_policy()
772 old = vma->vm_policy; in vma_replace_policy()
773 vma->vm_policy = new; /* protected by mmap_lock */ in vma_replace_policy()
797 prev = vma->vm_prev; in mbind_range()
798 if (start > vma->vm_start) in mbind_range()
801 for (; vma && vma->vm_start < end; prev = vma, vma = next) { in mbind_range()
802 next = vma->vm_next; in mbind_range()
803 vmstart = max(start, vma->vm_start); in mbind_range()
804 vmend = min(end, vma->vm_end); in mbind_range()
809 pgoff = vma->vm_pgoff + in mbind_range()
810 ((vmstart - vma->vm_start) >> PAGE_SHIFT); in mbind_range()
811 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, in mbind_range()
812 vma->anon_vma, vma->vm_file, pgoff, in mbind_range()
813 new_pol, vma->vm_userfaultfd_ctx); in mbind_range()
816 next = vma->vm_next; in mbind_range()
819 /* vma_merge() joined vma && vma->next, case 8 */ in mbind_range()
822 if (vma->vm_start != vmstart) { in mbind_range()
823 err = split_vma(vma->vm_mm, vma, vmstart, 1); in mbind_range()
827 if (vma->vm_end != vmend) { in mbind_range()
828 err = split_vma(vma->vm_mm, vma, vmend, 0); in mbind_range()
843 static long do_set_mempolicy(unsigned short mode, unsigned short flags, in do_set_mempolicy() argument
851 return -ENOMEM; in do_set_mempolicy()
853 new = mpol_new(mode, flags, nodes); in do_set_mempolicy()
864 task_lock(current); in do_set_mempolicy()
865 old = current->mempolicy; in do_set_mempolicy()
866 current->mempolicy = new; in do_set_mempolicy()
867 if (new && new->mode == MPOL_INTERLEAVE) in do_set_mempolicy()
868 current->il_prev = MAX_NUMNODES-1; in do_set_mempolicy()
869 task_unlock(current); in do_set_mempolicy()
888 switch (p->mode) { in get_policy_nodemask()
893 *nodes = p->nodes; in get_policy_nodemask()
924 struct mm_struct *mm = current->mm; in do_get_mempolicy()
926 struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL; in do_get_mempolicy()
930 return -EINVAL; in do_get_mempolicy()
934 return -EINVAL; in do_get_mempolicy()
936 task_lock(current); in do_get_mempolicy()
938 task_unlock(current); in do_get_mempolicy()
952 return -EFAULT; in do_get_mempolicy()
954 if (vma->vm_ops && vma->vm_ops->get_policy) in do_get_mempolicy()
955 pol = vma->vm_ops->get_policy(vma, addr); in do_get_mempolicy()
957 pol = vma->vm_policy; in do_get_mempolicy()
959 return -EINVAL; in do_get_mempolicy()
979 } else if (pol == current->mempolicy && in do_get_mempolicy()
980 pol->mode == MPOL_INTERLEAVE) { in do_get_mempolicy()
981 *policy = next_node_in(current->il_prev, pol->nodes); in do_get_mempolicy()
983 err = -EINVAL; in do_get_mempolicy()
988 pol->mode; in do_get_mempolicy()
993 *policy |= (pol->flags & MPOL_MODE_FLAGS); in do_get_mempolicy()
999 *nmask = pol->w.user_nodemask; in do_get_mempolicy()
1001 task_lock(current); in do_get_mempolicy()
1003 task_unlock(current); in do_get_mempolicy()
1029 list_add_tail(&head->lru, pagelist); in migrate_page_add()
1035 * Non-movable page may reach here. And, there may be in migrate_page_add()
1036 * temporary off LRU pages or non-LRU movable pages. in migrate_page_add()
1039 * should return -EIO for this case too. in migrate_page_add()
1041 return -EIO; in migrate_page_add()
1072 queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, in migrate_to_node()
1152 * Example: [2,3,4] -> [3,4,5] moves everything. in do_migrate_pages()
1153 * [0-7] - > [3,4,5] moves only 0,1,2,6,7. in do_migrate_pages()
1194 * list of pages handed to migrate_pages()--which is how we get here--
1202 vma = find_vma(current->mm, start); in new_page()
1205 if (address != -EFAULT) in new_page()
1207 vma = vma->vm_next; in new_page()
1234 return -EIO; in migrate_page_add()
1240 return -ENOSYS; in do_migrate_pages()
1250 unsigned short mode, unsigned short mode_flags, in do_mbind() argument
1253 struct mm_struct *mm = current->mm; in do_mbind()
1261 return -EINVAL; in do_mbind()
1263 return -EPERM; in do_mbind()
1266 return -EINVAL; in do_mbind()
1268 if (mode == MPOL_DEFAULT) in do_mbind()
1271 len = (len + PAGE_SIZE - 1) & PAGE_MASK; in do_mbind()
1275 return -EINVAL; in do_mbind()
1279 new = mpol_new(mode, mode_flags, nmask); in do_mbind()
1284 new->flags |= MPOL_F_MOF; in do_mbind()
1293 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n", in do_mbind()
1294 start, start + len, mode, mode_flags, in do_mbind()
1309 err = -ENOMEM; in do_mbind()
1337 err = -EIO; in do_mbind()
1370 return -EFAULT; in get_bitmap()
1373 mask[nlongs - 1] &= (1UL << (maxnode % BITS_PER_LONG)) - 1; in get_bitmap()
1382 --maxnode; in get_nodes()
1387 return -EINVAL; in get_nodes()
1399 return -EFAULT; in get_nodes()
1401 if (maxnode - bits >= MAX_NUMNODES) { in get_nodes()
1402 maxnode -= bits; in get_nodes()
1405 t &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1); in get_nodes()
1408 return -EINVAL; in get_nodes()
1418 unsigned long copy = ALIGN(maxnode-1, 64) / 8; in copy_nodes_to_user()
1427 return -EINVAL; in copy_nodes_to_user()
1428 if (clear_user((char __user *)mask + nbytes, copy - nbytes)) in copy_nodes_to_user()
1429 return -EFAULT; in copy_nodes_to_user()
1438 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0; in copy_nodes_to_user()
1442 static inline int sanitize_mpol_flags(int *mode, unsigned short *flags) in sanitize_mpol_flags() argument
1444 *flags = *mode & MPOL_MODE_FLAGS; in sanitize_mpol_flags()
1445 *mode &= ~MPOL_MODE_FLAGS; in sanitize_mpol_flags()
1447 if ((unsigned int)(*mode) >= MPOL_MAX) in sanitize_mpol_flags()
1448 return -EINVAL; in sanitize_mpol_flags()
1450 return -EINVAL; in sanitize_mpol_flags()
1452 if (*mode != MPOL_BIND) in sanitize_mpol_flags()
1453 return -EINVAL; in sanitize_mpol_flags()
1460 unsigned long mode, const unsigned long __user *nmask, in kernel_mbind() argument
1465 int lmode = mode; in kernel_mbind()
1481 unsigned long, mode, const unsigned long __user *, nmask, in SYSCALL_DEFINE6() argument
1484 return kernel_mbind(start, len, mode, nmask, maxnode, flags); in SYSCALL_DEFINE6()
1488 static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask, in kernel_set_mempolicy() argument
1493 int lmode = mode; in kernel_set_mempolicy()
1507 SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask, in SYSCALL_DEFINE3() argument
1510 return kernel_set_mempolicy(mode, nmask, maxnode); in SYSCALL_DEFINE3()
1526 return -ENOMEM; in kernel_migrate_pages()
1528 old = &scratch->mask1; in kernel_migrate_pages()
1529 new = &scratch->mask2; in kernel_migrate_pages()
1541 task = pid ? find_task_by_vpid(pid) : current; in kernel_migrate_pages()
1544 err = -ESRCH; in kernel_migrate_pages()
1549 err = -EINVAL; in kernel_migrate_pages()
1557 err = -EPERM; in kernel_migrate_pages()
1565 err = -EPERM; in kernel_migrate_pages()
1569 task_nodes = cpuset_mems_allowed(current); in kernel_migrate_pages()
1582 err = -EINVAL; in kernel_migrate_pages()
1621 return -EINVAL; in kernel_get_mempolicy()
1631 return -EFAULT; in kernel_get_mempolicy()
1648 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) in vma_migratable()
1667 if (vma->vm_file && in vma_migratable()
1668 gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping)) in vma_migratable()
1680 if (vma->vm_ops && vma->vm_ops->get_policy) { in __get_vma_policy()
1681 pol = vma->vm_ops->get_policy(vma, addr); in __get_vma_policy()
1682 } else if (vma->vm_policy) { in __get_vma_policy()
1683 pol = vma->vm_policy; in __get_vma_policy()
1687 * a pseudo vma whose vma->vm_ops=NULL. Take a reference in __get_vma_policy()
1705 * Falls back to current->mempolicy or system default policy, as necessary.
1707 * count--added by the get_policy() vm_op, as appropriate--to protect against
1717 pol = get_task_policy(current); in get_vma_policy()
1726 if (vma->vm_ops && vma->vm_ops->get_policy) { in vma_policy_mof()
1729 pol = vma->vm_ops->get_policy(vma, vma->vm_start); in vma_policy_mof()
1730 if (pol && (pol->flags & MPOL_F_MOF)) in vma_policy_mof()
1737 pol = vma->vm_policy; in vma_policy_mof()
1739 pol = get_task_policy(current); in vma_policy_mof()
1741 return pol->flags & MPOL_F_MOF; in vma_policy_mof()
1751 * if policy->nodes has movable memory only, in apply_policy_zone()
1754 * policy->nodes is intersect with node_states[N_MEMORY]. in apply_policy_zone()
1756 * policy->nodes has movable memory only. in apply_policy_zone()
1758 if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY])) in apply_policy_zone()
1770 int mode = policy->mode; in policy_nodemask() local
1773 if (unlikely(mode == MPOL_BIND) && in policy_nodemask()
1775 cpuset_nodemask_valid_mems_allowed(&policy->nodes)) in policy_nodemask()
1776 return &policy->nodes; in policy_nodemask()
1778 if (mode == MPOL_PREFERRED_MANY) in policy_nodemask()
1779 return &policy->nodes; in policy_nodemask()
1789 * secures the nodemask limit for 'bind' and 'prefer-many' policy.
1793 if (policy->mode == MPOL_PREFERRED) { in policy_node()
1794 nd = first_node(policy->nodes); in policy_node()
1801 WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE)); in policy_node()
1811 struct task_struct *me = current; in interleave_nodes()
1813 next = next_node_in(me->il_prev, policy->nodes); in interleave_nodes()
1815 me->il_prev = next; in interleave_nodes()
1831 policy = current->mempolicy; in mempolicy_slab_node()
1835 switch (policy->mode) { in mempolicy_slab_node()
1837 return first_node(policy->nodes); in mempolicy_slab_node()
1853 zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK]; in mempolicy_slab_node()
1855 &policy->nodes); in mempolicy_slab_node()
1856 return z->zone ? zone_to_nid(z->zone) : node; in mempolicy_slab_node()
1868 * node in pol->nodes (starting from n=0), wrapping around if n exceeds the
1873 nodemask_t nodemask = pol->nodes; in offset_il_node()
1881 * Between first_node() and next_node(), pol->nodes could be changed in offset_il_node()
1882 * by other threads. So we put pol->nodes in a local stack. in offset_il_node()
1905 * shift and PAGE_SHIFT, so the bit-shift is safe. in interleave_nid()
1911 off = vma->vm_pgoff >> (shift - PAGE_SHIFT); in interleave_nid()
1912 off += (addr - vma->vm_start) >> shift; in interleave_nid()
1925 * @nodemask: pointer to nodemask pointer for 'bind' and 'prefer-many' policy
1929 * If the effective policy is 'bind' or 'prefer-many', returns a pointer
1938 int mode; in huge_node() local
1942 mode = (*mpol)->mode; in huge_node()
1944 if (unlikely(mode == MPOL_INTERLEAVE)) { in huge_node()
1949 if (mode == MPOL_BIND || mode == MPOL_PREFERRED_MANY) in huge_node()
1950 *nodemask = &(*mpol)->nodes; in huge_node()
1958 * If the current task's mempolicy is "default" [NULL], return 'false'
1963 * of non-default mempolicy.
1966 * because the current task is examining it's own mempolicy and a task's
1975 if (!(mask && current->mempolicy)) in init_nodemask_of_mempolicy()
1978 task_lock(current); in init_nodemask_of_mempolicy()
1979 mempolicy = current->mempolicy; in init_nodemask_of_mempolicy()
1980 switch (mempolicy->mode) { in init_nodemask_of_mempolicy()
1985 *mask = mempolicy->nodes; in init_nodemask_of_mempolicy()
1995 task_unlock(current); in init_nodemask_of_mempolicy()
2021 mempolicy = tsk->mempolicy; in mempolicy_in_oom_domain()
2022 if (mempolicy && mempolicy->mode == MPOL_BIND) in mempolicy_in_oom_domain()
2023 ret = nodes_intersects(mempolicy->nodes, *mask); in mempolicy_in_oom_domain()
2062 page = __alloc_pages(preferred_gfp, order, nid, &pol->nodes); in alloc_pages_preferred_many()
2070 * alloc_pages_vma - Allocate a page for a VMA.
2095 if (pol->mode == MPOL_INTERLEAVE) { in alloc_pages_vma()
2104 if (pol->mode == MPOL_PREFERRED_MANY) { in alloc_pages_vma()
2114 * For hugepage allocation and non-interleave policy which in alloc_pages_vma()
2115 * allows the current node (or other explicitly preferred in alloc_pages_vma()
2116 * node) we only try to allocate from the current/preferred in alloc_pages_vma()
2120 * If the policy is interleave or does not allow the current in alloc_pages_vma()
2123 if (pol->mode == MPOL_PREFERRED) in alloc_pages_vma()
2124 hpage_node = first_node(pol->nodes); in alloc_pages_vma()
2160 * alloc_pages - Allocate pages.
2165 * first page is naturally aligned (eg an order-3 allocation will be aligned
2166 * to a multiple of 8 * PAGE_SIZE bytes). The NUMA policy of the current
2179 pol = get_task_policy(current); in alloc_pages()
2182 * No reference counting needed for current->mempolicy in alloc_pages()
2185 if (pol->mode == MPOL_INTERLEAVE) in alloc_pages()
2187 else if (pol->mode == MPOL_PREFERRED_MANY) in alloc_pages()
2205 dst->vm_policy = pol; in vma_dup_policy()
2210 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2216 * current's mempolicy may be rebinded by the other task(the task that changes
2217 * cpuset's mems), so we needn't do rebind work for current task.
2226 return ERR_PTR(-ENOMEM); in __mpol_dup()
2229 if (old == current->mempolicy) { in __mpol_dup()
2230 task_lock(current); in __mpol_dup()
2232 task_unlock(current); in __mpol_dup()
2237 nodemask_t mems = cpuset_mems_allowed(current); in __mpol_dup()
2240 atomic_set(&new->refcnt, 1); in __mpol_dup()
2249 if (a->mode != b->mode) in __mpol_equal()
2251 if (a->flags != b->flags) in __mpol_equal()
2254 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask)) in __mpol_equal()
2257 switch (a->mode) { in __mpol_equal()
2262 return !!nodes_equal(a->nodes, b->nodes); in __mpol_equal()
2275 * The policies are kept in Red-Black tree linked from the inode.
2276 * They are protected by the sp->lock rwlock, which should be held
2281 * lookup first element intersecting start-end. Caller holds sp->lock for
2287 struct rb_node *n = sp->root.rb_node; in sp_lookup()
2292 if (start >= p->end) in sp_lookup()
2293 n = n->rb_right; in sp_lookup()
2294 else if (end <= p->start) in sp_lookup()
2295 n = n->rb_left; in sp_lookup()
2307 if (w->end <= start) in sp_lookup()
2315 * Insert a new shared policy into the list. Caller holds sp->lock for
2320 struct rb_node **p = &sp->root.rb_node; in sp_insert()
2327 if (new->start < nd->start) in sp_insert()
2328 p = &(*p)->rb_left; in sp_insert()
2329 else if (new->end > nd->end) in sp_insert()
2330 p = &(*p)->rb_right; in sp_insert()
2334 rb_link_node(&new->nd, parent, p); in sp_insert()
2335 rb_insert_color(&new->nd, &sp->root); in sp_insert()
2336 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end, in sp_insert()
2337 new->policy ? new->policy->mode : 0); in sp_insert()
2347 if (!sp->root.rb_node) in mpol_shared_policy_lookup()
2349 read_lock(&sp->lock); in mpol_shared_policy_lookup()
2352 mpol_get(sn->policy); in mpol_shared_policy_lookup()
2353 pol = sn->policy; in mpol_shared_policy_lookup()
2355 read_unlock(&sp->lock); in mpol_shared_policy_lookup()
2361 mpol_put(n->policy); in sp_free()
2366 * mpol_misplaced - check whether current page node is valid in policy
2372 * Lookup current policy node id for vma,addr and "compare to" page's
2391 if (!(pol->flags & MPOL_F_MOF)) in mpol_misplaced()
2394 switch (pol->mode) { in mpol_misplaced()
2396 pgoff = vma->vm_pgoff; in mpol_misplaced()
2397 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT; in mpol_misplaced()
2402 if (node_isset(curnid, pol->nodes)) in mpol_misplaced()
2404 polnid = first_node(pol->nodes); in mpol_misplaced()
2413 if (pol->flags & MPOL_F_MORON) { in mpol_misplaced()
2414 if (node_isset(thisnid, pol->nodes)) in mpol_misplaced()
2422 * use current page if in policy nodemask, in mpol_misplaced()
2424 * If no allowed nodes, use current [!misplaced]. in mpol_misplaced()
2426 if (node_isset(curnid, pol->nodes)) in mpol_misplaced()
2431 &pol->nodes); in mpol_misplaced()
2432 polnid = zone_to_nid(z->zone); in mpol_misplaced()
2440 if (pol->flags & MPOL_F_MORON) { in mpol_misplaced()
2443 if (!should_numa_migrate_memory(current, page, curnid, thiscpu)) in mpol_misplaced()
2456 * Drop the (possibly final) reference to task->mempolicy. It needs to be
2457 * dropped after task->mempolicy is set to NULL so that any allocation done as
2466 pol = task->mempolicy; in mpol_put_task_policy()
2467 task->mempolicy = NULL; in mpol_put_task_policy()
2474 pr_debug("deleting %lx-l%lx\n", n->start, n->end); in sp_delete()
2475 rb_erase(&n->nd, &sp->root); in sp_delete()
2482 node->start = start; in sp_node_init()
2483 node->end = end; in sp_node_init()
2484 node->policy = pol; in sp_node_init()
2502 newpol->flags |= MPOL_F_SHARED; in sp_alloc()
2518 write_lock(&sp->lock); in shared_policy_replace()
2521 while (n && n->start < end) { in shared_policy_replace()
2522 struct rb_node *next = rb_next(&n->nd); in shared_policy_replace()
2523 if (n->start >= start) { in shared_policy_replace()
2524 if (n->end <= end) in shared_policy_replace()
2527 n->start = end; in shared_policy_replace()
2530 if (n->end > end) { in shared_policy_replace()
2534 *mpol_new = *n->policy; in shared_policy_replace()
2535 atomic_set(&mpol_new->refcnt, 1); in shared_policy_replace()
2536 sp_node_init(n_new, end, n->end, mpol_new); in shared_policy_replace()
2537 n->end = start; in shared_policy_replace()
2543 n->end = start; in shared_policy_replace()
2551 write_unlock(&sp->lock); in shared_policy_replace()
2563 write_unlock(&sp->lock); in shared_policy_replace()
2564 ret = -ENOMEM; in shared_policy_replace()
2575 * mpol_shared_policy_init - initialize shared policy for inode
2579 * Install non-NULL @mpol in inode's shared policy rb-tree.
2580 * On entry, the current task has a reference on a non-NULL @mpol.
2588 sp->root = RB_ROOT; /* empty tree == default mempolicy */ in mpol_shared_policy_init()
2589 rwlock_init(&sp->lock); in mpol_shared_policy_init()
2599 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask); in mpol_shared_policy_init()
2603 task_lock(current); in mpol_shared_policy_init()
2604 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch); in mpol_shared_policy_init()
2605 task_unlock(current); in mpol_shared_policy_init()
2609 /* Create pseudo-vma that contains just the policy */ in mpol_shared_policy_init()
2631 vma->vm_pgoff, in mpol_set_shared_policy()
2632 sz, npol ? npol->mode : -1, in mpol_set_shared_policy()
2633 npol ? npol->flags : -1, in mpol_set_shared_policy()
2634 npol ? nodes_addr(npol->nodes)[0] : NUMA_NO_NODE); in mpol_set_shared_policy()
2637 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol); in mpol_set_shared_policy()
2639 return -ENOMEM; in mpol_set_shared_policy()
2641 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new); in mpol_set_shared_policy()
2653 if (!p->root.rb_node) in mpol_free_shared_policy()
2655 write_lock(&p->lock); in mpol_free_shared_policy()
2656 next = rb_first(&p->root); in mpol_free_shared_policy()
2659 next = rb_next(&n->nd); in mpol_free_shared_policy()
2662 write_unlock(&p->lock); in mpol_free_shared_policy()
2675 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */ in check_numabalancing_enable()
2696 numabalancing_override = -1; in setup_numabalancing()
2730 .mode = MPOL_PREFERRED, in numa_policy_init()
2766 /* Reset policy of current process to default */
2789 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2794 * <mode>[=<flags>][:<nodelist>]
2805 int err = 1, mode; in mpol_parse_str() local
2808 *flags++ = '\0'; /* terminate mode string */ in mpol_parse_str()
2811 /* NUL-terminate mode or flags string */ in mpol_parse_str()
2820 mode = match_string(policy_modes, MPOL_MAX, str); in mpol_parse_str()
2821 if (mode < 0) in mpol_parse_str()
2824 switch (mode) { in mpol_parse_str()
2875 * mode flags. in mpol_parse_str()
2885 new = mpol_new(mode, mode_flags, &nodes); in mpol_parse_str()
2893 if (mode != MPOL_PREFERRED) { in mpol_parse_str()
2894 new->nodes = nodes; in mpol_parse_str()
2896 nodes_clear(new->nodes); in mpol_parse_str()
2897 node_set(first_node(nodes), new->nodes); in mpol_parse_str()
2899 new->mode = MPOL_LOCAL; in mpol_parse_str()
2906 new->w.user_nodemask = nodes; in mpol_parse_str()
2913 *--nodelist = ':'; in mpol_parse_str()
2915 *--flags = '='; in mpol_parse_str()
2923 * mpol_to_str - format a mempolicy structure for printing
2929 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
2936 unsigned short mode = MPOL_DEFAULT; in mpol_to_str() local
2939 if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) { in mpol_to_str()
2940 mode = pol->mode; in mpol_to_str()
2941 flags = pol->flags; in mpol_to_str()
2944 switch (mode) { in mpol_to_str()
2952 nodes = pol->nodes; in mpol_to_str()
2960 p += snprintf(p, maxlen, "%s", policy_modes[mode]); in mpol_to_str()
2963 p += snprintf(p, buffer + maxlen - p, "="); in mpol_to_str()
2969 p += snprintf(p, buffer + maxlen - p, "static"); in mpol_to_str()
2971 p += snprintf(p, buffer + maxlen - p, "relative"); in mpol_to_str()
2975 p += scnprintf(p, buffer + maxlen - p, ":%*pbl", in mpol_to_str()
2998 return -EINVAL; in numa_demotion_enabled_store()
3024 return -ENOMEM; in numa_init_sysfs()