15  * interleave     Allocate memory interleaved over a set of nodes,
22  * bind           Only allocate memory on a specific set of nodes,
26  *                the allocation to memory nodes instead
150 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
151 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
167 static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_interleave()  argument
169 	if (nodes_empty(*nodes))  in mpol_new_interleave()
171 	pol->v.nodes = *nodes;  in mpol_new_interleave()
175 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_preferred()  argument
177 	if (!nodes)  in mpol_new_preferred()
179 	else if (nodes_empty(*nodes))  in mpol_new_preferred()
180 		return -EINVAL;			/*  no allowed nodes */  in mpol_new_preferred()
182 		pol->v.preferred_node = first_node(*nodes);  in mpol_new_preferred()
186 static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_bind()  argument
188 	if (nodes_empty(*nodes))  in mpol_new_bind()
190 	pol->v.nodes = *nodes;  in mpol_new_bind()
196  * any, for the new policy.  mpol_new() has already validated the nodes
204 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)  in mpol_set_nodemask()  argument
215 	VM_BUG_ON(!nodes);  in mpol_set_nodemask()
216 	if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))  in mpol_set_nodemask()
217 		nodes = NULL;	/* explicit local allocation */  in mpol_set_nodemask()
220 			mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);  in mpol_set_nodemask()
222 			nodes_and(nsc->mask2, *nodes, nsc->mask1);  in mpol_set_nodemask()
225 			pol->w.user_nodemask = *nodes;  in mpol_set_nodemask()
231 	if (nodes)  in mpol_set_nodemask()
240  * initialization. You must invoke mpol_set_nodemask() to set nodes.
243 				  nodemask_t *nodes)  in mpol_new()  argument
247 	pr_debug("setting mode %d flags %d nodes[0] %lx\n",  in mpol_new()
248 		 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);  in mpol_new()
251 		if (nodes && !nodes_empty(*nodes))  in mpol_new()
255 	VM_BUG_ON(!nodes);  in mpol_new()
263 		if (nodes_empty(*nodes)) {  in mpol_new()
269 		if (!nodes_empty(*nodes) ||  in mpol_new()
274 	} else if (nodes_empty(*nodes))  in mpol_new()
294 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_default()  argument
298 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_nodemask()  argument
303 		nodes_and(tmp, pol->w.user_nodemask, *nodes);  in mpol_rebind_nodemask()
305 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_nodemask()
307 		nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed,  in mpol_rebind_nodemask()
308 								*nodes);  in mpol_rebind_nodemask()
309 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_nodemask()
313 		tmp = *nodes;  in mpol_rebind_nodemask()
315 	pol->v.nodes = tmp;  in mpol_rebind_nodemask()
319 						const nodemask_t *nodes)  in mpol_rebind_preferred()  argument
326 		if (node_isset(node, *nodes)) {  in mpol_rebind_preferred()
332 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_preferred()
337 						   *nodes);  in mpol_rebind_preferred()
338 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_preferred()
343  * mpol_rebind_policy - Migrate a policy to a different set of nodes
667  * If pages found in a given range are on a set of nodes (determined by
668  * @nodes and @flags,) it's isolated and queued to the pagelist which is
681 		nodemask_t *nodes, unsigned long flags,  in queue_pages_range()  argument
687 		.nmask = nodes,  in queue_pages_range()
793 			     nodemask_t *nodes)  in do_set_mempolicy()  argument
802 	new = mpol_new(mode, flags, nodes);  in do_set_mempolicy()
809 	ret = mpol_set_nodemask(new, nodes, scratch);  in do_set_mempolicy()
832 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)  in get_policy_nodemask()  argument
834 	nodes_clear(*nodes);  in get_policy_nodemask()
842 		*nodes = p->v.nodes;  in get_policy_nodemask()
846 			node_set(p->v.preferred_node, *nodes);  in get_policy_nodemask()
932 			*policy = next_node_in(current->il_prev, pol->v.nodes);  in do_get_mempolicy()
1087 	 * This lets us pick a pair of nodes to migrate between, such that  in do_migrate_pages()
1116 			 * However if the number of source nodes is not equal to  in do_migrate_pages()
1117 			 * the number of destination nodes we can not preserve  in do_migrate_pages()
1137 			/* dest not in remaining from nodes? */  in do_migrate_pages()
1261 	pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",  in do_mbind()
1327 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,  in get_nodes()  argument
1336 	nodes_clear(*nodes);  in get_nodes()
1349 	 * When the user specified more nodes than supported just check  in get_nodes()
1381 	if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))  in get_nodes()
1383 	nodes_addr(*nodes)[nlongs-1] &= endmask;  in get_nodes()  local
1389 			      nodemask_t *nodes)  in copy_nodes_to_user()  argument
1401 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;  in copy_nodes_to_user()
1408 	nodemask_t nodes;  in kernel_mbind()  local
1420 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_mbind()
1423 	return do_mbind(start, len, mode, mode_flags, &nodes, flags);  in kernel_mbind()
1438 	nodemask_t nodes;  in kernel_set_mempolicy()  local
1447 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_set_mempolicy()
1450 	return do_set_mempolicy(mode, flags, &nodes);  in kernel_set_mempolicy()
1509 	/* Is the user allowed to access the target nodes? */  in kernel_migrate_pages()
1564 	nodemask_t nodes;  in kernel_get_mempolicy()  local
1571 	err = do_get_mempolicy(&pval, &nodes, addr, flags);  in kernel_get_mempolicy()
1580 		err = copy_nodes_to_user(nmask, maxnode, &nodes);  in kernel_get_mempolicy()
1779 	 * if policy->v.nodes has movable memory only,  in apply_policy_zone()
1782 	 * policy->v.nodes is intersect with node_states[N_MEMORY].  in apply_policy_zone()
1784 	 * policy->v.nodes has movable memory only.  in apply_policy_zone()
1786 	if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY]))  in apply_policy_zone()
1793  * Return a nodemask representing a mempolicy for filtering nodes for
1801 			cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))  in policy_nodemask()
1802 		return &policy->v.nodes;  in policy_nodemask()
1831 	next = next_node_in(me->il_prev, policy->v.nodes);  in interleave_nodes()
1874 							&policy->v.nodes);  in mempolicy_slab_node()
1885  * node in pol->v.nodes (starting from n=0), wrapping around if n exceeds the
1886  * number of present nodes.
1890 	unsigned nnodes = nodes_weight(pol->v.nodes);  in offset_il_node()
1898 	nid = first_node(pol->v.nodes);  in offset_il_node()
1900 		nid = next_node(nid, pol->v.nodes);  in offset_il_node()
1956 			*nodemask = &(*mpol)->v.nodes;  in huge_node()
1999 		*mask =  mempolicy->v.nodes;  in init_nodemask_of_mempolicy()
2037 		 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to  in mempolicy_nodemask_intersects()
2038 		 * allocate from, they may fallback to other nodes when oom.  in mempolicy_nodemask_intersects()
2040 		 * nodes in mask.  in mempolicy_nodemask_intersects()
2045 		ret = nodes_intersects(mempolicy->v.nodes, *mask);  in mempolicy_nodemask_intersects()
2124 		 * node and don't fall back to other nodes, as the cost of  in alloc_pages_vma()
2262 		return !!nodes_equal(a->v.nodes, b->v.nodes);  in __mpol_equal()
2417 		 * allows binding to multiple nodes.  in mpol_misplaced()
2420 		 * If no allowed nodes, use current [!misplaced].  in mpol_misplaced()
2422 		if (node_isset(curnid, pol->v.nodes))  in mpol_misplaced()
2427 				&pol->v.nodes);  in mpol_misplaced()
2630 		 npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE);  in mpol_set_shared_policy()
2734 	 * enabled across suitably sized nodes (default is >= 16MB), or  in numa_policy_init()
2800 	nodemask_t nodes;  in mpol_parse_str()  local
2808 		if (nodelist_parse(nodelist, nodes))  in mpol_parse_str()
2810 		if (!nodes_subset(nodes, node_states[N_MEMORY]))  in mpol_parse_str()
2813 		nodes_clear(nodes);  in mpol_parse_str()
2837 		 * Default to online nodes with memory if no nodelist  in mpol_parse_str()
2840 			nodes = node_states[N_MEMORY];  in mpol_parse_str()
2879 	new = mpol_new(mode, mode_flags, &nodes);  in mpol_parse_str()
2884 	 * Save nodes for mpol_to_str() to show the tmpfs mount options  in mpol_parse_str()
2888 		new->v.nodes = nodes;  in mpol_parse_str()
2890 		new->v.preferred_node = first_node(nodes);  in mpol_parse_str()
2895 	 * Save nodes for contextualization: this will be used to "clone"  in mpol_parse_str()
2898 	new->w.user_nodemask = nodes;  in mpol_parse_str()
2927 	nodemask_t nodes = NODE_MASK_NONE;  in mpol_to_str()  local
2943 			node_set(pol->v.preferred_node, nodes);  in mpol_to_str()
2947 		nodes = pol->v.nodes;  in mpol_to_str()
2969 	if (!nodes_empty(nodes))  in mpol_to_str()
2971 			       nodemask_pr_args(&nodes));  in mpol_to_str()