158 	struct mempolicy *pol = p->mempolicy;  in get_task_policy()  local
161 	if (pol)  in get_task_policy()
162 		return pol;  in get_task_policy()
166 		pol = &preferred_node_policy[node];  in get_task_policy()
168 		if (pol->mode)  in get_task_policy()
169 			return pol;  in get_task_policy()
176 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
177 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
180 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)  in mpol_store_user_nodemask()  argument
182 	return pol->flags & MPOL_MODE_FLAGS;  in mpol_store_user_nodemask()
193 static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_interleave()  argument
197 	pol->v.nodes = *nodes;  in mpol_new_interleave()
201 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_preferred()  argument
204 		pol->flags |= MPOL_F_LOCAL;	/* local allocation */  in mpol_new_preferred()
208 		pol->v.preferred_node = first_node(*nodes);  in mpol_new_preferred()
212 static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_bind()  argument
216 	pol->v.nodes = *nodes;  in mpol_new_bind()
229 static int mpol_set_nodemask(struct mempolicy *pol,  in mpol_set_nodemask()  argument
235 	if (pol == NULL)  in mpol_set_nodemask()
242 	if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))  in mpol_set_nodemask()
245 		if (pol->flags & MPOL_F_RELATIVE_NODES)  in mpol_set_nodemask()
250 		if (mpol_store_user_nodemask(pol))  in mpol_set_nodemask()
251 			pol->w.user_nodemask = *nodes;  in mpol_set_nodemask()
253 			pol->w.cpuset_mems_allowed =  in mpol_set_nodemask()
258 		ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);  in mpol_set_nodemask()
260 		ret = mpol_ops[pol->mode].create(pol, NULL);  in mpol_set_nodemask()
320 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_default()  argument
324 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_nodemask()  argument
328 	if (pol->flags & MPOL_F_STATIC_NODES)  in mpol_rebind_nodemask()
329 		nodes_and(tmp, pol->w.user_nodemask, *nodes);  in mpol_rebind_nodemask()
330 	else if (pol->flags & MPOL_F_RELATIVE_NODES)  in mpol_rebind_nodemask()
331 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_nodemask()
333 		nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed,  in mpol_rebind_nodemask()
335 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_nodemask()
341 	pol->v.nodes = tmp;  in mpol_rebind_nodemask()
344 static void mpol_rebind_preferred(struct mempolicy *pol,  in mpol_rebind_preferred()  argument
349 	if (pol->flags & MPOL_F_STATIC_NODES) {  in mpol_rebind_preferred()
350 		int node = first_node(pol->w.user_nodemask);  in mpol_rebind_preferred()
353 			pol->v.preferred_node = node;  in mpol_rebind_preferred()
354 			pol->flags &= ~MPOL_F_LOCAL;  in mpol_rebind_preferred()
356 			pol->flags |= MPOL_F_LOCAL;  in mpol_rebind_preferred()
357 	} else if (pol->flags & MPOL_F_RELATIVE_NODES) {  in mpol_rebind_preferred()
358 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_preferred()
359 		pol->v.preferred_node = first_node(tmp);  in mpol_rebind_preferred()
360 	} else if (!(pol->flags & MPOL_F_LOCAL)) {  in mpol_rebind_preferred()
361 		pol->v.preferred_node = node_remap(pol->v.preferred_node,  in mpol_rebind_preferred()
362 						   pol->w.cpuset_mems_allowed,  in mpol_rebind_preferred()
364 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_preferred()
375 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)  in mpol_rebind_policy()  argument
377 	if (!pol)  in mpol_rebind_policy()
379 	if (!mpol_store_user_nodemask(pol) && !(pol->flags & MPOL_F_LOCAL) &&  in mpol_rebind_policy()
380 	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))  in mpol_rebind_policy()
383 	mpol_ops[pol->mode].rebind(pol, newmask);  in mpol_rebind_policy()
770 						struct mempolicy *pol)  in vma_replace_policy()  argument
781 	new = mpol_dup(pol);  in vma_replace_policy()
945 	struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;  in do_get_mempolicy()  local
974 			pol = vma->vm_ops->get_policy(vma, addr);  in do_get_mempolicy()
976 			pol = vma->vm_policy;  in do_get_mempolicy()
980 	if (!pol)  in do_get_mempolicy()
981 		pol = &default_policy;	/* indicates default behavior */  in do_get_mempolicy()
991 			pol_refcount = pol;  in do_get_mempolicy()
993 			mpol_get(pol);  in do_get_mempolicy()
998 		} else if (pol == current->mempolicy &&  in do_get_mempolicy()
999 				pol->mode == MPOL_INTERLEAVE) {  in do_get_mempolicy()
1000 			*policy = next_node_in(current->il_prev, pol->v.nodes);  in do_get_mempolicy()
1006 		*policy = pol == &default_policy ? MPOL_DEFAULT :  in do_get_mempolicy()
1007 						pol->mode;  in do_get_mempolicy()
1012 		*policy |= (pol->flags & MPOL_MODE_FLAGS);  in do_get_mempolicy()
1017 		if (mpol_store_user_nodemask(pol)) {  in do_get_mempolicy()
1018 			*nmask = pol->w.user_nodemask;  in do_get_mempolicy()
1021 			get_policy_nodemask(pol, nmask);  in do_get_mempolicy()
1027 	mpol_cond_put(pol);  in do_get_mempolicy()
1782 	struct mempolicy *pol = NULL;  in __get_vma_policy()  local
1786 			pol = vma->vm_ops->get_policy(vma, addr);  in __get_vma_policy()
1788 			pol = vma->vm_policy;  in __get_vma_policy()
1796 			if (mpol_needs_cond_ref(pol))  in __get_vma_policy()
1797 				mpol_get(pol);  in __get_vma_policy()
1801 	return pol;  in __get_vma_policy()
1819 	struct mempolicy *pol = __get_vma_policy(vma, addr);  in get_vma_policy()  local
1821 	if (!pol)  in get_vma_policy()
1822 		pol = get_task_policy(current);  in get_vma_policy()
1824 	return pol;  in get_vma_policy()
1829 	struct mempolicy *pol;  in vma_policy_mof()  local
1834 		pol = vma->vm_ops->get_policy(vma, vma->vm_start);  in vma_policy_mof()
1835 		if (pol && (pol->flags & MPOL_F_MOF))  in vma_policy_mof()
1837 		mpol_cond_put(pol);  in vma_policy_mof()
1842 	pol = vma->vm_policy;  in vma_policy_mof()
1843 	if (!pol)  in vma_policy_mof()
1844 		pol = get_task_policy(current);  in vma_policy_mof()
1846 	return pol->flags & MPOL_F_MOF;  in vma_policy_mof()
1964 static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)  in offset_il_node()  argument
1966 	unsigned nnodes = nodes_weight(pol->v.nodes);  in offset_il_node()
1974 	nid = first_node(pol->v.nodes);  in offset_il_node()
1976 		nid = next_node(nid, pol->v.nodes);  in offset_il_node()
1981 static inline unsigned interleave_nid(struct mempolicy *pol,  in interleave_nid()  argument
1997 		return offset_il_node(pol, off);  in interleave_nid()
1999 		return interleave_nodes(pol);  in interleave_nid()
2176 	struct mempolicy *pol;  in alloc_pages_vma()  local
2181 	pol = get_vma_policy(vma, addr);  in alloc_pages_vma()
2183 	if (pol->mode == MPOL_INTERLEAVE) {  in alloc_pages_vma()
2186 		nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);  in alloc_pages_vma()
2187 		mpol_cond_put(pol);  in alloc_pages_vma()
2205 		if (pol->mode == MPOL_PREFERRED && !(pol->flags & MPOL_F_LOCAL))  in alloc_pages_vma()
2206 			hpage_node = pol->v.preferred_node;  in alloc_pages_vma()
2208 		nmask = policy_nodemask(gfp, pol);  in alloc_pages_vma()
2210 			mpol_cond_put(pol);  in alloc_pages_vma()
2232 	nmask = policy_nodemask(gfp, pol);  in alloc_pages_vma()
2233 	preferred_nid = policy_node(gfp, pol, node);  in alloc_pages_vma()
2235 	mpol_cond_put(pol);  in alloc_pages_vma()
2258 	struct mempolicy *pol = &default_policy;  in alloc_pages_current()  local
2262 		pol = get_task_policy(current);  in alloc_pages_current()
2268 	if (pol->mode == MPOL_INTERLEAVE)  in alloc_pages_current()
2269 		page = alloc_page_interleave(gfp, order, interleave_nodes(pol));  in alloc_pages_current()
2272 				policy_node(gfp, pol, numa_node_id()),  in alloc_pages_current()
2273 				policy_nodemask(gfp, pol));  in alloc_pages_current()
2281 	struct mempolicy *pol = mpol_dup(vma_policy(src));  in vma_dup_policy()  local
2283 	if (IS_ERR(pol))  in vma_dup_policy()
2284 		return PTR_ERR(pol);  in vma_dup_policy()
2285 	dst->vm_policy = pol;  in vma_dup_policy()
2425 	struct mempolicy *pol = NULL;  in mpol_shared_policy_lookup()  local
2434 		pol = sn->policy;  in mpol_shared_policy_lookup()
2437 	return pol;  in mpol_shared_policy_lookup()
2465 	struct mempolicy *pol;  in mpol_misplaced()  local
2474 	pol = get_vma_policy(vma, addr);  in mpol_misplaced()
2475 	if (!(pol->flags & MPOL_F_MOF))  in mpol_misplaced()
2478 	switch (pol->mode) {  in mpol_misplaced()
2482 		polnid = offset_il_node(pol, pgoff);  in mpol_misplaced()
2486 		if (pol->flags & MPOL_F_LOCAL)  in mpol_misplaced()
2489 			polnid = pol->v.preferred_node;  in mpol_misplaced()
2500 		if (node_isset(curnid, pol->v.nodes))  in mpol_misplaced()
2505 				&pol->v.nodes);  in mpol_misplaced()
2514 	if (pol->flags & MPOL_F_MORON) {  in mpol_misplaced()
2524 	mpol_cond_put(pol);  in mpol_misplaced()
2537 	struct mempolicy *pol;  in mpol_put_task_policy()  local
2540 	pol = task->mempolicy;  in mpol_put_task_policy()
2543 	mpol_put(pol);  in mpol_put_task_policy()
2554 			unsigned long end, struct mempolicy *pol)  in sp_node_init()  argument
2558 	node->policy = pol;  in sp_node_init()
2562 				struct mempolicy *pol)  in sp_alloc()  argument
2571 	newpol = mpol_dup(pol);  in sp_alloc()
3006 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)  in mpol_to_str()  argument
3013 	if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {  in mpol_to_str()
3014 		mode = pol->mode;  in mpol_to_str()
3015 		flags = pol->flags;  in mpol_to_str()
3025 			node_set(pol->v.preferred_node, nodes);  in mpol_to_str()
3029 		nodes = pol->v.nodes;  in mpol_to_str()