132 	struct mempolicy *pol = p->mempolicy;  in get_task_policy()  local
135 	if (pol)  in get_task_policy()
136 		return pol;  in get_task_policy()
140 		pol = &preferred_node_policy[node];  in get_task_policy()
142 		if (pol->mode)  in get_task_policy()
143 			return pol;  in get_task_policy()
150 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
151 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
154 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)  in mpol_store_user_nodemask()  argument
156 	return pol->flags & MPOL_MODE_FLAGS;  in mpol_store_user_nodemask()
167 static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_interleave()  argument
171 	pol->v.nodes = *nodes;  in mpol_new_interleave()
175 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_preferred()  argument
178 		pol->flags |= MPOL_F_LOCAL;	/* local allocation */  in mpol_new_preferred()
182 		pol->v.preferred_node = first_node(*nodes);  in mpol_new_preferred()
186 static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_bind()  argument
190 	pol->v.nodes = *nodes;  in mpol_new_bind()
203 static int mpol_set_nodemask(struct mempolicy *pol,  in mpol_set_nodemask()  argument
209 	if (pol == NULL)  in mpol_set_nodemask()
216 	if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))  in mpol_set_nodemask()
219 		if (pol->flags & MPOL_F_RELATIVE_NODES)  in mpol_set_nodemask()
224 		if (mpol_store_user_nodemask(pol))  in mpol_set_nodemask()
225 			pol->w.user_nodemask = *nodes;  in mpol_set_nodemask()
227 			pol->w.cpuset_mems_allowed =  in mpol_set_nodemask()
232 		ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);  in mpol_set_nodemask()
234 		ret = mpol_ops[pol->mode].create(pol, NULL);  in mpol_set_nodemask()
294 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_default()  argument
298 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_nodemask()  argument
302 	if (pol->flags & MPOL_F_STATIC_NODES)  in mpol_rebind_nodemask()
303 		nodes_and(tmp, pol->w.user_nodemask, *nodes);  in mpol_rebind_nodemask()
304 	else if (pol->flags & MPOL_F_RELATIVE_NODES)  in mpol_rebind_nodemask()
305 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_nodemask()
307 		nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed,  in mpol_rebind_nodemask()
309 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_nodemask()
315 	pol->v.nodes = tmp;  in mpol_rebind_nodemask()
318 static void mpol_rebind_preferred(struct mempolicy *pol,  in mpol_rebind_preferred()  argument
323 	if (pol->flags & MPOL_F_STATIC_NODES) {  in mpol_rebind_preferred()
324 		int node = first_node(pol->w.user_nodemask);  in mpol_rebind_preferred()
327 			pol->v.preferred_node = node;  in mpol_rebind_preferred()
328 			pol->flags &= ~MPOL_F_LOCAL;  in mpol_rebind_preferred()
330 			pol->flags |= MPOL_F_LOCAL;  in mpol_rebind_preferred()
331 	} else if (pol->flags & MPOL_F_RELATIVE_NODES) {  in mpol_rebind_preferred()
332 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_preferred()
333 		pol->v.preferred_node = first_node(tmp);  in mpol_rebind_preferred()
334 	} else if (!(pol->flags & MPOL_F_LOCAL)) {  in mpol_rebind_preferred()
335 		pol->v.preferred_node = node_remap(pol->v.preferred_node,  in mpol_rebind_preferred()
336 						   pol->w.cpuset_mems_allowed,  in mpol_rebind_preferred()
338 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_preferred()
349 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)  in mpol_rebind_policy()  argument
351 	if (!pol)  in mpol_rebind_policy()
353 	if (!mpol_store_user_nodemask(pol) && !(pol->flags & MPOL_F_LOCAL) &&  in mpol_rebind_policy()
354 	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))  in mpol_rebind_policy()
357 	mpol_ops[pol->mode].rebind(pol, newmask);  in mpol_rebind_policy()
699 						struct mempolicy *pol)  in vma_replace_policy()  argument
710 	new = mpol_dup(pol);  in vma_replace_policy()
877 	struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;  in do_get_mempolicy()  local
906 			pol = vma->vm_ops->get_policy(vma, addr);  in do_get_mempolicy()
908 			pol = vma->vm_policy;  in do_get_mempolicy()
912 	if (!pol)  in do_get_mempolicy()
913 		pol = &default_policy;	/* indicates default behavior */  in do_get_mempolicy()
923 			pol_refcount = pol;  in do_get_mempolicy()
925 			mpol_get(pol);  in do_get_mempolicy()
930 		} else if (pol == current->mempolicy &&  in do_get_mempolicy()
931 				pol->mode == MPOL_INTERLEAVE) {  in do_get_mempolicy()
932 			*policy = next_node_in(current->il_prev, pol->v.nodes);  in do_get_mempolicy()
938 		*policy = pol == &default_policy ? MPOL_DEFAULT :  in do_get_mempolicy()
939 						pol->mode;  in do_get_mempolicy()
944 		*policy |= (pol->flags & MPOL_MODE_FLAGS);  in do_get_mempolicy()
949 		if (mpol_store_user_nodemask(pol)) {  in do_get_mempolicy()
950 			*nmask = pol->w.user_nodemask;  in do_get_mempolicy()
953 			get_policy_nodemask(pol, nmask);  in do_get_mempolicy()
959 	mpol_cond_put(pol);  in do_get_mempolicy()
1705 	struct mempolicy *pol = NULL;  in __get_vma_policy()  local
1709 			pol = vma->vm_ops->get_policy(vma, addr);  in __get_vma_policy()
1711 			pol = vma->vm_policy;  in __get_vma_policy()
1719 			if (mpol_needs_cond_ref(pol))  in __get_vma_policy()
1720 				mpol_get(pol);  in __get_vma_policy()
1724 	return pol;  in __get_vma_policy()
1742 	struct mempolicy *pol = __get_vma_policy(vma, addr);  in get_vma_policy()  local
1744 	if (!pol)  in get_vma_policy()
1745 		pol = get_task_policy(current);  in get_vma_policy()
1747 	return pol;  in get_vma_policy()
1752 	struct mempolicy *pol;  in vma_policy_mof()  local
1757 		pol = vma->vm_ops->get_policy(vma, vma->vm_start);  in vma_policy_mof()
1758 		if (pol && (pol->flags & MPOL_F_MOF))  in vma_policy_mof()
1760 		mpol_cond_put(pol);  in vma_policy_mof()
1765 	pol = vma->vm_policy;  in vma_policy_mof()
1766 	if (!pol)  in vma_policy_mof()
1767 		pol = get_task_policy(current);  in vma_policy_mof()
1769 	return pol->flags & MPOL_F_MOF;  in vma_policy_mof()
1888 static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)  in offset_il_node()  argument
1890 	unsigned nnodes = nodes_weight(pol->v.nodes);  in offset_il_node()
1898 	nid = first_node(pol->v.nodes);  in offset_il_node()
1900 		nid = next_node(nid, pol->v.nodes);  in offset_il_node()
1905 static inline unsigned interleave_nid(struct mempolicy *pol,  in interleave_nid()  argument
1921 		return offset_il_node(pol, off);  in interleave_nid()
1923 		return interleave_nodes(pol);  in interleave_nid()
2101 	struct mempolicy *pol;  in alloc_pages_vma()  local
2106 	pol = get_vma_policy(vma, addr);  in alloc_pages_vma()
2108 	if (pol->mode == MPOL_INTERLEAVE) {  in alloc_pages_vma()
2111 		nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);  in alloc_pages_vma()
2112 		mpol_cond_put(pol);  in alloc_pages_vma()
2130 		if (pol->mode == MPOL_PREFERRED && !(pol->flags & MPOL_F_LOCAL))  in alloc_pages_vma()
2131 			hpage_node = pol->v.preferred_node;  in alloc_pages_vma()
2133 		nmask = policy_nodemask(gfp, pol);  in alloc_pages_vma()
2135 			mpol_cond_put(pol);  in alloc_pages_vma()
2153 	nmask = policy_nodemask(gfp, pol);  in alloc_pages_vma()
2154 	preferred_nid = policy_node(gfp, pol, node);  in alloc_pages_vma()
2156 	mpol_cond_put(pol);  in alloc_pages_vma()
2179 	struct mempolicy *pol = &default_policy;  in alloc_pages_current()  local
2183 		pol = get_task_policy(current);  in alloc_pages_current()
2189 	if (pol->mode == MPOL_INTERLEAVE)  in alloc_pages_current()
2190 		page = alloc_page_interleave(gfp, order, interleave_nodes(pol));  in alloc_pages_current()
2193 				policy_node(gfp, pol, numa_node_id()),  in alloc_pages_current()
2194 				policy_nodemask(gfp, pol));  in alloc_pages_current()
2202 	struct mempolicy *pol = mpol_dup(vma_policy(src));  in vma_dup_policy()  local
2204 	if (IS_ERR(pol))  in vma_dup_policy()
2205 		return PTR_ERR(pol);  in vma_dup_policy()
2206 	dst->vm_policy = pol;  in vma_dup_policy()
2347 	struct mempolicy *pol = NULL;  in mpol_shared_policy_lookup()  local
2356 		pol = sn->policy;  in mpol_shared_policy_lookup()
2359 	return pol;  in mpol_shared_policy_lookup()
2387 	struct mempolicy *pol;  in mpol_misplaced()  local
2396 	pol = get_vma_policy(vma, addr);  in mpol_misplaced()
2397 	if (!(pol->flags & MPOL_F_MOF))  in mpol_misplaced()
2400 	switch (pol->mode) {  in mpol_misplaced()
2404 		polnid = offset_il_node(pol, pgoff);  in mpol_misplaced()
2408 		if (pol->flags & MPOL_F_LOCAL)  in mpol_misplaced()
2411 			polnid = pol->v.preferred_node;  in mpol_misplaced()
2422 		if (node_isset(curnid, pol->v.nodes))  in mpol_misplaced()
2427 				&pol->v.nodes);  in mpol_misplaced()
2436 	if (pol->flags & MPOL_F_MORON) {  in mpol_misplaced()
2446 	mpol_cond_put(pol);  in mpol_misplaced()
2459 	struct mempolicy *pol;  in mpol_put_task_policy()  local
2462 	pol = task->mempolicy;  in mpol_put_task_policy()
2465 	mpol_put(pol);  in mpol_put_task_policy()
2476 			unsigned long end, struct mempolicy *pol)  in sp_node_init()  argument
2480 	node->policy = pol;  in sp_node_init()
2484 				struct mempolicy *pol)  in sp_alloc()  argument
2493 	newpol = mpol_dup(pol);  in sp_alloc()
2924 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)  in mpol_to_str()  argument
2931 	if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {  in mpol_to_str()
2932 		mode = pol->mode;  in mpol_to_str()
2933 		flags = pol->flags;  in mpol_to_str()
2943 			node_set(pol->v.preferred_node, nodes);  in mpol_to_str()
2947 		nodes = pol->v.nodes;  in mpol_to_str()