blob: ade57322fa2a9155b01108c96b3cc44b97751a32 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
Linus Torvalds1da177e2005-04-16 15:20:36 -070066*/
67
68#include <linux/mempolicy.h>
69#include <linux/mm.h>
70#include <linux/highmem.h>
71#include <linux/hugetlb.h>
72#include <linux/kernel.h>
73#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <linux/nodemask.h>
75#include <linux/cpuset.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/slab.h>
77#include <linux/string.h>
78#include <linux/module.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070079#include <linux/nsproxy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070080#include <linux/interrupt.h>
81#include <linux/init.h>
82#include <linux/compat.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080083#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080084#include <linux/seq_file.h>
85#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080086#include <linux/migrate.h>
Hugh Dickins62b61f62009-12-14 17:59:33 -080087#include <linux/ksm.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070088#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070089#include <linux/security.h>
Adrian Bunkdbcb0f12007-10-16 01:26:26 -070090#include <linux/syscalls.h>
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -070091#include <linux/ctype.h>
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -080092#include <linux/mm_inline.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080093
Linus Torvalds1da177e2005-04-16 15:20:36 -070094#include <asm/tlbflush.h>
95#include <asm/uaccess.h>
96
Nick Piggin62695a82008-10-18 20:26:09 -070097#include "internal.h"
98
Christoph Lameter38e35862006-01-08 01:01:01 -080099/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800100#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -0800101#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800102#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800103
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800104static struct kmem_cache *policy_cache;
105static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107/* Highest zone. An specific allocation for a zone below that is not
108 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800109enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700111/*
112 * run-time system-wide default policy => local allocation
113 */
Andi Kleend42c6992005-07-06 19:56:03 +0200114struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 .refcnt = ATOMIC_INIT(1), /* never free it */
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700116 .mode = MPOL_PREFERRED,
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700117 .flags = MPOL_F_LOCAL,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118};
119
David Rientjes37012942008-04-28 02:12:33 -0700120static const struct mempolicy_operations {
121 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
122 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
123} mpol_ops[MPOL_MAX];
124
Mel Gorman19770b32008-04-28 02:12:18 -0700125/* Check that the nodemask contains at least one populated zone */
David Rientjes37012942008-04-28 02:12:33 -0700126static int is_valid_nodemask(const nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127{
Mel Gorman19770b32008-04-28 02:12:18 -0700128 int nd, k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
Mel Gorman19770b32008-04-28 02:12:18 -0700130 for_each_node_mask(nd, *nodemask) {
131 struct zone *z;
132
133 for (k = 0; k <= policy_zone; k++) {
134 z = &NODE_DATA(nd)->node_zones[k];
135 if (z->present_pages > 0)
136 return 1;
Andi Kleendd942ae2006-02-17 01:39:16 +0100137 }
138 }
Mel Gorman19770b32008-04-28 02:12:18 -0700139
140 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141}
142
David Rientjesf5b087b2008-04-28 02:12:27 -0700143static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
144{
Bob Liu6d556292010-05-24 14:31:59 -0700145 return pol->flags & MPOL_MODE_FLAGS;
David Rientjes4c50bc02008-04-28 02:12:30 -0700146}
147
148static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
149 const nodemask_t *rel)
150{
151 nodemask_t tmp;
152 nodes_fold(tmp, *orig, nodes_weight(*rel));
153 nodes_onto(*ret, tmp, *rel);
David Rientjesf5b087b2008-04-28 02:12:27 -0700154}
155
David Rientjes37012942008-04-28 02:12:33 -0700156static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
157{
158 if (nodes_empty(*nodes))
159 return -EINVAL;
160 pol->v.nodes = *nodes;
161 return 0;
162}
163
164static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
165{
166 if (!nodes)
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700167 pol->flags |= MPOL_F_LOCAL; /* local allocation */
David Rientjes37012942008-04-28 02:12:33 -0700168 else if (nodes_empty(*nodes))
169 return -EINVAL; /* no allowed nodes */
170 else
171 pol->v.preferred_node = first_node(*nodes);
172 return 0;
173}
174
175static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
176{
177 if (!is_valid_nodemask(nodes))
178 return -EINVAL;
179 pol->v.nodes = *nodes;
180 return 0;
181}
182
Miao Xie58568d22009-06-16 15:31:49 -0700183/*
184 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
185 * any, for the new policy. mpol_new() has already validated the nodes
186 * parameter with respect to the policy mode and flags. But, we need to
187 * handle an empty nodemask with MPOL_PREFERRED here.
188 *
189 * Must be called holding task's alloc_lock to protect task's mems_allowed
190 * and mempolicy. May also be called holding the mmap_semaphore for write.
191 */
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700192static int mpol_set_nodemask(struct mempolicy *pol,
193 const nodemask_t *nodes, struct nodemask_scratch *nsc)
Miao Xie58568d22009-06-16 15:31:49 -0700194{
Miao Xie58568d22009-06-16 15:31:49 -0700195 int ret;
196
197 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
198 if (pol == NULL)
199 return 0;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700200 /* Check N_HIGH_MEMORY */
201 nodes_and(nsc->mask1,
202 cpuset_current_mems_allowed, node_states[N_HIGH_MEMORY]);
Miao Xie58568d22009-06-16 15:31:49 -0700203
204 VM_BUG_ON(!nodes);
205 if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
206 nodes = NULL; /* explicit local allocation */
207 else {
208 if (pol->flags & MPOL_F_RELATIVE_NODES)
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700209 mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1);
Miao Xie58568d22009-06-16 15:31:49 -0700210 else
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700211 nodes_and(nsc->mask2, *nodes, nsc->mask1);
212
Miao Xie58568d22009-06-16 15:31:49 -0700213 if (mpol_store_user_nodemask(pol))
214 pol->w.user_nodemask = *nodes;
215 else
216 pol->w.cpuset_mems_allowed =
217 cpuset_current_mems_allowed;
218 }
219
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700220 if (nodes)
221 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
222 else
223 ret = mpol_ops[pol->mode].create(pol, NULL);
Miao Xie58568d22009-06-16 15:31:49 -0700224 return ret;
225}
226
227/*
228 * This function just creates a new policy, does some check and simple
229 * initialization. You must invoke mpol_set_nodemask() to set nodes.
230 */
David Rientjes028fec42008-04-28 02:12:25 -0700231static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
232 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233{
234 struct mempolicy *policy;
235
David Rientjes028fec42008-04-28 02:12:25 -0700236 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
237 mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
Paul Mundt140d5a42007-07-15 23:38:16 -0700238
David Rientjes3e1f0642008-04-28 02:12:34 -0700239 if (mode == MPOL_DEFAULT) {
240 if (nodes && !nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700241 return ERR_PTR(-EINVAL);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700242 return NULL; /* simply delete any existing policy */
David Rientjes37012942008-04-28 02:12:33 -0700243 }
David Rientjes3e1f0642008-04-28 02:12:34 -0700244 VM_BUG_ON(!nodes);
245
246 /*
247 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
248 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
249 * All other modes require a valid pointer to a non-empty nodemask.
250 */
251 if (mode == MPOL_PREFERRED) {
252 if (nodes_empty(*nodes)) {
253 if (((flags & MPOL_F_STATIC_NODES) ||
254 (flags & MPOL_F_RELATIVE_NODES)))
255 return ERR_PTR(-EINVAL);
David Rientjes3e1f0642008-04-28 02:12:34 -0700256 }
257 } else if (nodes_empty(*nodes))
258 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
260 if (!policy)
261 return ERR_PTR(-ENOMEM);
262 atomic_set(&policy->refcnt, 1);
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700263 policy->mode = mode;
David Rientjes3e1f0642008-04-28 02:12:34 -0700264 policy->flags = flags;
David Rientjesf5b087b2008-04-28 02:12:27 -0700265
David Rientjes37012942008-04-28 02:12:33 -0700266 return policy;
267}
268
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700269/* Slow path of a mpol destructor. */
270void __mpol_put(struct mempolicy *p)
271{
272 if (!atomic_dec_and_test(&p->refcnt))
273 return;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700274 kmem_cache_free(policy_cache, p);
275}
276
David Rientjes37012942008-04-28 02:12:33 -0700277static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
278{
279}
280
281static void mpol_rebind_nodemask(struct mempolicy *pol,
282 const nodemask_t *nodes)
283{
284 nodemask_t tmp;
285
286 if (pol->flags & MPOL_F_STATIC_NODES)
287 nodes_and(tmp, pol->w.user_nodemask, *nodes);
288 else if (pol->flags & MPOL_F_RELATIVE_NODES)
289 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
290 else {
291 nodes_remap(tmp, pol->v.nodes, pol->w.cpuset_mems_allowed,
292 *nodes);
293 pol->w.cpuset_mems_allowed = *nodes;
294 }
295
296 pol->v.nodes = tmp;
297 if (!node_isset(current->il_next, tmp)) {
298 current->il_next = next_node(current->il_next, tmp);
299 if (current->il_next >= MAX_NUMNODES)
300 current->il_next = first_node(tmp);
301 if (current->il_next >= MAX_NUMNODES)
302 current->il_next = numa_node_id();
303 }
304}
305
306static void mpol_rebind_preferred(struct mempolicy *pol,
307 const nodemask_t *nodes)
308{
309 nodemask_t tmp;
310
David Rientjes37012942008-04-28 02:12:33 -0700311 if (pol->flags & MPOL_F_STATIC_NODES) {
312 int node = first_node(pol->w.user_nodemask);
313
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700314 if (node_isset(node, *nodes)) {
David Rientjes37012942008-04-28 02:12:33 -0700315 pol->v.preferred_node = node;
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700316 pol->flags &= ~MPOL_F_LOCAL;
317 } else
318 pol->flags |= MPOL_F_LOCAL;
David Rientjes37012942008-04-28 02:12:33 -0700319 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
320 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
321 pol->v.preferred_node = first_node(tmp);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700322 } else if (!(pol->flags & MPOL_F_LOCAL)) {
David Rientjes37012942008-04-28 02:12:33 -0700323 pol->v.preferred_node = node_remap(pol->v.preferred_node,
324 pol->w.cpuset_mems_allowed,
325 *nodes);
326 pol->w.cpuset_mems_allowed = *nodes;
327 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328}
329
David Rientjes1d0d2682008-04-28 02:12:32 -0700330/* Migrate a policy to a different set of nodes */
331static void mpol_rebind_policy(struct mempolicy *pol,
332 const nodemask_t *newmask)
333{
David Rientjes1d0d2682008-04-28 02:12:32 -0700334 if (!pol)
335 return;
David Rientjes1d0d2682008-04-28 02:12:32 -0700336 if (!mpol_store_user_nodemask(pol) &&
337 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
338 return;
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700339 mpol_ops[pol->mode].rebind(pol, newmask);
David Rientjes1d0d2682008-04-28 02:12:32 -0700340}
341
342/*
343 * Wrapper for mpol_rebind_policy() that just requires task
344 * pointer, and updates task mempolicy.
Miao Xie58568d22009-06-16 15:31:49 -0700345 *
346 * Called with task's alloc_lock held.
David Rientjes1d0d2682008-04-28 02:12:32 -0700347 */
348
349void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
350{
351 mpol_rebind_policy(tsk->mempolicy, new);
352}
353
354/*
355 * Rebind each vma in mm to new nodemask.
356 *
357 * Call holding a reference to mm. Takes mm->mmap_sem during call.
358 */
359
360void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
361{
362 struct vm_area_struct *vma;
363
364 down_write(&mm->mmap_sem);
365 for (vma = mm->mmap; vma; vma = vma->vm_next)
366 mpol_rebind_policy(vma->vm_policy, new);
367 up_write(&mm->mmap_sem);
368}
369
David Rientjes37012942008-04-28 02:12:33 -0700370static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
371 [MPOL_DEFAULT] = {
372 .rebind = mpol_rebind_default,
373 },
374 [MPOL_INTERLEAVE] = {
375 .create = mpol_new_interleave,
376 .rebind = mpol_rebind_nodemask,
377 },
378 [MPOL_PREFERRED] = {
379 .create = mpol_new_preferred,
380 .rebind = mpol_rebind_preferred,
381 },
382 [MPOL_BIND] = {
383 .create = mpol_new_bind,
384 .rebind = mpol_rebind_nodemask,
385 },
386};
387
Christoph Lameter397874d2006-03-06 15:42:53 -0800388static void gather_stats(struct page *, void *, int pte_dirty);
Christoph Lameterfc301282006-01-18 17:42:29 -0800389static void migrate_page_add(struct page *page, struct list_head *pagelist,
390 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800391
Christoph Lameter38e35862006-01-08 01:01:01 -0800392/* Scan through pages checking if pages follow certain conditions. */
Nick Pigginb5810032005-10-29 18:16:12 -0700393static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800394 unsigned long addr, unsigned long end,
395 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800396 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397{
Hugh Dickins91612e02005-06-21 17:15:07 -0700398 pte_t *orig_pte;
399 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700400 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700401
Hugh Dickins705e87c2005-10-29 18:16:27 -0700402 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700403 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800404 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800405 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700406
407 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800409 page = vm_normal_page(vma, addr, *pte);
410 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800412 /*
Hugh Dickins62b61f62009-12-14 17:59:33 -0800413 * vm_normal_page() filters out zero pages, but there might
414 * still be PageReserved pages to skip, perhaps in a VDSO.
415 * And we cannot move PageKsm pages sensibly or safely yet.
Nick Piggin053837f2006-01-18 17:42:27 -0800416 */
Hugh Dickins62b61f62009-12-14 17:59:33 -0800417 if (PageReserved(page) || PageKsm(page))
Christoph Lameterf4598c82006-01-12 01:05:20 -0800418 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800419 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800420 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
421 continue;
422
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800423 if (flags & MPOL_MF_STATS)
Christoph Lameter397874d2006-03-06 15:42:53 -0800424 gather_stats(page, private, pte_dirty(*pte));
Nick Piggin053837f2006-01-18 17:42:27 -0800425 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800426 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800427 else
428 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700429 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700430 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700431 return addr != end;
432}
433
Nick Pigginb5810032005-10-29 18:16:12 -0700434static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800435 unsigned long addr, unsigned long end,
436 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800437 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700438{
439 pmd_t *pmd;
440 unsigned long next;
441
442 pmd = pmd_offset(pud, addr);
443 do {
444 next = pmd_addr_end(addr, end);
445 if (pmd_none_or_clear_bad(pmd))
446 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800447 if (check_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800448 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700449 return -EIO;
450 } while (pmd++, addr = next, addr != end);
451 return 0;
452}
453
Nick Pigginb5810032005-10-29 18:16:12 -0700454static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800455 unsigned long addr, unsigned long end,
456 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800457 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700458{
459 pud_t *pud;
460 unsigned long next;
461
462 pud = pud_offset(pgd, addr);
463 do {
464 next = pud_addr_end(addr, end);
465 if (pud_none_or_clear_bad(pud))
466 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800467 if (check_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800468 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700469 return -EIO;
470 } while (pud++, addr = next, addr != end);
471 return 0;
472}
473
Nick Pigginb5810032005-10-29 18:16:12 -0700474static inline int check_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800475 unsigned long addr, unsigned long end,
476 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800477 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700478{
479 pgd_t *pgd;
480 unsigned long next;
481
Nick Pigginb5810032005-10-29 18:16:12 -0700482 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700483 do {
484 next = pgd_addr_end(addr, end);
485 if (pgd_none_or_clear_bad(pgd))
486 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800487 if (check_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800488 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700489 return -EIO;
490 } while (pgd++, addr = next, addr != end);
491 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492}
493
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800494/*
495 * Check if all pages in a range are on a set of nodes.
496 * If pagelist != NULL then isolate pages from the LRU and
497 * put them on the pagelist.
498 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499static struct vm_area_struct *
500check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800501 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502{
503 int err;
504 struct vm_area_struct *first, *vma, *prev;
505
Nick Piggin053837f2006-01-18 17:42:27 -0800506
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 first = find_vma(mm, start);
508 if (!first)
509 return ERR_PTR(-EFAULT);
510 prev = NULL;
511 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800512 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
513 if (!vma->vm_next && vma->vm_end < end)
514 return ERR_PTR(-EFAULT);
515 if (prev && prev->vm_end < vma->vm_start)
516 return ERR_PTR(-EFAULT);
517 }
518 if (!is_vm_hugetlb_page(vma) &&
519 ((flags & MPOL_MF_STRICT) ||
520 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
521 vma_migratable(vma)))) {
Andi Kleen5b952b32005-09-13 01:25:08 -0700522 unsigned long endvma = vma->vm_end;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800523
Andi Kleen5b952b32005-09-13 01:25:08 -0700524 if (endvma > end)
525 endvma = end;
526 if (vma->vm_start > start)
527 start = vma->vm_start;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800528 err = check_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800529 flags, private);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 if (err) {
531 first = ERR_PTR(err);
532 break;
533 }
534 }
535 prev = vma;
536 }
537 return first;
538}
539
540/* Apply policy to a single VMA */
541static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
542{
543 int err = 0;
544 struct mempolicy *old = vma->vm_policy;
545
Paul Mundt140d5a42007-07-15 23:38:16 -0700546 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 vma->vm_start, vma->vm_end, vma->vm_pgoff,
548 vma->vm_ops, vma->vm_file,
549 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
550
551 if (vma->vm_ops && vma->vm_ops->set_policy)
552 err = vma->vm_ops->set_policy(vma, new);
553 if (!err) {
554 mpol_get(new);
555 vma->vm_policy = new;
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700556 mpol_put(old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 }
558 return err;
559}
560
561/* Step 2: apply policy to a range and do splits. */
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800562static int mbind_range(struct mm_struct *mm, unsigned long start,
563 unsigned long end, struct mempolicy *new_pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564{
565 struct vm_area_struct *next;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800566 struct vm_area_struct *prev;
567 struct vm_area_struct *vma;
568 int err = 0;
569 pgoff_t pgoff;
570 unsigned long vmstart;
571 unsigned long vmend;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800573 vma = find_vma_prev(mm, start, &prev);
574 if (!vma || vma->vm_start > start)
575 return -EFAULT;
576
577 for (; vma && vma->vm_start < end; prev = vma, vma = next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 next = vma->vm_next;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800579 vmstart = max(start, vma->vm_start);
580 vmend = min(end, vma->vm_end);
581
582 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
583 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
584 vma->anon_vma, vma->vm_file, pgoff, new_pol);
585 if (prev) {
586 vma = prev;
587 next = vma->vm_next;
588 continue;
589 }
590 if (vma->vm_start != vmstart) {
591 err = split_vma(vma->vm_mm, vma, vmstart, 1);
592 if (err)
593 goto out;
594 }
595 if (vma->vm_end != vmend) {
596 err = split_vma(vma->vm_mm, vma, vmend, 0);
597 if (err)
598 goto out;
599 }
600 err = policy_vma(vma, new_pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 if (err)
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800602 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 }
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800604
605 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 return err;
607}
608
Paul Jacksonc61afb12006-03-24 03:16:08 -0800609/*
610 * Update task->flags PF_MEMPOLICY bit: set iff non-default
611 * mempolicy. Allows more rapid checking of this (combined perhaps
612 * with other PF_* flag bits) on memory allocation hot code paths.
613 *
614 * If called from outside this file, the task 'p' should -only- be
615 * a newly forked child not yet visible on the task list, because
616 * manipulating the task flags of a visible task is not safe.
617 *
618 * The above limitation is why this routine has the funny name
619 * mpol_fix_fork_child_flag().
620 *
621 * It is also safe to call this with a task pointer of current,
622 * which the static wrapper mpol_set_task_struct_flag() does,
623 * for use within this file.
624 */
625
626void mpol_fix_fork_child_flag(struct task_struct *p)
627{
628 if (p->mempolicy)
629 p->flags |= PF_MEMPOLICY;
630 else
631 p->flags &= ~PF_MEMPOLICY;
632}
633
634static void mpol_set_task_struct_flag(void)
635{
636 mpol_fix_fork_child_flag(current);
637}
638
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639/* Set the process memory policy */
David Rientjes028fec42008-04-28 02:12:25 -0700640static long do_set_mempolicy(unsigned short mode, unsigned short flags,
641 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642{
Miao Xie58568d22009-06-16 15:31:49 -0700643 struct mempolicy *new, *old;
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700644 struct mm_struct *mm = current->mm;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700645 NODEMASK_SCRATCH(scratch);
Miao Xie58568d22009-06-16 15:31:49 -0700646 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700648 if (!scratch)
649 return -ENOMEM;
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700650
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700651 new = mpol_new(mode, flags, nodes);
652 if (IS_ERR(new)) {
653 ret = PTR_ERR(new);
654 goto out;
655 }
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700656 /*
657 * prevent changing our mempolicy while show_numa_maps()
658 * is using it.
659 * Note: do_set_mempolicy() can be called at init time
660 * with no 'mm'.
661 */
662 if (mm)
663 down_write(&mm->mmap_sem);
Miao Xie58568d22009-06-16 15:31:49 -0700664 task_lock(current);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700665 ret = mpol_set_nodemask(new, nodes, scratch);
Miao Xie58568d22009-06-16 15:31:49 -0700666 if (ret) {
667 task_unlock(current);
668 if (mm)
669 up_write(&mm->mmap_sem);
670 mpol_put(new);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700671 goto out;
Miao Xie58568d22009-06-16 15:31:49 -0700672 }
673 old = current->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674 current->mempolicy = new;
Paul Jacksonc61afb12006-03-24 03:16:08 -0800675 mpol_set_task_struct_flag();
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700676 if (new && new->mode == MPOL_INTERLEAVE &&
David Rientjesf5b087b2008-04-28 02:12:27 -0700677 nodes_weight(new->v.nodes))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700678 current->il_next = first_node(new->v.nodes);
Miao Xie58568d22009-06-16 15:31:49 -0700679 task_unlock(current);
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700680 if (mm)
681 up_write(&mm->mmap_sem);
682
Miao Xie58568d22009-06-16 15:31:49 -0700683 mpol_put(old);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700684 ret = 0;
685out:
686 NODEMASK_SCRATCH_FREE(scratch);
687 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688}
689
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700690/*
691 * Return nodemask for policy for get_mempolicy() query
Miao Xie58568d22009-06-16 15:31:49 -0700692 *
693 * Called with task's alloc_lock held
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700694 */
695static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700697 nodes_clear(*nodes);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700698 if (p == &default_policy)
699 return;
700
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700701 switch (p->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -0700702 case MPOL_BIND:
703 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700705 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 break;
707 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700708 if (!(p->flags & MPOL_F_LOCAL))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700709 node_set(p->v.preferred_node, *nodes);
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700710 /* else return empty node mask for local allocation */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711 break;
712 default:
713 BUG();
714 }
715}
716
717static int lookup_node(struct mm_struct *mm, unsigned long addr)
718{
719 struct page *p;
720 int err;
721
722 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
723 if (err >= 0) {
724 err = page_to_nid(p);
725 put_page(p);
726 }
727 return err;
728}
729
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730/* Retrieve NUMA policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700731static long do_get_mempolicy(int *policy, nodemask_t *nmask,
732 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700734 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 struct mm_struct *mm = current->mm;
736 struct vm_area_struct *vma = NULL;
737 struct mempolicy *pol = current->mempolicy;
738
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700739 if (flags &
740 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 return -EINVAL;
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700742
743 if (flags & MPOL_F_MEMS_ALLOWED) {
744 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
745 return -EINVAL;
746 *policy = 0; /* just so it's initialized */
Miao Xie58568d22009-06-16 15:31:49 -0700747 task_lock(current);
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700748 *nmask = cpuset_current_mems_allowed;
Miao Xie58568d22009-06-16 15:31:49 -0700749 task_unlock(current);
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700750 return 0;
751 }
752
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 if (flags & MPOL_F_ADDR) {
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700754 /*
755 * Do NOT fall back to task policy if the
756 * vma/shared policy at addr is NULL. We
757 * want to return MPOL_DEFAULT in this case.
758 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 down_read(&mm->mmap_sem);
760 vma = find_vma_intersection(mm, addr, addr+1);
761 if (!vma) {
762 up_read(&mm->mmap_sem);
763 return -EFAULT;
764 }
765 if (vma->vm_ops && vma->vm_ops->get_policy)
766 pol = vma->vm_ops->get_policy(vma, addr);
767 else
768 pol = vma->vm_policy;
769 } else if (addr)
770 return -EINVAL;
771
772 if (!pol)
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700773 pol = &default_policy; /* indicates default behavior */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774
775 if (flags & MPOL_F_NODE) {
776 if (flags & MPOL_F_ADDR) {
777 err = lookup_node(mm, addr);
778 if (err < 0)
779 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700780 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 } else if (pol == current->mempolicy &&
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700782 pol->mode == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700783 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 } else {
785 err = -EINVAL;
786 goto out;
787 }
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700788 } else {
789 *policy = pol == &default_policy ? MPOL_DEFAULT :
790 pol->mode;
David Rientjesd79df632008-07-04 12:24:13 -0700791 /*
792 * Internal mempolicy flags must be masked off before exposing
793 * the policy to userspace.
794 */
795 *policy |= (pol->flags & MPOL_MODE_FLAGS);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700796 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
798 if (vma) {
799 up_read(&current->mm->mmap_sem);
800 vma = NULL;
801 }
802
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 err = 0;
Miao Xie58568d22009-06-16 15:31:49 -0700804 if (nmask) {
Lee Schermerhornc6b6ef82010-03-23 13:35:41 -0700805 if (mpol_store_user_nodemask(pol)) {
806 *nmask = pol->w.user_nodemask;
807 } else {
808 task_lock(current);
809 get_policy_nodemask(pol, nmask);
810 task_unlock(current);
811 }
Miao Xie58568d22009-06-16 15:31:49 -0700812 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
814 out:
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700815 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 if (vma)
817 up_read(&current->mm->mmap_sem);
818 return err;
819}
820
Christoph Lameterb20a3502006-03-22 00:09:12 -0800821#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700822/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800823 * page migration
824 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800825static void migrate_page_add(struct page *page, struct list_head *pagelist,
826 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800827{
828 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800829 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800830 */
Nick Piggin62695a82008-10-18 20:26:09 -0700831 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
832 if (!isolate_lru_page(page)) {
833 list_add_tail(&page->lru, pagelist);
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -0800834 inc_zone_page_state(page, NR_ISOLATED_ANON +
835 page_is_file_cache(page));
Nick Piggin62695a82008-10-18 20:26:09 -0700836 }
837 }
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800838}
839
Christoph Lameter742755a2006-06-23 02:03:55 -0700840static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700841{
Mel Gorman6484eb32009-06-16 15:31:54 -0700842 return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700843}
844
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800845/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800846 * Migrate pages from one node to a target node.
847 * Returns error or the number of pages not migrated.
848 */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700849static int migrate_to_node(struct mm_struct *mm, int source, int dest,
850 int flags)
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800851{
852 nodemask_t nmask;
853 LIST_HEAD(pagelist);
854 int err = 0;
855
856 nodes_clear(nmask);
857 node_set(source, nmask);
858
859 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
860 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
861
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700862 if (!list_empty(&pagelist))
Hugh Dickins62b61f62009-12-14 17:59:33 -0800863 err = migrate_pages(&pagelist, new_node_page, dest, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700864
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800865 return err;
866}
867
868/*
869 * Move pages between the two nodesets so as to preserve the physical
870 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -0800871 *
872 * Returns the number of page that could not be moved.
873 */
874int do_migrate_pages(struct mm_struct *mm,
875 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
876{
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800877 int busy = 0;
Christoph Lameter0aedadf2008-11-06 12:53:30 -0800878 int err;
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800879 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -0800880
Christoph Lameter0aedadf2008-11-06 12:53:30 -0800881 err = migrate_prep();
882 if (err)
883 return err;
884
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700885 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -0800886
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700887 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
888 if (err)
889 goto out;
890
KOSAKI Motohiroda0aa132010-03-05 13:41:59 -0800891 /*
892 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
893 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
894 * bit in 'tmp', and return that <source, dest> pair for migration.
895 * The pair of nodemasks 'to' and 'from' define the map.
896 *
897 * If no pair of bits is found that way, fallback to picking some
898 * pair of 'source' and 'dest' bits that are not the same. If the
899 * 'source' and 'dest' bits are the same, this represents a node
900 * that will be migrating to itself, so no pages need move.
901 *
902 * If no bits are left in 'tmp', or if all remaining bits left
903 * in 'tmp' correspond to the same bit in 'to', return false
904 * (nothing left to migrate).
905 *
906 * This lets us pick a pair of nodes to migrate between, such that
907 * if possible the dest node is not already occupied by some other
908 * source node, minimizing the risk of overloading the memory on a
909 * node that would happen if we migrated incoming memory to a node
910 * before migrating outgoing memory source that same node.
911 *
912 * A single scan of tmp is sufficient. As we go, we remember the
913 * most recent <s, d> pair that moved (s != d). If we find a pair
914 * that not only moved, but what's better, moved to an empty slot
915 * (d is not set in tmp), then we break out then, with that pair.
916 * Otherwise when we finish scannng from_tmp, we at least have the
917 * most recent <s, d> pair that moved. If we get all the way through
918 * the scan of tmp without finding any node that moved, much less
919 * moved to an empty node, then there is nothing left worth migrating.
920 */
Christoph Lameterd4984712006-01-08 01:00:55 -0800921
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800922 tmp = *from_nodes;
923 while (!nodes_empty(tmp)) {
924 int s,d;
925 int source = -1;
926 int dest = 0;
927
928 for_each_node_mask(s, tmp) {
929 d = node_remap(s, *from_nodes, *to_nodes);
930 if (s == d)
931 continue;
932
933 source = s; /* Node moved. Memorize */
934 dest = d;
935
936 /* dest not in remaining from nodes? */
937 if (!node_isset(dest, tmp))
938 break;
939 }
940 if (source == -1)
941 break;
942
943 node_clear(source, tmp);
944 err = migrate_to_node(mm, source, dest, flags);
945 if (err > 0)
946 busy += err;
947 if (err < 0)
948 break;
Christoph Lameter39743882006-01-08 01:00:51 -0800949 }
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700950out:
Christoph Lameter39743882006-01-08 01:00:51 -0800951 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800952 if (err < 0)
953 return err;
954 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800955
Christoph Lameter39743882006-01-08 01:00:51 -0800956}
957
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800958/*
959 * Allocate a new page for page migration based on vma policy.
960 * Start assuming that page is mapped by vma pointed to by @private.
961 * Search forward from there, if not. N.B., this assumes that the
962 * list of pages handed to migrate_pages()--which is how we get here--
963 * is in virtual address order.
964 */
Christoph Lameter742755a2006-06-23 02:03:55 -0700965static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700966{
967 struct vm_area_struct *vma = (struct vm_area_struct *)private;
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800968 unsigned long uninitialized_var(address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700969
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800970 while (vma) {
971 address = page_address_in_vma(page, vma);
972 if (address != -EFAULT)
973 break;
974 vma = vma->vm_next;
975 }
976
977 /*
978 * if !vma, alloc_page_vma() will use task or system default policy
979 */
980 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700981}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800982#else
983
984static void migrate_page_add(struct page *page, struct list_head *pagelist,
985 unsigned long flags)
986{
987}
988
989int do_migrate_pages(struct mm_struct *mm,
990 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
991{
992 return -ENOSYS;
993}
Christoph Lameter95a402c2006-06-23 02:03:53 -0700994
Keith Owens69939742006-10-11 01:21:28 -0700995static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700996{
997 return NULL;
998}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800999#endif
1000
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001001static long do_mbind(unsigned long start, unsigned long len,
David Rientjes028fec42008-04-28 02:12:25 -07001002 unsigned short mode, unsigned short mode_flags,
1003 nodemask_t *nmask, unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001004{
1005 struct vm_area_struct *vma;
1006 struct mm_struct *mm = current->mm;
1007 struct mempolicy *new;
1008 unsigned long end;
1009 int err;
1010 LIST_HEAD(pagelist);
1011
David Rientjesa3b51e02008-04-28 02:12:23 -07001012 if (flags & ~(unsigned long)(MPOL_MF_STRICT |
1013 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001014 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -08001015 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001016 return -EPERM;
1017
1018 if (start & ~PAGE_MASK)
1019 return -EINVAL;
1020
1021 if (mode == MPOL_DEFAULT)
1022 flags &= ~MPOL_MF_STRICT;
1023
1024 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1025 end = start + len;
1026
1027 if (end < start)
1028 return -EINVAL;
1029 if (end == start)
1030 return 0;
1031
David Rientjes028fec42008-04-28 02:12:25 -07001032 new = mpol_new(mode, mode_flags, nmask);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001033 if (IS_ERR(new))
1034 return PTR_ERR(new);
1035
1036 /*
1037 * If we are using the default policy then operation
1038 * on discontinuous address spaces is okay after all
1039 */
1040 if (!new)
1041 flags |= MPOL_MF_DISCONTIG_OK;
1042
David Rientjes028fec42008-04-28 02:12:25 -07001043 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1044 start, start + len, mode, mode_flags,
1045 nmask ? nodes_addr(*nmask)[0] : -1);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001046
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001047 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1048
1049 err = migrate_prep();
1050 if (err)
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001051 goto mpol_out;
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001052 }
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07001053 {
1054 NODEMASK_SCRATCH(scratch);
1055 if (scratch) {
1056 down_write(&mm->mmap_sem);
1057 task_lock(current);
1058 err = mpol_set_nodemask(new, nmask, scratch);
1059 task_unlock(current);
1060 if (err)
1061 up_write(&mm->mmap_sem);
1062 } else
1063 err = -ENOMEM;
1064 NODEMASK_SCRATCH_FREE(scratch);
1065 }
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001066 if (err)
1067 goto mpol_out;
1068
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001069 vma = check_range(mm, start, end, nmask,
1070 flags | MPOL_MF_INVERT, &pagelist);
1071
1072 err = PTR_ERR(vma);
1073 if (!IS_ERR(vma)) {
1074 int nr_failed = 0;
1075
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -08001076 err = mbind_range(mm, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001077
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001078 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -07001079 nr_failed = migrate_pages(&pagelist, new_vma_page,
Hugh Dickins62b61f62009-12-14 17:59:33 -08001080 (unsigned long)vma, 0);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001081
1082 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
1083 err = -EIO;
KOSAKI Motohiroab8a3e12009-10-26 16:49:58 -07001084 } else
1085 putback_lru_pages(&pagelist);
Christoph Lameterb20a3502006-03-22 00:09:12 -08001086
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001087 up_write(&mm->mmap_sem);
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001088 mpol_out:
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001089 mpol_put(new);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001090 return err;
1091}
1092
Christoph Lameter39743882006-01-08 01:00:51 -08001093/*
Christoph Lameter8bccd852005-10-29 18:16:59 -07001094 * User space interface with variable sized bitmaps for nodelists.
1095 */
1096
1097/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -08001098static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -07001099 unsigned long maxnode)
1100{
1101 unsigned long k;
1102 unsigned long nlongs;
1103 unsigned long endmask;
1104
1105 --maxnode;
1106 nodes_clear(*nodes);
1107 if (maxnode == 0 || !nmask)
1108 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -08001109 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -08001110 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001111
1112 nlongs = BITS_TO_LONGS(maxnode);
1113 if ((maxnode % BITS_PER_LONG) == 0)
1114 endmask = ~0UL;
1115 else
1116 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1117
1118 /* When the user specified more nodes than supported just check
1119 if the non supported part is all zero. */
1120 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1121 if (nlongs > PAGE_SIZE/sizeof(long))
1122 return -EINVAL;
1123 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1124 unsigned long t;
1125 if (get_user(t, nmask + k))
1126 return -EFAULT;
1127 if (k == nlongs - 1) {
1128 if (t & endmask)
1129 return -EINVAL;
1130 } else if (t)
1131 return -EINVAL;
1132 }
1133 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1134 endmask = ~0UL;
1135 }
1136
1137 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1138 return -EFAULT;
1139 nodes_addr(*nodes)[nlongs-1] &= endmask;
1140 return 0;
1141}
1142
1143/* Copy a kernel node mask to user space */
1144static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1145 nodemask_t *nodes)
1146{
1147 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1148 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1149
1150 if (copy > nbytes) {
1151 if (copy > PAGE_SIZE)
1152 return -EINVAL;
1153 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1154 return -EFAULT;
1155 copy = nbytes;
1156 }
1157 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1158}
1159
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001160SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1161 unsigned long, mode, unsigned long __user *, nmask,
1162 unsigned long, maxnode, unsigned, flags)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001163{
1164 nodemask_t nodes;
1165 int err;
David Rientjes028fec42008-04-28 02:12:25 -07001166 unsigned short mode_flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001167
David Rientjes028fec42008-04-28 02:12:25 -07001168 mode_flags = mode & MPOL_MODE_FLAGS;
1169 mode &= ~MPOL_MODE_FLAGS;
David Rientjesa3b51e02008-04-28 02:12:23 -07001170 if (mode >= MPOL_MAX)
1171 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001172 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1173 (mode_flags & MPOL_F_RELATIVE_NODES))
1174 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001175 err = get_nodes(&nodes, nmask, maxnode);
1176 if (err)
1177 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001178 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001179}
1180
1181/* Set the process memory policy */
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001182SYSCALL_DEFINE3(set_mempolicy, int, mode, unsigned long __user *, nmask,
1183 unsigned long, maxnode)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001184{
1185 int err;
1186 nodemask_t nodes;
David Rientjes028fec42008-04-28 02:12:25 -07001187 unsigned short flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001188
David Rientjes028fec42008-04-28 02:12:25 -07001189 flags = mode & MPOL_MODE_FLAGS;
1190 mode &= ~MPOL_MODE_FLAGS;
1191 if ((unsigned int)mode >= MPOL_MAX)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001192 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001193 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1194 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001195 err = get_nodes(&nodes, nmask, maxnode);
1196 if (err)
1197 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001198 return do_set_mempolicy(mode, flags, &nodes);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001199}
1200
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001201SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1202 const unsigned long __user *, old_nodes,
1203 const unsigned long __user *, new_nodes)
Christoph Lameter39743882006-01-08 01:00:51 -08001204{
David Howellsc69e8d92008-11-14 10:39:19 +11001205 const struct cred *cred = current_cred(), *tcred;
Christoph Lameter39743882006-01-08 01:00:51 -08001206 struct mm_struct *mm;
1207 struct task_struct *task;
1208 nodemask_t old;
1209 nodemask_t new;
1210 nodemask_t task_nodes;
1211 int err;
1212
1213 err = get_nodes(&old, old_nodes, maxnode);
1214 if (err)
1215 return err;
1216
1217 err = get_nodes(&new, new_nodes, maxnode);
1218 if (err)
1219 return err;
1220
1221 /* Find the mm_struct */
1222 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001223 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter39743882006-01-08 01:00:51 -08001224 if (!task) {
1225 read_unlock(&tasklist_lock);
1226 return -ESRCH;
1227 }
1228 mm = get_task_mm(task);
1229 read_unlock(&tasklist_lock);
1230
1231 if (!mm)
1232 return -EINVAL;
1233
1234 /*
1235 * Check if this process has the right to modify the specified
1236 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -08001237 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -08001238 * userid as the target process.
1239 */
David Howellsc69e8d92008-11-14 10:39:19 +11001240 rcu_read_lock();
1241 tcred = __task_cred(task);
David Howellsb6dff3e2008-11-14 10:39:16 +11001242 if (cred->euid != tcred->suid && cred->euid != tcred->uid &&
1243 cred->uid != tcred->suid && cred->uid != tcred->uid &&
Christoph Lameter74c00242006-03-14 19:50:21 -08001244 !capable(CAP_SYS_NICE)) {
David Howellsc69e8d92008-11-14 10:39:19 +11001245 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001246 err = -EPERM;
1247 goto out;
1248 }
David Howellsc69e8d92008-11-14 10:39:19 +11001249 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001250
1251 task_nodes = cpuset_mems_allowed(task);
1252 /* Is the user allowed to access the target nodes? */
Christoph Lameter74c00242006-03-14 19:50:21 -08001253 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001254 err = -EPERM;
1255 goto out;
1256 }
1257
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07001258 if (!nodes_subset(new, node_states[N_HIGH_MEMORY])) {
Christoph Lameter3b42d282007-08-31 00:12:08 -07001259 err = -EINVAL;
1260 goto out;
1261 }
1262
David Quigley86c3a762006-06-23 02:04:02 -07001263 err = security_task_movememory(task);
1264 if (err)
1265 goto out;
1266
Christoph Lameter511030b2006-02-28 16:58:57 -08001267 err = do_migrate_pages(mm, &old, &new,
Christoph Lameter74c00242006-03-14 19:50:21 -08001268 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter39743882006-01-08 01:00:51 -08001269out:
1270 mmput(mm);
1271 return err;
1272}
1273
1274
Christoph Lameter8bccd852005-10-29 18:16:59 -07001275/* Retrieve NUMA policy */
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001276SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1277 unsigned long __user *, nmask, unsigned long, maxnode,
1278 unsigned long, addr, unsigned long, flags)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001279{
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001280 int err;
1281 int uninitialized_var(pval);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001282 nodemask_t nodes;
1283
1284 if (nmask != NULL && maxnode < MAX_NUMNODES)
1285 return -EINVAL;
1286
1287 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1288
1289 if (err)
1290 return err;
1291
1292 if (policy && put_user(pval, policy))
1293 return -EFAULT;
1294
1295 if (nmask)
1296 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1297
1298 return err;
1299}
1300
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301#ifdef CONFIG_COMPAT
1302
1303asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1304 compat_ulong_t __user *nmask,
1305 compat_ulong_t maxnode,
1306 compat_ulong_t addr, compat_ulong_t flags)
1307{
1308 long err;
1309 unsigned long __user *nm = NULL;
1310 unsigned long nr_bits, alloc_size;
1311 DECLARE_BITMAP(bm, MAX_NUMNODES);
1312
1313 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1314 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1315
1316 if (nmask)
1317 nm = compat_alloc_user_space(alloc_size);
1318
1319 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1320
1321 if (!err && nmask) {
1322 err = copy_from_user(bm, nm, alloc_size);
1323 /* ensure entire bitmap is zeroed */
1324 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1325 err |= compat_put_bitmap(nmask, bm, nr_bits);
1326 }
1327
1328 return err;
1329}
1330
1331asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1332 compat_ulong_t maxnode)
1333{
1334 long err = 0;
1335 unsigned long __user *nm = NULL;
1336 unsigned long nr_bits, alloc_size;
1337 DECLARE_BITMAP(bm, MAX_NUMNODES);
1338
1339 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1340 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1341
1342 if (nmask) {
1343 err = compat_get_bitmap(bm, nmask, nr_bits);
1344 nm = compat_alloc_user_space(alloc_size);
1345 err |= copy_to_user(nm, bm, alloc_size);
1346 }
1347
1348 if (err)
1349 return -EFAULT;
1350
1351 return sys_set_mempolicy(mode, nm, nr_bits+1);
1352}
1353
1354asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1355 compat_ulong_t mode, compat_ulong_t __user *nmask,
1356 compat_ulong_t maxnode, compat_ulong_t flags)
1357{
1358 long err = 0;
1359 unsigned long __user *nm = NULL;
1360 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001361 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362
1363 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1364 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1365
1366 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001367 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001369 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370 }
1371
1372 if (err)
1373 return -EFAULT;
1374
1375 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1376}
1377
1378#endif
1379
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001380/*
1381 * get_vma_policy(@task, @vma, @addr)
1382 * @task - task for fallback if vma policy == default
1383 * @vma - virtual memory area whose policy is sought
1384 * @addr - address in @vma for shared policy lookup
1385 *
1386 * Returns effective policy for a VMA at specified address.
1387 * Falls back to @task or system default policy, as necessary.
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001388 * Current or other task's task mempolicy and non-shared vma policies
1389 * are protected by the task's mmap_sem, which must be held for read by
1390 * the caller.
1391 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1392 * count--added by the get_policy() vm_op, as appropriate--to protect against
1393 * freeing by another task. It is the caller's responsibility to free the
1394 * extra reference for shared policies.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001395 */
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001396static struct mempolicy *get_vma_policy(struct task_struct *task,
Christoph Lameter48fce342006-01-08 01:01:03 -08001397 struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001399 struct mempolicy *pol = task->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400
1401 if (vma) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001402 if (vma->vm_ops && vma->vm_ops->get_policy) {
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001403 struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
1404 addr);
1405 if (vpol)
1406 pol = vpol;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001407 } else if (vma->vm_policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408 pol = vma->vm_policy;
1409 }
1410 if (!pol)
1411 pol = &default_policy;
1412 return pol;
1413}
1414
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001415/*
1416 * Return a nodemask representing a mempolicy for filtering nodes for
1417 * page allocation
1418 */
1419static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
Mel Gorman19770b32008-04-28 02:12:18 -07001420{
1421 /* Lower zones don't get a nodemask applied for MPOL_BIND */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001422 if (unlikely(policy->mode == MPOL_BIND) &&
Mel Gorman19770b32008-04-28 02:12:18 -07001423 gfp_zone(gfp) >= policy_zone &&
1424 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1425 return &policy->v.nodes;
1426
1427 return NULL;
1428}
1429
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001430/* Return a zonelist indicated by gfp for node representing a mempolicy */
1431static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432{
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001433 int nd = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001435 switch (policy->mode) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001437 if (!(policy->flags & MPOL_F_LOCAL))
1438 nd = policy->v.preferred_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439 break;
1440 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07001441 /*
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001442 * Normally, MPOL_BIND allocations are node-local within the
1443 * allowed nodemask. However, if __GFP_THISNODE is set and the
Bob Liu6eb27e12010-05-24 14:32:00 -07001444 * current node isn't part of the mask, we use the zonelist for
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001445 * the first node in the mask instead.
Mel Gorman19770b32008-04-28 02:12:18 -07001446 */
Mel Gorman19770b32008-04-28 02:12:18 -07001447 if (unlikely(gfp & __GFP_THISNODE) &&
1448 unlikely(!node_isset(nd, policy->v.nodes)))
1449 nd = first_node(policy->v.nodes);
1450 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452 BUG();
1453 }
Mel Gorman0e884602008-04-28 02:12:14 -07001454 return node_zonelist(nd, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455}
1456
1457/* Do dynamic interleaving for a process */
1458static unsigned interleave_nodes(struct mempolicy *policy)
1459{
1460 unsigned nid, next;
1461 struct task_struct *me = current;
1462
1463 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001464 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001466 next = first_node(policy->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001467 if (next < MAX_NUMNODES)
1468 me->il_next = next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469 return nid;
1470}
1471
Christoph Lameterdc85da12006-01-18 17:42:36 -08001472/*
1473 * Depending on the memory policy provide a node from which to allocate the
1474 * next slab entry.
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001475 * @policy must be protected by freeing by the caller. If @policy is
1476 * the current task's mempolicy, this protection is implicit, as only the
1477 * task can change it's policy. The system default policy requires no
1478 * such protection.
Christoph Lameterdc85da12006-01-18 17:42:36 -08001479 */
1480unsigned slab_node(struct mempolicy *policy)
1481{
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001482 if (!policy || policy->flags & MPOL_F_LOCAL)
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001483 return numa_node_id();
Christoph Lameter765c4502006-09-27 01:50:08 -07001484
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001485 switch (policy->mode) {
1486 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001487 /*
1488 * handled MPOL_F_LOCAL above
1489 */
1490 return policy->v.preferred_node;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001491
Christoph Lameterdc85da12006-01-18 17:42:36 -08001492 case MPOL_INTERLEAVE:
1493 return interleave_nodes(policy);
1494
Mel Gormandd1a2392008-04-28 02:12:17 -07001495 case MPOL_BIND: {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001496 /*
1497 * Follow bind policy behavior and start allocation at the
1498 * first node.
1499 */
Mel Gorman19770b32008-04-28 02:12:18 -07001500 struct zonelist *zonelist;
1501 struct zone *zone;
1502 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1503 zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0];
1504 (void)first_zones_zonelist(zonelist, highest_zoneidx,
1505 &policy->v.nodes,
1506 &zone);
1507 return zone->node;
Mel Gormandd1a2392008-04-28 02:12:17 -07001508 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08001509
Christoph Lameterdc85da12006-01-18 17:42:36 -08001510 default:
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001511 BUG();
Christoph Lameterdc85da12006-01-18 17:42:36 -08001512 }
1513}
1514
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515/* Do static interleaving for a VMA with known offset. */
1516static unsigned offset_il_node(struct mempolicy *pol,
1517 struct vm_area_struct *vma, unsigned long off)
1518{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001519 unsigned nnodes = nodes_weight(pol->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001520 unsigned target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 int c;
1522 int nid = -1;
1523
David Rientjesf5b087b2008-04-28 02:12:27 -07001524 if (!nnodes)
1525 return numa_node_id();
1526 target = (unsigned int)off % nnodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527 c = 0;
1528 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001529 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 c++;
1531 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 return nid;
1533}
1534
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001535/* Determine a node number for interleave */
1536static inline unsigned interleave_nid(struct mempolicy *pol,
1537 struct vm_area_struct *vma, unsigned long addr, int shift)
1538{
1539 if (vma) {
1540 unsigned long off;
1541
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001542 /*
1543 * for small pages, there is no difference between
1544 * shift and PAGE_SHIFT, so the bit-shift is safe.
1545 * for huge pages, since vm_pgoff is in units of small
1546 * pages, we need to shift off the always 0 bits to get
1547 * a useful offset.
1548 */
1549 BUG_ON(shift < PAGE_SHIFT);
1550 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001551 off += (addr - vma->vm_start) >> shift;
1552 return offset_il_node(pol, vma, off);
1553 } else
1554 return interleave_nodes(pol);
1555}
1556
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001557#ifdef CONFIG_HUGETLBFS
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001558/*
1559 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1560 * @vma = virtual memory area whose policy is sought
1561 * @addr = address in @vma for shared policy lookup and interleave policy
1562 * @gfp_flags = for requested zone
Mel Gorman19770b32008-04-28 02:12:18 -07001563 * @mpol = pointer to mempolicy pointer for reference counted mempolicy
1564 * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001565 *
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001566 * Returns a zonelist suitable for a huge page allocation and a pointer
1567 * to the struct mempolicy for conditional unref after allocation.
1568 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1569 * @nodemask for filtering the zonelist.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001570 */
Mel Gorman396faf02007-07-17 04:03:13 -07001571struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
Mel Gorman19770b32008-04-28 02:12:18 -07001572 gfp_t gfp_flags, struct mempolicy **mpol,
1573 nodemask_t **nodemask)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001574{
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001575 struct zonelist *zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001576
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001577 *mpol = get_vma_policy(current, vma, addr);
Mel Gorman19770b32008-04-28 02:12:18 -07001578 *nodemask = NULL; /* assume !MPOL_BIND */
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001579
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001580 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1581 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
Andi Kleena5516432008-07-23 21:27:41 -07001582 huge_page_shift(hstate_vma(vma))), gfp_flags);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001583 } else {
1584 zl = policy_zonelist(gfp_flags, *mpol);
1585 if ((*mpol)->mode == MPOL_BIND)
1586 *nodemask = &(*mpol)->v.nodes;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001587 }
1588 return zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001589}
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001590
1591/*
1592 * init_nodemask_of_mempolicy
1593 *
1594 * If the current task's mempolicy is "default" [NULL], return 'false'
1595 * to indicate default policy. Otherwise, extract the policy nodemask
1596 * for 'bind' or 'interleave' policy into the argument nodemask, or
1597 * initialize the argument nodemask to contain the single node for
1598 * 'preferred' or 'local' policy and return 'true' to indicate presence
1599 * of non-default mempolicy.
1600 *
1601 * We don't bother with reference counting the mempolicy [mpol_get/put]
1602 * because the current task is examining it's own mempolicy and a task's
1603 * mempolicy is only ever changed by the task itself.
1604 *
1605 * N.B., it is the caller's responsibility to free a returned nodemask.
1606 */
1607bool init_nodemask_of_mempolicy(nodemask_t *mask)
1608{
1609 struct mempolicy *mempolicy;
1610 int nid;
1611
1612 if (!(mask && current->mempolicy))
1613 return false;
1614
1615 mempolicy = current->mempolicy;
1616 switch (mempolicy->mode) {
1617 case MPOL_PREFERRED:
1618 if (mempolicy->flags & MPOL_F_LOCAL)
1619 nid = numa_node_id();
1620 else
1621 nid = mempolicy->v.preferred_node;
1622 init_nodemask_of_node(mask, nid);
1623 break;
1624
1625 case MPOL_BIND:
1626 /* Fall through */
1627 case MPOL_INTERLEAVE:
1628 *mask = mempolicy->v.nodes;
1629 break;
1630
1631 default:
1632 BUG();
1633 }
1634
1635 return true;
1636}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001637#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001638
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639/* Allocate a page in interleaved policy.
1640 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001641static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1642 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643{
1644 struct zonelist *zl;
1645 struct page *page;
1646
Mel Gorman0e884602008-04-28 02:12:14 -07001647 zl = node_zonelist(nid, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648 page = __alloc_pages(gfp, order, zl);
Mel Gormandd1a2392008-04-28 02:12:17 -07001649 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
Christoph Lameterca889e62006-06-30 01:55:44 -07001650 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 return page;
1652}
1653
1654/**
1655 * alloc_page_vma - Allocate a page for a VMA.
1656 *
1657 * @gfp:
1658 * %GFP_USER user allocation.
1659 * %GFP_KERNEL kernel allocations,
1660 * %GFP_HIGHMEM highmem/user allocations,
1661 * %GFP_FS allocation should not call back into a file system.
1662 * %GFP_ATOMIC don't sleep.
1663 *
1664 * @vma: Pointer to VMA or NULL if not available.
1665 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1666 *
1667 * This function allocates a page from the kernel page pool and applies
1668 * a NUMA policy associated with the VMA or the current process.
1669 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1670 * mm_struct of the VMA to prevent it from going away. Should be used for
1671 * all allocations for pages that will be mapped into
1672 * user space. Returns NULL when no page can be allocated.
1673 *
1674 * Should be called with the mm_sem of the vma hold.
1675 */
1676struct page *
Al Virodd0fc662005-10-07 07:46:04 +01001677alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001678{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001679 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001680 struct zonelist *zl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001682 if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001684
1685 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001686 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001687 return alloc_page_interleave(gfp, 0, nid);
1688 }
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001689 zl = policy_zonelist(gfp, pol);
1690 if (unlikely(mpol_needs_cond_ref(pol))) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001691 /*
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001692 * slow path: ref counted shared policy
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001693 */
Mel Gorman19770b32008-04-28 02:12:18 -07001694 struct page *page = __alloc_pages_nodemask(gfp, 0,
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001695 zl, policy_nodemask(gfp, pol));
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001696 __mpol_put(pol);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001697 return page;
1698 }
1699 /*
1700 * fast path: default or task policy
1701 */
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001702 return __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703}
1704
1705/**
1706 * alloc_pages_current - Allocate pages.
1707 *
1708 * @gfp:
1709 * %GFP_USER user allocation,
1710 * %GFP_KERNEL kernel allocation,
1711 * %GFP_HIGHMEM highmem allocation,
1712 * %GFP_FS don't call back into a file system.
1713 * %GFP_ATOMIC don't sleep.
1714 * @order: Power of two of allocation size in pages. 0 is a single page.
1715 *
1716 * Allocate a page from the kernel page pool. When not in
1717 * interrupt context and apply the current process NUMA policy.
1718 * Returns NULL when no page can be allocated.
1719 *
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001720 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721 * 1) it's ok to take cpuset_sem (can WAIT), and
1722 * 2) allocating for current task (not interrupt).
1723 */
Al Virodd0fc662005-10-07 07:46:04 +01001724struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725{
1726 struct mempolicy *pol = current->mempolicy;
1727
Christoph Lameter9b819d22006-09-25 23:31:40 -07001728 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729 pol = &default_policy;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001730
1731 /*
1732 * No reference counting needed for current->mempolicy
1733 * nor system default_policy
1734 */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001735 if (pol->mode == MPOL_INTERLEAVE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
Mel Gorman19770b32008-04-28 02:12:18 -07001737 return __alloc_pages_nodemask(gfp, order,
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001738 policy_zonelist(gfp, pol), policy_nodemask(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739}
1740EXPORT_SYMBOL(alloc_pages_current);
1741
Paul Jackson42253992006-01-08 01:01:59 -08001742/*
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001743 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
Paul Jackson42253992006-01-08 01:01:59 -08001744 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1745 * with the mems_allowed returned by cpuset_mems_allowed(). This
1746 * keeps mempolicies cpuset relative after its cpuset moves. See
1747 * further kernel/cpuset.c update_nodemask().
1748 */
Paul Jackson42253992006-01-08 01:01:59 -08001749
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001750/* Slow path of a mempolicy duplicate */
1751struct mempolicy *__mpol_dup(struct mempolicy *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752{
1753 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1754
1755 if (!new)
1756 return ERR_PTR(-ENOMEM);
Paul E. McKenney99ee4ca2010-03-03 17:50:17 -08001757 rcu_read_lock();
Paul Jackson42253992006-01-08 01:01:59 -08001758 if (current_cpuset_is_being_rebound()) {
1759 nodemask_t mems = cpuset_mems_allowed(current);
1760 mpol_rebind_policy(old, &mems);
1761 }
Paul E. McKenney99ee4ca2010-03-03 17:50:17 -08001762 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001763 *new = *old;
1764 atomic_set(&new->refcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765 return new;
1766}
1767
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001768/*
1769 * If *frompol needs [has] an extra ref, copy *frompol to *tompol ,
1770 * eliminate the * MPOL_F_* flags that require conditional ref and
1771 * [NOTE!!!] drop the extra ref. Not safe to reference *frompol directly
1772 * after return. Use the returned value.
1773 *
1774 * Allows use of a mempolicy for, e.g., multiple allocations with a single
1775 * policy lookup, even if the policy needs/has extra ref on lookup.
1776 * shmem_readahead needs this.
1777 */
1778struct mempolicy *__mpol_cond_copy(struct mempolicy *tompol,
1779 struct mempolicy *frompol)
1780{
1781 if (!mpol_needs_cond_ref(frompol))
1782 return frompol;
1783
1784 *tompol = *frompol;
1785 tompol->flags &= ~MPOL_F_SHARED; /* copy doesn't need unref */
1786 __mpol_put(frompol);
1787 return tompol;
1788}
1789
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790/* Slow path of a mempolicy comparison */
1791int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1792{
1793 if (!a || !b)
1794 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001795 if (a->mode != b->mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 return 0;
Bob Liu19800502010-05-24 14:32:01 -07001797 if (a->flags != b->flags)
David Rientjesf5b087b2008-04-28 02:12:27 -07001798 return 0;
Bob Liu19800502010-05-24 14:32:01 -07001799 if (mpol_store_user_nodemask(a))
1800 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
1801 return 0;
1802
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001803 switch (a->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -07001804 case MPOL_BIND:
1805 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -07001807 return nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001808 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001809 return a->v.preferred_node == b->v.preferred_node &&
1810 a->flags == b->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811 default:
1812 BUG();
1813 return 0;
1814 }
1815}
1816
Linus Torvalds1da177e2005-04-16 15:20:36 -07001817/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818 * Shared memory backing store policy support.
1819 *
1820 * Remember policies even when nobody has shared memory mapped.
1821 * The policies are kept in Red-Black tree linked from the inode.
1822 * They are protected by the sp->lock spinlock, which should be held
1823 * for any accesses to the tree.
1824 */
1825
1826/* lookup first element intersecting start-end */
1827/* Caller holds sp->lock */
1828static struct sp_node *
1829sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1830{
1831 struct rb_node *n = sp->root.rb_node;
1832
1833 while (n) {
1834 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1835
1836 if (start >= p->end)
1837 n = n->rb_right;
1838 else if (end <= p->start)
1839 n = n->rb_left;
1840 else
1841 break;
1842 }
1843 if (!n)
1844 return NULL;
1845 for (;;) {
1846 struct sp_node *w = NULL;
1847 struct rb_node *prev = rb_prev(n);
1848 if (!prev)
1849 break;
1850 w = rb_entry(prev, struct sp_node, nd);
1851 if (w->end <= start)
1852 break;
1853 n = prev;
1854 }
1855 return rb_entry(n, struct sp_node, nd);
1856}
1857
1858/* Insert a new shared policy into the list. */
1859/* Caller holds sp->lock */
1860static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1861{
1862 struct rb_node **p = &sp->root.rb_node;
1863 struct rb_node *parent = NULL;
1864 struct sp_node *nd;
1865
1866 while (*p) {
1867 parent = *p;
1868 nd = rb_entry(parent, struct sp_node, nd);
1869 if (new->start < nd->start)
1870 p = &(*p)->rb_left;
1871 else if (new->end > nd->end)
1872 p = &(*p)->rb_right;
1873 else
1874 BUG();
1875 }
1876 rb_link_node(&new->nd, parent, p);
1877 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07001878 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001879 new->policy ? new->policy->mode : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001880}
1881
1882/* Find shared policy intersecting idx */
1883struct mempolicy *
1884mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1885{
1886 struct mempolicy *pol = NULL;
1887 struct sp_node *sn;
1888
1889 if (!sp->root.rb_node)
1890 return NULL;
1891 spin_lock(&sp->lock);
1892 sn = sp_lookup(sp, idx, idx+1);
1893 if (sn) {
1894 mpol_get(sn->policy);
1895 pol = sn->policy;
1896 }
1897 spin_unlock(&sp->lock);
1898 return pol;
1899}
1900
1901static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1902{
Paul Mundt140d5a42007-07-15 23:38:16 -07001903 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 rb_erase(&n->nd, &sp->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001905 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001906 kmem_cache_free(sn_cache, n);
1907}
1908
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001909static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
1910 struct mempolicy *pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001911{
1912 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1913
1914 if (!n)
1915 return NULL;
1916 n->start = start;
1917 n->end = end;
1918 mpol_get(pol);
Lee Schermerhornaab0b102008-04-28 02:13:13 -07001919 pol->flags |= MPOL_F_SHARED; /* for unref */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001920 n->policy = pol;
1921 return n;
1922}
1923
1924/* Replace a policy range. */
1925static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1926 unsigned long end, struct sp_node *new)
1927{
1928 struct sp_node *n, *new2 = NULL;
1929
1930restart:
1931 spin_lock(&sp->lock);
1932 n = sp_lookup(sp, start, end);
1933 /* Take care of old policies in the same range. */
1934 while (n && n->start < end) {
1935 struct rb_node *next = rb_next(&n->nd);
1936 if (n->start >= start) {
1937 if (n->end <= end)
1938 sp_delete(sp, n);
1939 else
1940 n->start = end;
1941 } else {
1942 /* Old policy spanning whole new range. */
1943 if (n->end > end) {
1944 if (!new2) {
1945 spin_unlock(&sp->lock);
1946 new2 = sp_alloc(end, n->end, n->policy);
1947 if (!new2)
1948 return -ENOMEM;
1949 goto restart;
1950 }
1951 n->end = start;
1952 sp_insert(sp, new2);
1953 new2 = NULL;
1954 break;
1955 } else
1956 n->end = start;
1957 }
1958 if (!next)
1959 break;
1960 n = rb_entry(next, struct sp_node, nd);
1961 }
1962 if (new)
1963 sp_insert(sp, new);
1964 spin_unlock(&sp->lock);
1965 if (new2) {
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001966 mpol_put(new2->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967 kmem_cache_free(sn_cache, new2);
1968 }
1969 return 0;
1970}
1971
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001972/**
1973 * mpol_shared_policy_init - initialize shared policy for inode
1974 * @sp: pointer to inode shared policy
1975 * @mpol: struct mempolicy to install
1976 *
1977 * Install non-NULL @mpol in inode's shared policy rb-tree.
1978 * On entry, the current task has a reference on a non-NULL @mpol.
1979 * This must be released on exit.
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07001980 * This is called at get_inode() calls and we can use GFP_KERNEL.
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001981 */
1982void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
Robin Holt7339ff82006-01-14 13:20:48 -08001983{
Miao Xie58568d22009-06-16 15:31:49 -07001984 int ret;
1985
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001986 sp->root = RB_ROOT; /* empty tree == default mempolicy */
1987 spin_lock_init(&sp->lock);
Robin Holt7339ff82006-01-14 13:20:48 -08001988
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001989 if (mpol) {
1990 struct vm_area_struct pvma;
1991 struct mempolicy *new;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07001992 NODEMASK_SCRATCH(scratch);
Robin Holt7339ff82006-01-14 13:20:48 -08001993
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07001994 if (!scratch)
1995 return;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001996 /* contextualize the tmpfs mount point mempolicy */
1997 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
Miao Xie58568d22009-06-16 15:31:49 -07001998 if (IS_ERR(new)) {
1999 mpol_put(mpol); /* drop our ref on sb mpol */
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002000 NODEMASK_SCRATCH_FREE(scratch);
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002001 return; /* no valid nodemask intersection */
Miao Xie58568d22009-06-16 15:31:49 -07002002 }
2003
2004 task_lock(current);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002005 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
Miao Xie58568d22009-06-16 15:31:49 -07002006 task_unlock(current);
2007 mpol_put(mpol); /* drop our ref on sb mpol */
2008 if (ret) {
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002009 NODEMASK_SCRATCH_FREE(scratch);
Miao Xie58568d22009-06-16 15:31:49 -07002010 mpol_put(new);
2011 return;
2012 }
Robin Holt7339ff82006-01-14 13:20:48 -08002013
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002014 /* Create pseudo-vma that contains just the policy */
2015 memset(&pvma, 0, sizeof(struct vm_area_struct));
2016 pvma.vm_end = TASK_SIZE; /* policy covers entire file */
2017 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
2018 mpol_put(new); /* drop initial ref */
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002019 NODEMASK_SCRATCH_FREE(scratch);
Robin Holt7339ff82006-01-14 13:20:48 -08002020 }
2021}
2022
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023int mpol_set_shared_policy(struct shared_policy *info,
2024 struct vm_area_struct *vma, struct mempolicy *npol)
2025{
2026 int err;
2027 struct sp_node *new = NULL;
2028 unsigned long sz = vma_pages(vma);
2029
David Rientjes028fec42008-04-28 02:12:25 -07002030 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031 vma->vm_pgoff,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002032 sz, npol ? npol->mode : -1,
David Rientjes028fec42008-04-28 02:12:25 -07002033 npol ? npol->flags : -1,
Paul Mundt140d5a42007-07-15 23:38:16 -07002034 npol ? nodes_addr(npol->v.nodes)[0] : -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035
2036 if (npol) {
2037 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2038 if (!new)
2039 return -ENOMEM;
2040 }
2041 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2042 if (err && new)
2043 kmem_cache_free(sn_cache, new);
2044 return err;
2045}
2046
2047/* Free a backing policy store on inode delete. */
2048void mpol_free_shared_policy(struct shared_policy *p)
2049{
2050 struct sp_node *n;
2051 struct rb_node *next;
2052
2053 if (!p->root.rb_node)
2054 return;
2055 spin_lock(&p->lock);
2056 next = rb_first(&p->root);
2057 while (next) {
2058 n = rb_entry(next, struct sp_node, nd);
2059 next = rb_next(&n->nd);
Andi Kleen90c50292005-07-27 11:43:50 -07002060 rb_erase(&n->nd, &p->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07002061 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002062 kmem_cache_free(sn_cache, n);
2063 }
2064 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065}
2066
2067/* assumes fs == KERNEL_DS */
2068void __init numa_policy_init(void)
2069{
Paul Mundtb71636e2007-07-15 23:38:15 -07002070 nodemask_t interleave_nodes;
2071 unsigned long largest = 0;
2072 int nid, prefer = 0;
2073
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074 policy_cache = kmem_cache_create("numa_policy",
2075 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09002076 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077
2078 sn_cache = kmem_cache_create("shared_policy_node",
2079 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09002080 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002081
Paul Mundtb71636e2007-07-15 23:38:15 -07002082 /*
2083 * Set interleaving policy for system init. Interleaving is only
2084 * enabled across suitably sized nodes (default is >= 16MB), or
2085 * fall back to the largest node if they're all smaller.
2086 */
2087 nodes_clear(interleave_nodes);
Christoph Lameter56bbd652007-10-16 01:25:35 -07002088 for_each_node_state(nid, N_HIGH_MEMORY) {
Paul Mundtb71636e2007-07-15 23:38:15 -07002089 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002090
Paul Mundtb71636e2007-07-15 23:38:15 -07002091 /* Preserve the largest node */
2092 if (largest < total_pages) {
2093 largest = total_pages;
2094 prefer = nid;
2095 }
2096
2097 /* Interleave this node? */
2098 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2099 node_set(nid, interleave_nodes);
2100 }
2101
2102 /* All too small, use the largest */
2103 if (unlikely(nodes_empty(interleave_nodes)))
2104 node_set(prefer, interleave_nodes);
2105
David Rientjes028fec42008-04-28 02:12:25 -07002106 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002107 printk("numa_policy_init: interleaving failed\n");
2108}
2109
Christoph Lameter8bccd852005-10-29 18:16:59 -07002110/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002111void numa_default_policy(void)
2112{
David Rientjes028fec42008-04-28 02:12:25 -07002113 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002114}
Paul Jackson68860ec2005-10-30 15:02:36 -08002115
Paul Jackson42253992006-01-08 01:01:59 -08002116/*
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002117 * Parse and format mempolicy from/to strings
2118 */
2119
2120/*
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002121 * "local" is pseudo-policy: MPOL_PREFERRED with MPOL_F_LOCAL flag
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002122 * Used only for mpol_parse_str() and mpol_to_str()
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002123 */
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002124#define MPOL_LOCAL MPOL_MAX
2125static const char * const policy_modes[] =
2126{
2127 [MPOL_DEFAULT] = "default",
2128 [MPOL_PREFERRED] = "prefer",
2129 [MPOL_BIND] = "bind",
2130 [MPOL_INTERLEAVE] = "interleave",
2131 [MPOL_LOCAL] = "local"
2132};
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002133
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002134
2135#ifdef CONFIG_TMPFS
2136/**
2137 * mpol_parse_str - parse string to mempolicy
2138 * @str: string containing mempolicy to parse
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002139 * @mpol: pointer to struct mempolicy pointer, returned on success.
2140 * @no_context: flag whether to "contextualize" the mempolicy
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002141 *
2142 * Format of input:
2143 * <mode>[=<flags>][:<nodelist>]
2144 *
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002145 * if @no_context is true, save the input nodemask in w.user_nodemask in
2146 * the returned mempolicy. This will be used to "clone" the mempolicy in
2147 * a specific context [cpuset] at a later time. Used to parse tmpfs mpol
2148 * mount option. Note that if 'static' or 'relative' mode flags were
2149 * specified, the input nodemask will already have been saved. Saving
2150 * it again is redundant, but safe.
2151 *
2152 * On success, returns 0, else 1
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002153 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002154int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002155{
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002156 struct mempolicy *new = NULL;
Lee Schermerhornb4652e82010-05-24 14:32:03 -07002157 unsigned short mode;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002158 unsigned short uninitialized_var(mode_flags);
2159 nodemask_t nodes;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002160 char *nodelist = strchr(str, ':');
2161 char *flags = strchr(str, '=');
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002162 int err = 1;
2163
2164 if (nodelist) {
2165 /* NUL-terminate mode or flags string */
2166 *nodelist++ = '\0';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002167 if (nodelist_parse(nodelist, nodes))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002168 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002169 if (!nodes_subset(nodes, node_states[N_HIGH_MEMORY]))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002170 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002171 } else
2172 nodes_clear(nodes);
2173
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002174 if (flags)
2175 *flags++ = '\0'; /* terminate mode string */
2176
Lee Schermerhornb4652e82010-05-24 14:32:03 -07002177 for (mode = 0; mode <= MPOL_LOCAL; mode++) {
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002178 if (!strcmp(str, policy_modes[mode])) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002179 break;
2180 }
2181 }
Lee Schermerhornb4652e82010-05-24 14:32:03 -07002182 if (mode > MPOL_LOCAL)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002183 goto out;
2184
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002185 switch (mode) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002186 case MPOL_PREFERRED:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002187 /*
2188 * Insist on a nodelist of one node only
2189 */
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002190 if (nodelist) {
2191 char *rest = nodelist;
2192 while (isdigit(*rest))
2193 rest++;
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002194 if (*rest)
2195 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002196 }
2197 break;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002198 case MPOL_INTERLEAVE:
2199 /*
2200 * Default to online nodes with memory if no nodelist
2201 */
2202 if (!nodelist)
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002203 nodes = node_states[N_HIGH_MEMORY];
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002204 break;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002205 case MPOL_LOCAL:
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002206 /*
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002207 * Don't allow a nodelist; mpol_new() checks flags
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002208 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002209 if (nodelist)
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002210 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002211 mode = MPOL_PREFERRED;
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002212 break;
Ravikiran G Thirumalai413b43d2010-03-23 13:35:28 -07002213 case MPOL_DEFAULT:
2214 /*
2215 * Insist on a empty nodelist
2216 */
2217 if (!nodelist)
2218 err = 0;
2219 goto out;
KOSAKI Motohirod69b2e62010-03-23 13:35:30 -07002220 case MPOL_BIND:
2221 /*
2222 * Insist on a nodelist
2223 */
2224 if (!nodelist)
2225 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002226 }
2227
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002228 mode_flags = 0;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002229 if (flags) {
2230 /*
2231 * Currently, we only support two mutually exclusive
2232 * mode flags.
2233 */
2234 if (!strcmp(flags, "static"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002235 mode_flags |= MPOL_F_STATIC_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002236 else if (!strcmp(flags, "relative"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002237 mode_flags |= MPOL_F_RELATIVE_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002238 else
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002239 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002240 }
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002241
2242 new = mpol_new(mode, mode_flags, &nodes);
2243 if (IS_ERR(new))
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002244 goto out;
2245
Lee Schermerhorne17f74a2010-05-24 14:32:02 -07002246 if (no_context) {
2247 /* save for contextualization */
2248 new->w.user_nodemask = nodes;
2249 } else {
Miao Xie58568d22009-06-16 15:31:49 -07002250 int ret;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002251 NODEMASK_SCRATCH(scratch);
2252 if (scratch) {
2253 task_lock(current);
2254 ret = mpol_set_nodemask(new, &nodes, scratch);
2255 task_unlock(current);
2256 } else
2257 ret = -ENOMEM;
2258 NODEMASK_SCRATCH_FREE(scratch);
2259 if (ret) {
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002260 mpol_put(new);
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002261 goto out;
Miao Xie58568d22009-06-16 15:31:49 -07002262 }
2263 }
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002264 err = 0;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002265
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002266out:
2267 /* Restore string for error message */
2268 if (nodelist)
2269 *--nodelist = ':';
2270 if (flags)
2271 *--flags = '=';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002272 if (!err)
2273 *mpol = new;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002274 return err;
2275}
2276#endif /* CONFIG_TMPFS */
2277
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002278/**
2279 * mpol_to_str - format a mempolicy structure for printing
2280 * @buffer: to contain formatted mempolicy string
2281 * @maxlen: length of @buffer
2282 * @pol: pointer to mempolicy to be formatted
2283 * @no_context: "context free" mempolicy - use nodemask in w.user_nodemask
2284 *
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002285 * Convert a mempolicy into a string.
2286 * Returns the number of characters in buffer (if positive)
2287 * or an error (negative)
2288 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002289int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol, int no_context)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002290{
2291 char *p = buffer;
2292 int l;
2293 nodemask_t nodes;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002294 unsigned short mode;
David Rientjesf5b087b2008-04-28 02:12:27 -07002295 unsigned short flags = pol ? pol->flags : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002296
Lee Schermerhorn22919902008-04-28 02:13:22 -07002297 /*
2298 * Sanity check: room for longest mode, flag and some nodes
2299 */
2300 VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16);
2301
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002302 if (!pol || pol == &default_policy)
2303 mode = MPOL_DEFAULT;
2304 else
2305 mode = pol->mode;
2306
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002307 switch (mode) {
2308 case MPOL_DEFAULT:
2309 nodes_clear(nodes);
2310 break;
2311
2312 case MPOL_PREFERRED:
2313 nodes_clear(nodes);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002314 if (flags & MPOL_F_LOCAL)
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002315 mode = MPOL_LOCAL; /* pseudo-policy */
2316 else
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002317 node_set(pol->v.preferred_node, nodes);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002318 break;
2319
2320 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07002321 /* Fall through */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002322 case MPOL_INTERLEAVE:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002323 if (no_context)
2324 nodes = pol->w.user_nodemask;
2325 else
2326 nodes = pol->v.nodes;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002327 break;
2328
2329 default:
2330 BUG();
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002331 }
2332
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002333 l = strlen(policy_modes[mode]);
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002334 if (buffer + maxlen < p + l + 1)
2335 return -ENOSPC;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002336
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002337 strcpy(p, policy_modes[mode]);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002338 p += l;
2339
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002340 if (flags & MPOL_MODE_FLAGS) {
David Rientjesf5b087b2008-04-28 02:12:27 -07002341 if (buffer + maxlen < p + 2)
2342 return -ENOSPC;
2343 *p++ = '=';
2344
Lee Schermerhorn22919902008-04-28 02:13:22 -07002345 /*
2346 * Currently, the only defined flags are mutually exclusive
2347 */
David Rientjesf5b087b2008-04-28 02:12:27 -07002348 if (flags & MPOL_F_STATIC_NODES)
Lee Schermerhorn22919902008-04-28 02:13:22 -07002349 p += snprintf(p, buffer + maxlen - p, "static");
2350 else if (flags & MPOL_F_RELATIVE_NODES)
2351 p += snprintf(p, buffer + maxlen - p, "relative");
David Rientjesf5b087b2008-04-28 02:12:27 -07002352 }
2353
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002354 if (!nodes_empty(nodes)) {
2355 if (buffer + maxlen < p + 2)
2356 return -ENOSPC;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002357 *p++ = ':';
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002358 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
2359 }
2360 return p - buffer;
2361}
2362
2363struct numa_maps {
2364 unsigned long pages;
2365 unsigned long anon;
Christoph Lameter397874d2006-03-06 15:42:53 -08002366 unsigned long active;
2367 unsigned long writeback;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002368 unsigned long mapcount_max;
Christoph Lameter397874d2006-03-06 15:42:53 -08002369 unsigned long dirty;
2370 unsigned long swapcache;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002371 unsigned long node[MAX_NUMNODES];
2372};
2373
Christoph Lameter397874d2006-03-06 15:42:53 -08002374static void gather_stats(struct page *page, void *private, int pte_dirty)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002375{
2376 struct numa_maps *md = private;
2377 int count = page_mapcount(page);
2378
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002379 md->pages++;
Christoph Lameter397874d2006-03-06 15:42:53 -08002380 if (pte_dirty || PageDirty(page))
2381 md->dirty++;
2382
2383 if (PageSwapCache(page))
2384 md->swapcache++;
2385
Lee Schermerhorn894bc312008-10-18 20:26:39 -07002386 if (PageActive(page) || PageUnevictable(page))
Christoph Lameter397874d2006-03-06 15:42:53 -08002387 md->active++;
2388
2389 if (PageWriteback(page))
2390 md->writeback++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002391
2392 if (PageAnon(page))
2393 md->anon++;
2394
Christoph Lameter397874d2006-03-06 15:42:53 -08002395 if (count > md->mapcount_max)
2396 md->mapcount_max = count;
2397
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002398 md->node[page_to_nid(page)]++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002399}
2400
Andrew Morton7f709ed2006-03-07 21:55:22 -08002401#ifdef CONFIG_HUGETLB_PAGE
Christoph Lameter397874d2006-03-06 15:42:53 -08002402static void check_huge_range(struct vm_area_struct *vma,
2403 unsigned long start, unsigned long end,
2404 struct numa_maps *md)
2405{
2406 unsigned long addr;
2407 struct page *page;
Andi Kleena5516432008-07-23 21:27:41 -07002408 struct hstate *h = hstate_vma(vma);
2409 unsigned long sz = huge_page_size(h);
Christoph Lameter397874d2006-03-06 15:42:53 -08002410
Andi Kleena5516432008-07-23 21:27:41 -07002411 for (addr = start; addr < end; addr += sz) {
2412 pte_t *ptep = huge_pte_offset(vma->vm_mm,
2413 addr & huge_page_mask(h));
Christoph Lameter397874d2006-03-06 15:42:53 -08002414 pte_t pte;
2415
2416 if (!ptep)
2417 continue;
2418
2419 pte = *ptep;
2420 if (pte_none(pte))
2421 continue;
2422
2423 page = pte_page(pte);
2424 if (!page)
2425 continue;
2426
2427 gather_stats(page, md, pte_dirty(*ptep));
2428 }
2429}
Andrew Morton7f709ed2006-03-07 21:55:22 -08002430#else
2431static inline void check_huge_range(struct vm_area_struct *vma,
2432 unsigned long start, unsigned long end,
2433 struct numa_maps *md)
2434{
2435}
2436#endif
Christoph Lameter397874d2006-03-06 15:42:53 -08002437
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002438/*
2439 * Display pages allocated per node and memory policy via /proc.
2440 */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002441int show_numa_map(struct seq_file *m, void *v)
2442{
Eric W. Biederman99f89552006-06-26 00:25:55 -07002443 struct proc_maps_private *priv = m->private;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002444 struct vm_area_struct *vma = v;
2445 struct numa_maps *md;
Christoph Lameter397874d2006-03-06 15:42:53 -08002446 struct file *file = vma->vm_file;
2447 struct mm_struct *mm = vma->vm_mm;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002448 struct mempolicy *pol;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002449 int n;
2450 char buffer[50];
2451
Christoph Lameter397874d2006-03-06 15:42:53 -08002452 if (!mm)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002453 return 0;
2454
2455 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
2456 if (!md)
2457 return 0;
2458
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002459 pol = get_vma_policy(priv->task, vma, vma->vm_start);
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002460 mpol_to_str(buffer, sizeof(buffer), pol, 0);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002461 mpol_cond_put(pol);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002462
Christoph Lameter397874d2006-03-06 15:42:53 -08002463 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002464
Christoph Lameter397874d2006-03-06 15:42:53 -08002465 if (file) {
2466 seq_printf(m, " file=");
Jan Blunckc32c2f62008-02-14 19:38:43 -08002467 seq_path(m, &file->f_path, "\n\t= ");
Christoph Lameter397874d2006-03-06 15:42:53 -08002468 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
2469 seq_printf(m, " heap");
2470 } else if (vma->vm_start <= mm->start_stack &&
2471 vma->vm_end >= mm->start_stack) {
2472 seq_printf(m, " stack");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002473 }
Christoph Lameter397874d2006-03-06 15:42:53 -08002474
2475 if (is_vm_hugetlb_page(vma)) {
2476 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
2477 seq_printf(m, " huge");
2478 } else {
2479 check_pgd_range(vma, vma->vm_start, vma->vm_end,
Christoph Lameter56bbd652007-10-16 01:25:35 -07002480 &node_states[N_HIGH_MEMORY], MPOL_MF_STATS, md);
Christoph Lameter397874d2006-03-06 15:42:53 -08002481 }
2482
2483 if (!md->pages)
2484 goto out;
2485
2486 if (md->anon)
2487 seq_printf(m," anon=%lu",md->anon);
2488
2489 if (md->dirty)
2490 seq_printf(m," dirty=%lu",md->dirty);
2491
2492 if (md->pages != md->anon && md->pages != md->dirty)
2493 seq_printf(m, " mapped=%lu", md->pages);
2494
2495 if (md->mapcount_max > 1)
2496 seq_printf(m, " mapmax=%lu", md->mapcount_max);
2497
2498 if (md->swapcache)
2499 seq_printf(m," swapcache=%lu", md->swapcache);
2500
2501 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
2502 seq_printf(m," active=%lu", md->active);
2503
2504 if (md->writeback)
2505 seq_printf(m," writeback=%lu", md->writeback);
2506
Christoph Lameter56bbd652007-10-16 01:25:35 -07002507 for_each_node_state(n, N_HIGH_MEMORY)
Christoph Lameter397874d2006-03-06 15:42:53 -08002508 if (md->node[n])
2509 seq_printf(m, " N%d=%lu", n, md->node[n]);
2510out:
2511 seq_putc(m, '\n');
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002512 kfree(md);
2513
2514 if (m->count < m->size)
Eric W. Biederman99f89552006-06-26 00:25:55 -07002515 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002516 return 0;
2517}