blob: 78b18a60b9b22aa44dc2e39504e1cf544080e0ee [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
Linus Torvalds1da177e2005-04-16 15:20:36 -070066*/
67
68#include <linux/mempolicy.h>
69#include <linux/mm.h>
70#include <linux/highmem.h>
71#include <linux/hugetlb.h>
72#include <linux/kernel.h>
73#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <linux/nodemask.h>
75#include <linux/cpuset.h>
76#include <linux/gfp.h>
77#include <linux/slab.h>
78#include <linux/string.h>
79#include <linux/module.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070080#include <linux/nsproxy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#include <linux/interrupt.h>
82#include <linux/init.h>
83#include <linux/compat.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080084#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080085#include <linux/seq_file.h>
86#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080087#include <linux/migrate.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070088#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070089#include <linux/security.h>
Adrian Bunkdbcb0f12007-10-16 01:26:26 -070090#include <linux/syscalls.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080091
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#include <asm/tlbflush.h>
93#include <asm/uaccess.h>
94
Christoph Lameter38e35862006-01-08 01:01:01 -080095/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080096#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -080097#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080098#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080099
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800100static struct kmem_cache *policy_cache;
101static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103/* Highest zone. An specific allocation for a zone below that is not
104 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800105enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
Andi Kleend42c6992005-07-06 19:56:03 +0200107struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 .refcnt = ATOMIC_INIT(1), /* never free it */
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700109 .mode = MPOL_DEFAULT,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110};
111
David Rientjes37012942008-04-28 02:12:33 -0700112static const struct mempolicy_operations {
113 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
114 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
115} mpol_ops[MPOL_MAX];
116
Mel Gorman19770b32008-04-28 02:12:18 -0700117/* Check that the nodemask contains at least one populated zone */
David Rientjes37012942008-04-28 02:12:33 -0700118static int is_valid_nodemask(const nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119{
Mel Gorman19770b32008-04-28 02:12:18 -0700120 int nd, k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121
Mel Gorman19770b32008-04-28 02:12:18 -0700122 /* Check that there is something useful in this mask */
123 k = policy_zone;
124
125 for_each_node_mask(nd, *nodemask) {
126 struct zone *z;
127
128 for (k = 0; k <= policy_zone; k++) {
129 z = &NODE_DATA(nd)->node_zones[k];
130 if (z->present_pages > 0)
131 return 1;
Andi Kleendd942ae2006-02-17 01:39:16 +0100132 }
133 }
Mel Gorman19770b32008-04-28 02:12:18 -0700134
135 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136}
137
David Rientjesf5b087b2008-04-28 02:12:27 -0700138static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
139{
David Rientjes4c50bc02008-04-28 02:12:30 -0700140 return pol->flags & (MPOL_F_STATIC_NODES | MPOL_F_RELATIVE_NODES);
141}
142
143static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
144 const nodemask_t *rel)
145{
146 nodemask_t tmp;
147 nodes_fold(tmp, *orig, nodes_weight(*rel));
148 nodes_onto(*ret, tmp, *rel);
David Rientjesf5b087b2008-04-28 02:12:27 -0700149}
150
David Rientjes37012942008-04-28 02:12:33 -0700151static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
152{
153 if (nodes_empty(*nodes))
154 return -EINVAL;
155 pol->v.nodes = *nodes;
156 return 0;
157}
158
159static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
160{
161 if (!nodes)
162 pol->v.preferred_node = -1; /* local allocation */
163 else if (nodes_empty(*nodes))
164 return -EINVAL; /* no allowed nodes */
165 else
166 pol->v.preferred_node = first_node(*nodes);
167 return 0;
168}
169
170static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
171{
172 if (!is_valid_nodemask(nodes))
173 return -EINVAL;
174 pol->v.nodes = *nodes;
175 return 0;
176}
177
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178/* Create a new policy */
David Rientjes028fec42008-04-28 02:12:25 -0700179static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
180 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181{
182 struct mempolicy *policy;
David Rientjesf5b087b2008-04-28 02:12:27 -0700183 nodemask_t cpuset_context_nmask;
David Rientjes37012942008-04-28 02:12:33 -0700184 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185
David Rientjes028fec42008-04-28 02:12:25 -0700186 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
187 mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
Paul Mundt140d5a42007-07-15 23:38:16 -0700188
David Rientjes3e1f0642008-04-28 02:12:34 -0700189 if (mode == MPOL_DEFAULT) {
190 if (nodes && !nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700191 return ERR_PTR(-EINVAL);
David Rientjes3e1f0642008-04-28 02:12:34 -0700192 return NULL;
David Rientjes37012942008-04-28 02:12:33 -0700193 }
David Rientjes3e1f0642008-04-28 02:12:34 -0700194 VM_BUG_ON(!nodes);
195
196 /*
197 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
198 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
199 * All other modes require a valid pointer to a non-empty nodemask.
200 */
201 if (mode == MPOL_PREFERRED) {
202 if (nodes_empty(*nodes)) {
203 if (((flags & MPOL_F_STATIC_NODES) ||
204 (flags & MPOL_F_RELATIVE_NODES)))
205 return ERR_PTR(-EINVAL);
206 nodes = NULL; /* flag local alloc */
207 }
208 } else if (nodes_empty(*nodes))
209 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
211 if (!policy)
212 return ERR_PTR(-ENOMEM);
213 atomic_set(&policy->refcnt, 1);
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700214 policy->mode = mode;
David Rientjes3e1f0642008-04-28 02:12:34 -0700215 policy->flags = flags;
David Rientjesf5b087b2008-04-28 02:12:27 -0700216
David Rientjes3e1f0642008-04-28 02:12:34 -0700217 if (nodes) {
218 /*
219 * cpuset related setup doesn't apply to local allocation
220 */
David Rientjes37012942008-04-28 02:12:33 -0700221 cpuset_update_task_memory_state();
222 if (flags & MPOL_F_RELATIVE_NODES)
223 mpol_relative_nodemask(&cpuset_context_nmask, nodes,
224 &cpuset_current_mems_allowed);
225 else
226 nodes_and(cpuset_context_nmask, *nodes,
227 cpuset_current_mems_allowed);
228 if (mpol_store_user_nodemask(policy))
229 policy->w.user_nodemask = *nodes;
230 else
231 policy->w.cpuset_mems_allowed =
232 cpuset_mems_allowed(current);
233 }
234
235 ret = mpol_ops[mode].create(policy,
David Rientjes3e1f0642008-04-28 02:12:34 -0700236 nodes ? &cpuset_context_nmask : NULL);
David Rientjes37012942008-04-28 02:12:33 -0700237 if (ret < 0) {
238 kmem_cache_free(policy_cache, policy);
239 return ERR_PTR(ret);
240 }
241 return policy;
242}
243
244static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
245{
246}
247
248static void mpol_rebind_nodemask(struct mempolicy *pol,
249 const nodemask_t *nodes)
250{
251 nodemask_t tmp;
252
253 if (pol->flags & MPOL_F_STATIC_NODES)
254 nodes_and(tmp, pol->w.user_nodemask, *nodes);
255 else if (pol->flags & MPOL_F_RELATIVE_NODES)
256 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
257 else {
258 nodes_remap(tmp, pol->v.nodes, pol->w.cpuset_mems_allowed,
259 *nodes);
260 pol->w.cpuset_mems_allowed = *nodes;
261 }
262
263 pol->v.nodes = tmp;
264 if (!node_isset(current->il_next, tmp)) {
265 current->il_next = next_node(current->il_next, tmp);
266 if (current->il_next >= MAX_NUMNODES)
267 current->il_next = first_node(tmp);
268 if (current->il_next >= MAX_NUMNODES)
269 current->il_next = numa_node_id();
270 }
271}
272
273static void mpol_rebind_preferred(struct mempolicy *pol,
274 const nodemask_t *nodes)
275{
276 nodemask_t tmp;
277
David Rientjes37012942008-04-28 02:12:33 -0700278 if (pol->flags & MPOL_F_STATIC_NODES) {
279 int node = first_node(pol->w.user_nodemask);
280
281 if (node_isset(node, *nodes))
282 pol->v.preferred_node = node;
283 else
284 pol->v.preferred_node = -1;
David Rientjes37012942008-04-28 02:12:33 -0700285 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
286 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
287 pol->v.preferred_node = first_node(tmp);
David Rientjes3e1f0642008-04-28 02:12:34 -0700288 } else if (pol->v.preferred_node != -1) {
David Rientjes37012942008-04-28 02:12:33 -0700289 pol->v.preferred_node = node_remap(pol->v.preferred_node,
290 pol->w.cpuset_mems_allowed,
291 *nodes);
292 pol->w.cpuset_mems_allowed = *nodes;
293 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294}
295
David Rientjes1d0d2682008-04-28 02:12:32 -0700296/* Migrate a policy to a different set of nodes */
297static void mpol_rebind_policy(struct mempolicy *pol,
298 const nodemask_t *newmask)
299{
David Rientjes1d0d2682008-04-28 02:12:32 -0700300 if (!pol)
301 return;
David Rientjes1d0d2682008-04-28 02:12:32 -0700302 if (!mpol_store_user_nodemask(pol) &&
303 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
304 return;
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700305 mpol_ops[pol->mode].rebind(pol, newmask);
David Rientjes1d0d2682008-04-28 02:12:32 -0700306}
307
308/*
309 * Wrapper for mpol_rebind_policy() that just requires task
310 * pointer, and updates task mempolicy.
311 */
312
313void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
314{
315 mpol_rebind_policy(tsk->mempolicy, new);
316}
317
318/*
319 * Rebind each vma in mm to new nodemask.
320 *
321 * Call holding a reference to mm. Takes mm->mmap_sem during call.
322 */
323
324void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
325{
326 struct vm_area_struct *vma;
327
328 down_write(&mm->mmap_sem);
329 for (vma = mm->mmap; vma; vma = vma->vm_next)
330 mpol_rebind_policy(vma->vm_policy, new);
331 up_write(&mm->mmap_sem);
332}
333
David Rientjes37012942008-04-28 02:12:33 -0700334static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
335 [MPOL_DEFAULT] = {
336 .rebind = mpol_rebind_default,
337 },
338 [MPOL_INTERLEAVE] = {
339 .create = mpol_new_interleave,
340 .rebind = mpol_rebind_nodemask,
341 },
342 [MPOL_PREFERRED] = {
343 .create = mpol_new_preferred,
344 .rebind = mpol_rebind_preferred,
345 },
346 [MPOL_BIND] = {
347 .create = mpol_new_bind,
348 .rebind = mpol_rebind_nodemask,
349 },
350};
351
Christoph Lameter397874d2006-03-06 15:42:53 -0800352static void gather_stats(struct page *, void *, int pte_dirty);
Christoph Lameterfc301282006-01-18 17:42:29 -0800353static void migrate_page_add(struct page *page, struct list_head *pagelist,
354 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800355
Christoph Lameter38e35862006-01-08 01:01:01 -0800356/* Scan through pages checking if pages follow certain conditions. */
Nick Pigginb5810032005-10-29 18:16:12 -0700357static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800358 unsigned long addr, unsigned long end,
359 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800360 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361{
Hugh Dickins91612e02005-06-21 17:15:07 -0700362 pte_t *orig_pte;
363 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700364 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700365
Hugh Dickins705e87c2005-10-29 18:16:27 -0700366 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700367 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800368 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800369 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700370
371 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800373 page = vm_normal_page(vma, addr, *pte);
374 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800376 /*
377 * The check for PageReserved here is important to avoid
378 * handling zero pages and other pages that may have been
379 * marked special by the system.
380 *
381 * If the PageReserved would not be checked here then f.e.
382 * the location of the zero page could have an influence
383 * on MPOL_MF_STRICT, zero pages would be counted for
384 * the per node stats, and there would be useless attempts
385 * to put zero pages on the migration list.
386 */
Christoph Lameterf4598c82006-01-12 01:05:20 -0800387 if (PageReserved(page))
388 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800389 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800390 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
391 continue;
392
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800393 if (flags & MPOL_MF_STATS)
Christoph Lameter397874d2006-03-06 15:42:53 -0800394 gather_stats(page, private, pte_dirty(*pte));
Nick Piggin053837f2006-01-18 17:42:27 -0800395 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800396 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800397 else
398 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700399 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700400 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700401 return addr != end;
402}
403
Nick Pigginb5810032005-10-29 18:16:12 -0700404static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800405 unsigned long addr, unsigned long end,
406 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800407 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700408{
409 pmd_t *pmd;
410 unsigned long next;
411
412 pmd = pmd_offset(pud, addr);
413 do {
414 next = pmd_addr_end(addr, end);
415 if (pmd_none_or_clear_bad(pmd))
416 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800417 if (check_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800418 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700419 return -EIO;
420 } while (pmd++, addr = next, addr != end);
421 return 0;
422}
423
Nick Pigginb5810032005-10-29 18:16:12 -0700424static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800425 unsigned long addr, unsigned long end,
426 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800427 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700428{
429 pud_t *pud;
430 unsigned long next;
431
432 pud = pud_offset(pgd, addr);
433 do {
434 next = pud_addr_end(addr, end);
435 if (pud_none_or_clear_bad(pud))
436 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800437 if (check_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800438 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700439 return -EIO;
440 } while (pud++, addr = next, addr != end);
441 return 0;
442}
443
Nick Pigginb5810032005-10-29 18:16:12 -0700444static inline int check_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800445 unsigned long addr, unsigned long end,
446 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800447 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700448{
449 pgd_t *pgd;
450 unsigned long next;
451
Nick Pigginb5810032005-10-29 18:16:12 -0700452 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700453 do {
454 next = pgd_addr_end(addr, end);
455 if (pgd_none_or_clear_bad(pgd))
456 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800457 if (check_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800458 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700459 return -EIO;
460 } while (pgd++, addr = next, addr != end);
461 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462}
463
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800464/*
465 * Check if all pages in a range are on a set of nodes.
466 * If pagelist != NULL then isolate pages from the LRU and
467 * put them on the pagelist.
468 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469static struct vm_area_struct *
470check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800471 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472{
473 int err;
474 struct vm_area_struct *first, *vma, *prev;
475
Christoph Lameter90036ee2006-03-16 23:03:59 -0800476 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
Christoph Lameter90036ee2006-03-16 23:03:59 -0800477
Christoph Lameterb20a3502006-03-22 00:09:12 -0800478 err = migrate_prep();
479 if (err)
480 return ERR_PTR(err);
Christoph Lameter90036ee2006-03-16 23:03:59 -0800481 }
Nick Piggin053837f2006-01-18 17:42:27 -0800482
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 first = find_vma(mm, start);
484 if (!first)
485 return ERR_PTR(-EFAULT);
486 prev = NULL;
487 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800488 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
489 if (!vma->vm_next && vma->vm_end < end)
490 return ERR_PTR(-EFAULT);
491 if (prev && prev->vm_end < vma->vm_start)
492 return ERR_PTR(-EFAULT);
493 }
494 if (!is_vm_hugetlb_page(vma) &&
495 ((flags & MPOL_MF_STRICT) ||
496 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
497 vma_migratable(vma)))) {
Andi Kleen5b952b32005-09-13 01:25:08 -0700498 unsigned long endvma = vma->vm_end;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800499
Andi Kleen5b952b32005-09-13 01:25:08 -0700500 if (endvma > end)
501 endvma = end;
502 if (vma->vm_start > start)
503 start = vma->vm_start;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800504 err = check_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800505 flags, private);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 if (err) {
507 first = ERR_PTR(err);
508 break;
509 }
510 }
511 prev = vma;
512 }
513 return first;
514}
515
516/* Apply policy to a single VMA */
517static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
518{
519 int err = 0;
520 struct mempolicy *old = vma->vm_policy;
521
Paul Mundt140d5a42007-07-15 23:38:16 -0700522 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 vma->vm_start, vma->vm_end, vma->vm_pgoff,
524 vma->vm_ops, vma->vm_file,
525 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
526
527 if (vma->vm_ops && vma->vm_ops->set_policy)
528 err = vma->vm_ops->set_policy(vma, new);
529 if (!err) {
530 mpol_get(new);
531 vma->vm_policy = new;
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700532 mpol_put(old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 }
534 return err;
535}
536
537/* Step 2: apply policy to a range and do splits. */
538static int mbind_range(struct vm_area_struct *vma, unsigned long start,
539 unsigned long end, struct mempolicy *new)
540{
541 struct vm_area_struct *next;
542 int err;
543
544 err = 0;
545 for (; vma && vma->vm_start < end; vma = next) {
546 next = vma->vm_next;
547 if (vma->vm_start < start)
548 err = split_vma(vma->vm_mm, vma, start, 1);
549 if (!err && vma->vm_end > end)
550 err = split_vma(vma->vm_mm, vma, end, 0);
551 if (!err)
552 err = policy_vma(vma, new);
553 if (err)
554 break;
555 }
556 return err;
557}
558
Paul Jacksonc61afb12006-03-24 03:16:08 -0800559/*
560 * Update task->flags PF_MEMPOLICY bit: set iff non-default
561 * mempolicy. Allows more rapid checking of this (combined perhaps
562 * with other PF_* flag bits) on memory allocation hot code paths.
563 *
564 * If called from outside this file, the task 'p' should -only- be
565 * a newly forked child not yet visible on the task list, because
566 * manipulating the task flags of a visible task is not safe.
567 *
568 * The above limitation is why this routine has the funny name
569 * mpol_fix_fork_child_flag().
570 *
571 * It is also safe to call this with a task pointer of current,
572 * which the static wrapper mpol_set_task_struct_flag() does,
573 * for use within this file.
574 */
575
576void mpol_fix_fork_child_flag(struct task_struct *p)
577{
578 if (p->mempolicy)
579 p->flags |= PF_MEMPOLICY;
580 else
581 p->flags &= ~PF_MEMPOLICY;
582}
583
584static void mpol_set_task_struct_flag(void)
585{
586 mpol_fix_fork_child_flag(current);
587}
588
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589/* Set the process memory policy */
David Rientjes028fec42008-04-28 02:12:25 -0700590static long do_set_mempolicy(unsigned short mode, unsigned short flags,
591 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 struct mempolicy *new;
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700594 struct mm_struct *mm = current->mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
David Rientjes028fec42008-04-28 02:12:25 -0700596 new = mpol_new(mode, flags, nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 if (IS_ERR(new))
598 return PTR_ERR(new);
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700599
600 /*
601 * prevent changing our mempolicy while show_numa_maps()
602 * is using it.
603 * Note: do_set_mempolicy() can be called at init time
604 * with no 'mm'.
605 */
606 if (mm)
607 down_write(&mm->mmap_sem);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700608 mpol_put(current->mempolicy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 current->mempolicy = new;
Paul Jacksonc61afb12006-03-24 03:16:08 -0800610 mpol_set_task_struct_flag();
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700611 if (new && new->mode == MPOL_INTERLEAVE &&
David Rientjesf5b087b2008-04-28 02:12:27 -0700612 nodes_weight(new->v.nodes))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700613 current->il_next = first_node(new->v.nodes);
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700614 if (mm)
615 up_write(&mm->mmap_sem);
616
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 return 0;
618}
619
620/* Fill a zone bitmap for a policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700621static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700623 nodes_clear(*nodes);
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700624 switch (p->mode) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 case MPOL_DEFAULT:
626 break;
Mel Gorman19770b32008-04-28 02:12:18 -0700627 case MPOL_BIND:
628 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700630 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 break;
632 case MPOL_PREFERRED:
Christoph Lameter56bbd652007-10-16 01:25:35 -0700633 /* or use current node instead of memory_map? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 if (p->v.preferred_node < 0)
Christoph Lameter56bbd652007-10-16 01:25:35 -0700635 *nodes = node_states[N_HIGH_MEMORY];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 else
Andi Kleendfcd3c02005-10-29 18:15:48 -0700637 node_set(p->v.preferred_node, *nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 break;
639 default:
640 BUG();
641 }
642}
643
644static int lookup_node(struct mm_struct *mm, unsigned long addr)
645{
646 struct page *p;
647 int err;
648
649 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
650 if (err >= 0) {
651 err = page_to_nid(p);
652 put_page(p);
653 }
654 return err;
655}
656
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657/* Retrieve NUMA policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700658static long do_get_mempolicy(int *policy, nodemask_t *nmask,
659 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700661 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 struct mm_struct *mm = current->mm;
663 struct vm_area_struct *vma = NULL;
664 struct mempolicy *pol = current->mempolicy;
665
Paul Jacksoncf2a473c2006-01-08 01:01:54 -0800666 cpuset_update_task_memory_state();
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700667 if (flags &
668 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 return -EINVAL;
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700670
671 if (flags & MPOL_F_MEMS_ALLOWED) {
672 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
673 return -EINVAL;
674 *policy = 0; /* just so it's initialized */
675 *nmask = cpuset_current_mems_allowed;
676 return 0;
677 }
678
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 if (flags & MPOL_F_ADDR) {
680 down_read(&mm->mmap_sem);
681 vma = find_vma_intersection(mm, addr, addr+1);
682 if (!vma) {
683 up_read(&mm->mmap_sem);
684 return -EFAULT;
685 }
686 if (vma->vm_ops && vma->vm_ops->get_policy)
687 pol = vma->vm_ops->get_policy(vma, addr);
688 else
689 pol = vma->vm_policy;
690 } else if (addr)
691 return -EINVAL;
692
693 if (!pol)
694 pol = &default_policy;
695
696 if (flags & MPOL_F_NODE) {
697 if (flags & MPOL_F_ADDR) {
698 err = lookup_node(mm, addr);
699 if (err < 0)
700 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700701 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 } else if (pol == current->mempolicy &&
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700703 pol->mode == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700704 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 } else {
706 err = -EINVAL;
707 goto out;
708 }
709 } else
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700710 *policy = pol->mode | pol->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711
712 if (vma) {
713 up_read(&current->mm->mmap_sem);
714 vma = NULL;
715 }
716
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 err = 0;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700718 if (nmask)
719 get_zonemask(pol, nmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
721 out:
722 if (vma)
723 up_read(&current->mm->mmap_sem);
724 return err;
725}
726
Christoph Lameterb20a3502006-03-22 00:09:12 -0800727#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700728/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800729 * page migration
730 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800731static void migrate_page_add(struct page *page, struct list_head *pagelist,
732 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800733{
734 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800735 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800736 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800737 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
738 isolate_lru_page(page, pagelist);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800739}
740
Christoph Lameter742755a2006-06-23 02:03:55 -0700741static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700742{
Mel Gorman769848c2007-07-17 04:03:05 -0700743 return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700744}
745
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800746/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800747 * Migrate pages from one node to a target node.
748 * Returns error or the number of pages not migrated.
749 */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700750static int migrate_to_node(struct mm_struct *mm, int source, int dest,
751 int flags)
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800752{
753 nodemask_t nmask;
754 LIST_HEAD(pagelist);
755 int err = 0;
756
757 nodes_clear(nmask);
758 node_set(source, nmask);
759
760 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
761 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
762
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700763 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700764 err = migrate_pages(&pagelist, new_node_page, dest);
765
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800766 return err;
767}
768
769/*
770 * Move pages between the two nodesets so as to preserve the physical
771 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -0800772 *
773 * Returns the number of page that could not be moved.
774 */
775int do_migrate_pages(struct mm_struct *mm,
776 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
777{
778 LIST_HEAD(pagelist);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800779 int busy = 0;
780 int err = 0;
781 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -0800782
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800783 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -0800784
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700785 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
786 if (err)
787 goto out;
788
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800789/*
790 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
791 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
792 * bit in 'tmp', and return that <source, dest> pair for migration.
793 * The pair of nodemasks 'to' and 'from' define the map.
794 *
795 * If no pair of bits is found that way, fallback to picking some
796 * pair of 'source' and 'dest' bits that are not the same. If the
797 * 'source' and 'dest' bits are the same, this represents a node
798 * that will be migrating to itself, so no pages need move.
799 *
800 * If no bits are left in 'tmp', or if all remaining bits left
801 * in 'tmp' correspond to the same bit in 'to', return false
802 * (nothing left to migrate).
803 *
804 * This lets us pick a pair of nodes to migrate between, such that
805 * if possible the dest node is not already occupied by some other
806 * source node, minimizing the risk of overloading the memory on a
807 * node that would happen if we migrated incoming memory to a node
808 * before migrating outgoing memory source that same node.
809 *
810 * A single scan of tmp is sufficient. As we go, we remember the
811 * most recent <s, d> pair that moved (s != d). If we find a pair
812 * that not only moved, but what's better, moved to an empty slot
813 * (d is not set in tmp), then we break out then, with that pair.
814 * Otherwise when we finish scannng from_tmp, we at least have the
815 * most recent <s, d> pair that moved. If we get all the way through
816 * the scan of tmp without finding any node that moved, much less
817 * moved to an empty node, then there is nothing left worth migrating.
818 */
Christoph Lameterd4984712006-01-08 01:00:55 -0800819
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800820 tmp = *from_nodes;
821 while (!nodes_empty(tmp)) {
822 int s,d;
823 int source = -1;
824 int dest = 0;
825
826 for_each_node_mask(s, tmp) {
827 d = node_remap(s, *from_nodes, *to_nodes);
828 if (s == d)
829 continue;
830
831 source = s; /* Node moved. Memorize */
832 dest = d;
833
834 /* dest not in remaining from nodes? */
835 if (!node_isset(dest, tmp))
836 break;
837 }
838 if (source == -1)
839 break;
840
841 node_clear(source, tmp);
842 err = migrate_to_node(mm, source, dest, flags);
843 if (err > 0)
844 busy += err;
845 if (err < 0)
846 break;
Christoph Lameter39743882006-01-08 01:00:51 -0800847 }
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700848out:
Christoph Lameter39743882006-01-08 01:00:51 -0800849 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800850 if (err < 0)
851 return err;
852 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800853
Christoph Lameter39743882006-01-08 01:00:51 -0800854}
855
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800856/*
857 * Allocate a new page for page migration based on vma policy.
858 * Start assuming that page is mapped by vma pointed to by @private.
859 * Search forward from there, if not. N.B., this assumes that the
860 * list of pages handed to migrate_pages()--which is how we get here--
861 * is in virtual address order.
862 */
Christoph Lameter742755a2006-06-23 02:03:55 -0700863static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700864{
865 struct vm_area_struct *vma = (struct vm_area_struct *)private;
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800866 unsigned long uninitialized_var(address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700867
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800868 while (vma) {
869 address = page_address_in_vma(page, vma);
870 if (address != -EFAULT)
871 break;
872 vma = vma->vm_next;
873 }
874
875 /*
876 * if !vma, alloc_page_vma() will use task or system default policy
877 */
878 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700879}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800880#else
881
882static void migrate_page_add(struct page *page, struct list_head *pagelist,
883 unsigned long flags)
884{
885}
886
887int do_migrate_pages(struct mm_struct *mm,
888 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
889{
890 return -ENOSYS;
891}
Christoph Lameter95a402c2006-06-23 02:03:53 -0700892
Keith Owens69939742006-10-11 01:21:28 -0700893static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700894{
895 return NULL;
896}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800897#endif
898
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700899static long do_mbind(unsigned long start, unsigned long len,
David Rientjes028fec42008-04-28 02:12:25 -0700900 unsigned short mode, unsigned short mode_flags,
901 nodemask_t *nmask, unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800902{
903 struct vm_area_struct *vma;
904 struct mm_struct *mm = current->mm;
905 struct mempolicy *new;
906 unsigned long end;
907 int err;
908 LIST_HEAD(pagelist);
909
David Rientjesa3b51e02008-04-28 02:12:23 -0700910 if (flags & ~(unsigned long)(MPOL_MF_STRICT |
911 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800912 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -0800913 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800914 return -EPERM;
915
916 if (start & ~PAGE_MASK)
917 return -EINVAL;
918
919 if (mode == MPOL_DEFAULT)
920 flags &= ~MPOL_MF_STRICT;
921
922 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
923 end = start + len;
924
925 if (end < start)
926 return -EINVAL;
927 if (end == start)
928 return 0;
929
David Rientjes028fec42008-04-28 02:12:25 -0700930 new = mpol_new(mode, mode_flags, nmask);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800931 if (IS_ERR(new))
932 return PTR_ERR(new);
933
934 /*
935 * If we are using the default policy then operation
936 * on discontinuous address spaces is okay after all
937 */
938 if (!new)
939 flags |= MPOL_MF_DISCONTIG_OK;
940
David Rientjes028fec42008-04-28 02:12:25 -0700941 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
942 start, start + len, mode, mode_flags,
943 nmask ? nodes_addr(*nmask)[0] : -1);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800944
945 down_write(&mm->mmap_sem);
946 vma = check_range(mm, start, end, nmask,
947 flags | MPOL_MF_INVERT, &pagelist);
948
949 err = PTR_ERR(vma);
950 if (!IS_ERR(vma)) {
951 int nr_failed = 0;
952
953 err = mbind_range(vma, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800954
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800955 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700956 nr_failed = migrate_pages(&pagelist, new_vma_page,
957 (unsigned long)vma);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800958
959 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
960 err = -EIO;
961 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800962
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800963 up_write(&mm->mmap_sem);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700964 mpol_put(new);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800965 return err;
966}
967
Christoph Lameter39743882006-01-08 01:00:51 -0800968/*
Christoph Lameter8bccd852005-10-29 18:16:59 -0700969 * User space interface with variable sized bitmaps for nodelists.
970 */
971
972/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -0800973static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -0700974 unsigned long maxnode)
975{
976 unsigned long k;
977 unsigned long nlongs;
978 unsigned long endmask;
979
980 --maxnode;
981 nodes_clear(*nodes);
982 if (maxnode == 0 || !nmask)
983 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -0800984 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -0800985 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700986
987 nlongs = BITS_TO_LONGS(maxnode);
988 if ((maxnode % BITS_PER_LONG) == 0)
989 endmask = ~0UL;
990 else
991 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
992
993 /* When the user specified more nodes than supported just check
994 if the non supported part is all zero. */
995 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
996 if (nlongs > PAGE_SIZE/sizeof(long))
997 return -EINVAL;
998 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
999 unsigned long t;
1000 if (get_user(t, nmask + k))
1001 return -EFAULT;
1002 if (k == nlongs - 1) {
1003 if (t & endmask)
1004 return -EINVAL;
1005 } else if (t)
1006 return -EINVAL;
1007 }
1008 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1009 endmask = ~0UL;
1010 }
1011
1012 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1013 return -EFAULT;
1014 nodes_addr(*nodes)[nlongs-1] &= endmask;
1015 return 0;
1016}
1017
1018/* Copy a kernel node mask to user space */
1019static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1020 nodemask_t *nodes)
1021{
1022 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1023 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1024
1025 if (copy > nbytes) {
1026 if (copy > PAGE_SIZE)
1027 return -EINVAL;
1028 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1029 return -EFAULT;
1030 copy = nbytes;
1031 }
1032 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1033}
1034
1035asmlinkage long sys_mbind(unsigned long start, unsigned long len,
1036 unsigned long mode,
1037 unsigned long __user *nmask, unsigned long maxnode,
1038 unsigned flags)
1039{
1040 nodemask_t nodes;
1041 int err;
David Rientjes028fec42008-04-28 02:12:25 -07001042 unsigned short mode_flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001043
David Rientjes028fec42008-04-28 02:12:25 -07001044 mode_flags = mode & MPOL_MODE_FLAGS;
1045 mode &= ~MPOL_MODE_FLAGS;
David Rientjesa3b51e02008-04-28 02:12:23 -07001046 if (mode >= MPOL_MAX)
1047 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001048 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1049 (mode_flags & MPOL_F_RELATIVE_NODES))
1050 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001051 err = get_nodes(&nodes, nmask, maxnode);
1052 if (err)
1053 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001054 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001055}
1056
1057/* Set the process memory policy */
1058asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
1059 unsigned long maxnode)
1060{
1061 int err;
1062 nodemask_t nodes;
David Rientjes028fec42008-04-28 02:12:25 -07001063 unsigned short flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001064
David Rientjes028fec42008-04-28 02:12:25 -07001065 flags = mode & MPOL_MODE_FLAGS;
1066 mode &= ~MPOL_MODE_FLAGS;
1067 if ((unsigned int)mode >= MPOL_MAX)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001068 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001069 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1070 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001071 err = get_nodes(&nodes, nmask, maxnode);
1072 if (err)
1073 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001074 return do_set_mempolicy(mode, flags, &nodes);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001075}
1076
Christoph Lameter39743882006-01-08 01:00:51 -08001077asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
1078 const unsigned long __user *old_nodes,
1079 const unsigned long __user *new_nodes)
1080{
1081 struct mm_struct *mm;
1082 struct task_struct *task;
1083 nodemask_t old;
1084 nodemask_t new;
1085 nodemask_t task_nodes;
1086 int err;
1087
1088 err = get_nodes(&old, old_nodes, maxnode);
1089 if (err)
1090 return err;
1091
1092 err = get_nodes(&new, new_nodes, maxnode);
1093 if (err)
1094 return err;
1095
1096 /* Find the mm_struct */
1097 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001098 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter39743882006-01-08 01:00:51 -08001099 if (!task) {
1100 read_unlock(&tasklist_lock);
1101 return -ESRCH;
1102 }
1103 mm = get_task_mm(task);
1104 read_unlock(&tasklist_lock);
1105
1106 if (!mm)
1107 return -EINVAL;
1108
1109 /*
1110 * Check if this process has the right to modify the specified
1111 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -08001112 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -08001113 * userid as the target process.
1114 */
1115 if ((current->euid != task->suid) && (current->euid != task->uid) &&
1116 (current->uid != task->suid) && (current->uid != task->uid) &&
Christoph Lameter74c00242006-03-14 19:50:21 -08001117 !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001118 err = -EPERM;
1119 goto out;
1120 }
1121
1122 task_nodes = cpuset_mems_allowed(task);
1123 /* Is the user allowed to access the target nodes? */
Christoph Lameter74c00242006-03-14 19:50:21 -08001124 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001125 err = -EPERM;
1126 goto out;
1127 }
1128
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07001129 if (!nodes_subset(new, node_states[N_HIGH_MEMORY])) {
Christoph Lameter3b42d282007-08-31 00:12:08 -07001130 err = -EINVAL;
1131 goto out;
1132 }
1133
David Quigley86c3a762006-06-23 02:04:02 -07001134 err = security_task_movememory(task);
1135 if (err)
1136 goto out;
1137
Christoph Lameter511030b2006-02-28 16:58:57 -08001138 err = do_migrate_pages(mm, &old, &new,
Christoph Lameter74c00242006-03-14 19:50:21 -08001139 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter39743882006-01-08 01:00:51 -08001140out:
1141 mmput(mm);
1142 return err;
1143}
1144
1145
Christoph Lameter8bccd852005-10-29 18:16:59 -07001146/* Retrieve NUMA policy */
1147asmlinkage long sys_get_mempolicy(int __user *policy,
1148 unsigned long __user *nmask,
1149 unsigned long maxnode,
1150 unsigned long addr, unsigned long flags)
1151{
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001152 int err;
1153 int uninitialized_var(pval);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001154 nodemask_t nodes;
1155
1156 if (nmask != NULL && maxnode < MAX_NUMNODES)
1157 return -EINVAL;
1158
1159 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1160
1161 if (err)
1162 return err;
1163
1164 if (policy && put_user(pval, policy))
1165 return -EFAULT;
1166
1167 if (nmask)
1168 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1169
1170 return err;
1171}
1172
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173#ifdef CONFIG_COMPAT
1174
1175asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1176 compat_ulong_t __user *nmask,
1177 compat_ulong_t maxnode,
1178 compat_ulong_t addr, compat_ulong_t flags)
1179{
1180 long err;
1181 unsigned long __user *nm = NULL;
1182 unsigned long nr_bits, alloc_size;
1183 DECLARE_BITMAP(bm, MAX_NUMNODES);
1184
1185 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1186 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1187
1188 if (nmask)
1189 nm = compat_alloc_user_space(alloc_size);
1190
1191 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1192
1193 if (!err && nmask) {
1194 err = copy_from_user(bm, nm, alloc_size);
1195 /* ensure entire bitmap is zeroed */
1196 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1197 err |= compat_put_bitmap(nmask, bm, nr_bits);
1198 }
1199
1200 return err;
1201}
1202
1203asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1204 compat_ulong_t maxnode)
1205{
1206 long err = 0;
1207 unsigned long __user *nm = NULL;
1208 unsigned long nr_bits, alloc_size;
1209 DECLARE_BITMAP(bm, MAX_NUMNODES);
1210
1211 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1212 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1213
1214 if (nmask) {
1215 err = compat_get_bitmap(bm, nmask, nr_bits);
1216 nm = compat_alloc_user_space(alloc_size);
1217 err |= copy_to_user(nm, bm, alloc_size);
1218 }
1219
1220 if (err)
1221 return -EFAULT;
1222
1223 return sys_set_mempolicy(mode, nm, nr_bits+1);
1224}
1225
1226asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1227 compat_ulong_t mode, compat_ulong_t __user *nmask,
1228 compat_ulong_t maxnode, compat_ulong_t flags)
1229{
1230 long err = 0;
1231 unsigned long __user *nm = NULL;
1232 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001233 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234
1235 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1236 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1237
1238 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001239 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001241 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 }
1243
1244 if (err)
1245 return -EFAULT;
1246
1247 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1248}
1249
1250#endif
1251
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001252/*
1253 * get_vma_policy(@task, @vma, @addr)
1254 * @task - task for fallback if vma policy == default
1255 * @vma - virtual memory area whose policy is sought
1256 * @addr - address in @vma for shared policy lookup
1257 *
1258 * Returns effective policy for a VMA at specified address.
1259 * Falls back to @task or system default policy, as necessary.
1260 * Returned policy has extra reference count if shared, vma,
1261 * or some other task's policy [show_numa_maps() can pass
1262 * @task != current]. It is the caller's responsibility to
1263 * free the reference in these cases.
1264 */
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001265static struct mempolicy *get_vma_policy(struct task_struct *task,
Christoph Lameter48fce342006-01-08 01:01:03 -08001266 struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001268 struct mempolicy *pol = task->mempolicy;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001269 int shared_pol = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270
1271 if (vma) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001272 if (vma->vm_ops && vma->vm_ops->get_policy) {
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001273 struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
1274 addr);
1275 if (vpol)
1276 pol = vpol;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001277 shared_pol = 1; /* if pol non-NULL, add ref below */
1278 } else if (vma->vm_policy &&
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001279 vma->vm_policy->mode != MPOL_DEFAULT)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280 pol = vma->vm_policy;
1281 }
1282 if (!pol)
1283 pol = &default_policy;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001284 else if (!shared_pol && pol != current->mempolicy)
1285 mpol_get(pol); /* vma or other task's policy */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 return pol;
1287}
1288
Mel Gorman19770b32008-04-28 02:12:18 -07001289/* Return a nodemask representing a mempolicy */
1290static nodemask_t *nodemask_policy(gfp_t gfp, struct mempolicy *policy)
1291{
1292 /* Lower zones don't get a nodemask applied for MPOL_BIND */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001293 if (unlikely(policy->mode == MPOL_BIND) &&
Mel Gorman19770b32008-04-28 02:12:18 -07001294 gfp_zone(gfp) >= policy_zone &&
1295 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1296 return &policy->v.nodes;
1297
1298 return NULL;
1299}
1300
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301/* Return a zonelist representing a mempolicy */
Al Virodd0fc662005-10-07 07:46:04 +01001302static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303{
1304 int nd;
1305
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001306 switch (policy->mode) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307 case MPOL_PREFERRED:
1308 nd = policy->v.preferred_node;
1309 if (nd < 0)
1310 nd = numa_node_id();
1311 break;
1312 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07001313 /*
1314 * Normally, MPOL_BIND allocations node-local are node-local
1315 * within the allowed nodemask. However, if __GFP_THISNODE is
1316 * set and the current node is part of the mask, we use the
1317 * the zonelist for the first node in the mask instead.
1318 */
1319 nd = numa_node_id();
1320 if (unlikely(gfp & __GFP_THISNODE) &&
1321 unlikely(!node_isset(nd, policy->v.nodes)))
1322 nd = first_node(policy->v.nodes);
1323 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 case MPOL_INTERLEAVE: /* should not happen */
1325 case MPOL_DEFAULT:
1326 nd = numa_node_id();
1327 break;
1328 default:
1329 nd = 0;
1330 BUG();
1331 }
Mel Gorman0e884602008-04-28 02:12:14 -07001332 return node_zonelist(nd, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333}
1334
1335/* Do dynamic interleaving for a process */
1336static unsigned interleave_nodes(struct mempolicy *policy)
1337{
1338 unsigned nid, next;
1339 struct task_struct *me = current;
1340
1341 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001342 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001344 next = first_node(policy->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001345 if (next < MAX_NUMNODES)
1346 me->il_next = next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 return nid;
1348}
1349
Christoph Lameterdc85da12006-01-18 17:42:36 -08001350/*
1351 * Depending on the memory policy provide a node from which to allocate the
1352 * next slab entry.
1353 */
1354unsigned slab_node(struct mempolicy *policy)
1355{
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001356 unsigned short pol = policy ? policy->mode : MPOL_DEFAULT;
Christoph Lameter765c4502006-09-27 01:50:08 -07001357
1358 switch (pol) {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001359 case MPOL_INTERLEAVE:
1360 return interleave_nodes(policy);
1361
Mel Gormandd1a2392008-04-28 02:12:17 -07001362 case MPOL_BIND: {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001363 /*
1364 * Follow bind policy behavior and start allocation at the
1365 * first node.
1366 */
Mel Gorman19770b32008-04-28 02:12:18 -07001367 struct zonelist *zonelist;
1368 struct zone *zone;
1369 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1370 zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0];
1371 (void)first_zones_zonelist(zonelist, highest_zoneidx,
1372 &policy->v.nodes,
1373 &zone);
1374 return zone->node;
Mel Gormandd1a2392008-04-28 02:12:17 -07001375 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08001376
1377 case MPOL_PREFERRED:
1378 if (policy->v.preferred_node >= 0)
1379 return policy->v.preferred_node;
1380 /* Fall through */
1381
1382 default:
1383 return numa_node_id();
1384 }
1385}
1386
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387/* Do static interleaving for a VMA with known offset. */
1388static unsigned offset_il_node(struct mempolicy *pol,
1389 struct vm_area_struct *vma, unsigned long off)
1390{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001391 unsigned nnodes = nodes_weight(pol->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001392 unsigned target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393 int c;
1394 int nid = -1;
1395
David Rientjesf5b087b2008-04-28 02:12:27 -07001396 if (!nnodes)
1397 return numa_node_id();
1398 target = (unsigned int)off % nnodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399 c = 0;
1400 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001401 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402 c++;
1403 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001404 return nid;
1405}
1406
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001407/* Determine a node number for interleave */
1408static inline unsigned interleave_nid(struct mempolicy *pol,
1409 struct vm_area_struct *vma, unsigned long addr, int shift)
1410{
1411 if (vma) {
1412 unsigned long off;
1413
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001414 /*
1415 * for small pages, there is no difference between
1416 * shift and PAGE_SHIFT, so the bit-shift is safe.
1417 * for huge pages, since vm_pgoff is in units of small
1418 * pages, we need to shift off the always 0 bits to get
1419 * a useful offset.
1420 */
1421 BUG_ON(shift < PAGE_SHIFT);
1422 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001423 off += (addr - vma->vm_start) >> shift;
1424 return offset_il_node(pol, vma, off);
1425 } else
1426 return interleave_nodes(pol);
1427}
1428
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001429#ifdef CONFIG_HUGETLBFS
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001430/*
1431 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1432 * @vma = virtual memory area whose policy is sought
1433 * @addr = address in @vma for shared policy lookup and interleave policy
1434 * @gfp_flags = for requested zone
Mel Gorman19770b32008-04-28 02:12:18 -07001435 * @mpol = pointer to mempolicy pointer for reference counted mempolicy
1436 * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001437 *
1438 * Returns a zonelist suitable for a huge page allocation.
Mel Gorman19770b32008-04-28 02:12:18 -07001439 * If the effective policy is 'BIND, returns pointer to local node's zonelist,
1440 * and a pointer to the mempolicy's @nodemask for filtering the zonelist.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001441 * If it is also a policy for which get_vma_policy() returns an extra
Mel Gorman19770b32008-04-28 02:12:18 -07001442 * reference, we must hold that reference until after the allocation.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001443 * In that case, return policy via @mpol so hugetlb allocation can drop
Mel Gorman19770b32008-04-28 02:12:18 -07001444 * the reference. For non-'BIND referenced policies, we can/do drop the
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001445 * reference here, so the caller doesn't need to know about the special case
1446 * for default and current task policy.
1447 */
Mel Gorman396faf02007-07-17 04:03:13 -07001448struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
Mel Gorman19770b32008-04-28 02:12:18 -07001449 gfp_t gfp_flags, struct mempolicy **mpol,
1450 nodemask_t **nodemask)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001451{
1452 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001453 struct zonelist *zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001454
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001455 *mpol = NULL; /* probably no unref needed */
Mel Gorman19770b32008-04-28 02:12:18 -07001456 *nodemask = NULL; /* assume !MPOL_BIND */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001457 if (pol->mode == MPOL_BIND) {
Mel Gorman19770b32008-04-28 02:12:18 -07001458 *nodemask = &pol->v.nodes;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001459 } else if (pol->mode == MPOL_INTERLEAVE) {
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001460 unsigned nid;
1461
1462 nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
Lee Schermerhorn69682d82008-03-10 11:43:45 -07001463 if (unlikely(pol != &default_policy &&
1464 pol != current->mempolicy))
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001465 __mpol_put(pol); /* finished with pol */
Mel Gorman0e884602008-04-28 02:12:14 -07001466 return node_zonelist(nid, gfp_flags);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001467 }
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001468
1469 zl = zonelist_policy(GFP_HIGHUSER, pol);
1470 if (unlikely(pol != &default_policy && pol != current->mempolicy)) {
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001471 if (pol->mode != MPOL_BIND)
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001472 __mpol_put(pol); /* finished with pol */
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001473 else
1474 *mpol = pol; /* unref needed after allocation */
1475 }
1476 return zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001477}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001478#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001479
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480/* Allocate a page in interleaved policy.
1481 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001482static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1483 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484{
1485 struct zonelist *zl;
1486 struct page *page;
1487
Mel Gorman0e884602008-04-28 02:12:14 -07001488 zl = node_zonelist(nid, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 page = __alloc_pages(gfp, order, zl);
Mel Gormandd1a2392008-04-28 02:12:17 -07001490 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
Christoph Lameterca889e62006-06-30 01:55:44 -07001491 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492 return page;
1493}
1494
1495/**
1496 * alloc_page_vma - Allocate a page for a VMA.
1497 *
1498 * @gfp:
1499 * %GFP_USER user allocation.
1500 * %GFP_KERNEL kernel allocations,
1501 * %GFP_HIGHMEM highmem/user allocations,
1502 * %GFP_FS allocation should not call back into a file system.
1503 * %GFP_ATOMIC don't sleep.
1504 *
1505 * @vma: Pointer to VMA or NULL if not available.
1506 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1507 *
1508 * This function allocates a page from the kernel page pool and applies
1509 * a NUMA policy associated with the VMA or the current process.
1510 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1511 * mm_struct of the VMA to prevent it from going away. Should be used for
1512 * all allocations for pages that will be mapped into
1513 * user space. Returns NULL when no page can be allocated.
1514 *
1515 * Should be called with the mm_sem of the vma hold.
1516 */
1517struct page *
Al Virodd0fc662005-10-07 07:46:04 +01001518alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001520 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001521 struct zonelist *zl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
Paul Jacksoncf2a473c2006-01-08 01:01:54 -08001523 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001525 if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001527
1528 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
Lee Schermerhorn69682d82008-03-10 11:43:45 -07001529 if (unlikely(pol != &default_policy &&
1530 pol != current->mempolicy))
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001531 __mpol_put(pol); /* finished with pol */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 return alloc_page_interleave(gfp, 0, nid);
1533 }
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001534 zl = zonelist_policy(gfp, pol);
1535 if (pol != &default_policy && pol != current->mempolicy) {
1536 /*
1537 * slow path: ref counted policy -- shared or vma
1538 */
Mel Gorman19770b32008-04-28 02:12:18 -07001539 struct page *page = __alloc_pages_nodemask(gfp, 0,
1540 zl, nodemask_policy(gfp, pol));
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001541 __mpol_put(pol);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001542 return page;
1543 }
1544 /*
1545 * fast path: default or task policy
1546 */
Mel Gorman19770b32008-04-28 02:12:18 -07001547 return __alloc_pages_nodemask(gfp, 0, zl, nodemask_policy(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548}
1549
1550/**
1551 * alloc_pages_current - Allocate pages.
1552 *
1553 * @gfp:
1554 * %GFP_USER user allocation,
1555 * %GFP_KERNEL kernel allocation,
1556 * %GFP_HIGHMEM highmem allocation,
1557 * %GFP_FS don't call back into a file system.
1558 * %GFP_ATOMIC don't sleep.
1559 * @order: Power of two of allocation size in pages. 0 is a single page.
1560 *
1561 * Allocate a page from the kernel page pool. When not in
1562 * interrupt context and apply the current process NUMA policy.
1563 * Returns NULL when no page can be allocated.
1564 *
Paul Jacksoncf2a473c2006-01-08 01:01:54 -08001565 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566 * 1) it's ok to take cpuset_sem (can WAIT), and
1567 * 2) allocating for current task (not interrupt).
1568 */
Al Virodd0fc662005-10-07 07:46:04 +01001569struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570{
1571 struct mempolicy *pol = current->mempolicy;
1572
1573 if ((gfp & __GFP_WAIT) && !in_interrupt())
Paul Jacksoncf2a473c2006-01-08 01:01:54 -08001574 cpuset_update_task_memory_state();
Christoph Lameter9b819d22006-09-25 23:31:40 -07001575 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576 pol = &default_policy;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001577 if (pol->mode == MPOL_INTERLEAVE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
Mel Gorman19770b32008-04-28 02:12:18 -07001579 return __alloc_pages_nodemask(gfp, order,
1580 zonelist_policy(gfp, pol), nodemask_policy(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581}
1582EXPORT_SYMBOL(alloc_pages_current);
1583
Paul Jackson42253992006-01-08 01:01:59 -08001584/*
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001585 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
Paul Jackson42253992006-01-08 01:01:59 -08001586 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1587 * with the mems_allowed returned by cpuset_mems_allowed(). This
1588 * keeps mempolicies cpuset relative after its cpuset moves. See
1589 * further kernel/cpuset.c update_nodemask().
1590 */
Paul Jackson42253992006-01-08 01:01:59 -08001591
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001592/* Slow path of a mempolicy duplicate */
1593struct mempolicy *__mpol_dup(struct mempolicy *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594{
1595 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1596
1597 if (!new)
1598 return ERR_PTR(-ENOMEM);
Paul Jackson42253992006-01-08 01:01:59 -08001599 if (current_cpuset_is_being_rebound()) {
1600 nodemask_t mems = cpuset_mems_allowed(current);
1601 mpol_rebind_policy(old, &mems);
1602 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603 *new = *old;
1604 atomic_set(&new->refcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605 return new;
1606}
1607
David Rientjesf5b087b2008-04-28 02:12:27 -07001608static int mpol_match_intent(const struct mempolicy *a,
1609 const struct mempolicy *b)
1610{
1611 if (a->flags != b->flags)
1612 return 0;
1613 if (!mpol_store_user_nodemask(a))
1614 return 1;
1615 return nodes_equal(a->w.user_nodemask, b->w.user_nodemask);
1616}
1617
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618/* Slow path of a mempolicy comparison */
1619int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1620{
1621 if (!a || !b)
1622 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001623 if (a->mode != b->mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001625 if (a->mode != MPOL_DEFAULT && !mpol_match_intent(a, b))
David Rientjesf5b087b2008-04-28 02:12:27 -07001626 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001627 switch (a->mode) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628 case MPOL_DEFAULT:
1629 return 1;
Mel Gorman19770b32008-04-28 02:12:18 -07001630 case MPOL_BIND:
1631 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -07001633 return nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001634 case MPOL_PREFERRED:
1635 return a->v.preferred_node == b->v.preferred_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001636 default:
1637 BUG();
1638 return 0;
1639 }
1640}
1641
1642/* Slow path of a mpol destructor. */
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001643void __mpol_put(struct mempolicy *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001644{
1645 if (!atomic_dec_and_test(&p->refcnt))
1646 return;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001647 p->mode = MPOL_DEFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648 kmem_cache_free(policy_cache, p);
1649}
1650
1651/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 * Shared memory backing store policy support.
1653 *
1654 * Remember policies even when nobody has shared memory mapped.
1655 * The policies are kept in Red-Black tree linked from the inode.
1656 * They are protected by the sp->lock spinlock, which should be held
1657 * for any accesses to the tree.
1658 */
1659
1660/* lookup first element intersecting start-end */
1661/* Caller holds sp->lock */
1662static struct sp_node *
1663sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1664{
1665 struct rb_node *n = sp->root.rb_node;
1666
1667 while (n) {
1668 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1669
1670 if (start >= p->end)
1671 n = n->rb_right;
1672 else if (end <= p->start)
1673 n = n->rb_left;
1674 else
1675 break;
1676 }
1677 if (!n)
1678 return NULL;
1679 for (;;) {
1680 struct sp_node *w = NULL;
1681 struct rb_node *prev = rb_prev(n);
1682 if (!prev)
1683 break;
1684 w = rb_entry(prev, struct sp_node, nd);
1685 if (w->end <= start)
1686 break;
1687 n = prev;
1688 }
1689 return rb_entry(n, struct sp_node, nd);
1690}
1691
1692/* Insert a new shared policy into the list. */
1693/* Caller holds sp->lock */
1694static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1695{
1696 struct rb_node **p = &sp->root.rb_node;
1697 struct rb_node *parent = NULL;
1698 struct sp_node *nd;
1699
1700 while (*p) {
1701 parent = *p;
1702 nd = rb_entry(parent, struct sp_node, nd);
1703 if (new->start < nd->start)
1704 p = &(*p)->rb_left;
1705 else if (new->end > nd->end)
1706 p = &(*p)->rb_right;
1707 else
1708 BUG();
1709 }
1710 rb_link_node(&new->nd, parent, p);
1711 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07001712 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001713 new->policy ? new->policy->mode : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714}
1715
1716/* Find shared policy intersecting idx */
1717struct mempolicy *
1718mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1719{
1720 struct mempolicy *pol = NULL;
1721 struct sp_node *sn;
1722
1723 if (!sp->root.rb_node)
1724 return NULL;
1725 spin_lock(&sp->lock);
1726 sn = sp_lookup(sp, idx, idx+1);
1727 if (sn) {
1728 mpol_get(sn->policy);
1729 pol = sn->policy;
1730 }
1731 spin_unlock(&sp->lock);
1732 return pol;
1733}
1734
1735static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1736{
Paul Mundt140d5a42007-07-15 23:38:16 -07001737 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738 rb_erase(&n->nd, &sp->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001739 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 kmem_cache_free(sn_cache, n);
1741}
1742
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001743static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
1744 struct mempolicy *pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745{
1746 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1747
1748 if (!n)
1749 return NULL;
1750 n->start = start;
1751 n->end = end;
1752 mpol_get(pol);
Lee Schermerhornaab0b102008-04-28 02:13:13 -07001753 pol->flags |= MPOL_F_SHARED; /* for unref */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754 n->policy = pol;
1755 return n;
1756}
1757
1758/* Replace a policy range. */
1759static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1760 unsigned long end, struct sp_node *new)
1761{
1762 struct sp_node *n, *new2 = NULL;
1763
1764restart:
1765 spin_lock(&sp->lock);
1766 n = sp_lookup(sp, start, end);
1767 /* Take care of old policies in the same range. */
1768 while (n && n->start < end) {
1769 struct rb_node *next = rb_next(&n->nd);
1770 if (n->start >= start) {
1771 if (n->end <= end)
1772 sp_delete(sp, n);
1773 else
1774 n->start = end;
1775 } else {
1776 /* Old policy spanning whole new range. */
1777 if (n->end > end) {
1778 if (!new2) {
1779 spin_unlock(&sp->lock);
1780 new2 = sp_alloc(end, n->end, n->policy);
1781 if (!new2)
1782 return -ENOMEM;
1783 goto restart;
1784 }
1785 n->end = start;
1786 sp_insert(sp, new2);
1787 new2 = NULL;
1788 break;
1789 } else
1790 n->end = start;
1791 }
1792 if (!next)
1793 break;
1794 n = rb_entry(next, struct sp_node, nd);
1795 }
1796 if (new)
1797 sp_insert(sp, new);
1798 spin_unlock(&sp->lock);
1799 if (new2) {
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001800 mpol_put(new2->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801 kmem_cache_free(sn_cache, new2);
1802 }
1803 return 0;
1804}
1805
David Rientjesa3b51e02008-04-28 02:12:23 -07001806void mpol_shared_policy_init(struct shared_policy *info, unsigned short policy,
David Rientjes028fec42008-04-28 02:12:25 -07001807 unsigned short flags, nodemask_t *policy_nodes)
Robin Holt7339ff82006-01-14 13:20:48 -08001808{
1809 info->root = RB_ROOT;
1810 spin_lock_init(&info->lock);
1811
1812 if (policy != MPOL_DEFAULT) {
1813 struct mempolicy *newpol;
1814
1815 /* Falls back to MPOL_DEFAULT on any error */
David Rientjes028fec42008-04-28 02:12:25 -07001816 newpol = mpol_new(policy, flags, policy_nodes);
Robin Holt7339ff82006-01-14 13:20:48 -08001817 if (!IS_ERR(newpol)) {
1818 /* Create pseudo-vma that contains just the policy */
1819 struct vm_area_struct pvma;
1820
1821 memset(&pvma, 0, sizeof(struct vm_area_struct));
1822 /* Policy covers entire file */
1823 pvma.vm_end = TASK_SIZE;
1824 mpol_set_shared_policy(info, &pvma, newpol);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001825 mpol_put(newpol);
Robin Holt7339ff82006-01-14 13:20:48 -08001826 }
1827 }
1828}
1829
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830int mpol_set_shared_policy(struct shared_policy *info,
1831 struct vm_area_struct *vma, struct mempolicy *npol)
1832{
1833 int err;
1834 struct sp_node *new = NULL;
1835 unsigned long sz = vma_pages(vma);
1836
David Rientjes028fec42008-04-28 02:12:25 -07001837 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838 vma->vm_pgoff,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001839 sz, npol ? npol->mode : -1,
David Rientjes028fec42008-04-28 02:12:25 -07001840 npol ? npol->flags : -1,
Paul Mundt140d5a42007-07-15 23:38:16 -07001841 npol ? nodes_addr(npol->v.nodes)[0] : -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842
1843 if (npol) {
1844 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1845 if (!new)
1846 return -ENOMEM;
1847 }
1848 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1849 if (err && new)
1850 kmem_cache_free(sn_cache, new);
1851 return err;
1852}
1853
1854/* Free a backing policy store on inode delete. */
1855void mpol_free_shared_policy(struct shared_policy *p)
1856{
1857 struct sp_node *n;
1858 struct rb_node *next;
1859
1860 if (!p->root.rb_node)
1861 return;
1862 spin_lock(&p->lock);
1863 next = rb_first(&p->root);
1864 while (next) {
1865 n = rb_entry(next, struct sp_node, nd);
1866 next = rb_next(&n->nd);
Andi Kleen90c50292005-07-27 11:43:50 -07001867 rb_erase(&n->nd, &p->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001868 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869 kmem_cache_free(sn_cache, n);
1870 }
1871 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872}
1873
1874/* assumes fs == KERNEL_DS */
1875void __init numa_policy_init(void)
1876{
Paul Mundtb71636e2007-07-15 23:38:15 -07001877 nodemask_t interleave_nodes;
1878 unsigned long largest = 0;
1879 int nid, prefer = 0;
1880
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881 policy_cache = kmem_cache_create("numa_policy",
1882 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09001883 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884
1885 sn_cache = kmem_cache_create("shared_policy_node",
1886 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09001887 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001888
Paul Mundtb71636e2007-07-15 23:38:15 -07001889 /*
1890 * Set interleaving policy for system init. Interleaving is only
1891 * enabled across suitably sized nodes (default is >= 16MB), or
1892 * fall back to the largest node if they're all smaller.
1893 */
1894 nodes_clear(interleave_nodes);
Christoph Lameter56bbd652007-10-16 01:25:35 -07001895 for_each_node_state(nid, N_HIGH_MEMORY) {
Paul Mundtb71636e2007-07-15 23:38:15 -07001896 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897
Paul Mundtb71636e2007-07-15 23:38:15 -07001898 /* Preserve the largest node */
1899 if (largest < total_pages) {
1900 largest = total_pages;
1901 prefer = nid;
1902 }
1903
1904 /* Interleave this node? */
1905 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
1906 node_set(nid, interleave_nodes);
1907 }
1908
1909 /* All too small, use the largest */
1910 if (unlikely(nodes_empty(interleave_nodes)))
1911 node_set(prefer, interleave_nodes);
1912
David Rientjes028fec42008-04-28 02:12:25 -07001913 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914 printk("numa_policy_init: interleaving failed\n");
1915}
1916
Christoph Lameter8bccd852005-10-29 18:16:59 -07001917/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918void numa_default_policy(void)
1919{
David Rientjes028fec42008-04-28 02:12:25 -07001920 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921}
Paul Jackson68860ec2005-10-30 15:02:36 -08001922
Paul Jackson42253992006-01-08 01:01:59 -08001923/*
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001924 * Display pages allocated per node and memory policy via /proc.
1925 */
Helge Deller15ad7cd2006-12-06 20:40:36 -08001926static const char * const policy_types[] =
1927 { "default", "prefer", "bind", "interleave" };
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001928
1929/*
1930 * Convert a mempolicy into a string.
1931 * Returns the number of characters in buffer (if positive)
1932 * or an error (negative)
1933 */
1934static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
1935{
1936 char *p = buffer;
1937 int l;
1938 nodemask_t nodes;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001939 unsigned short mode = pol ? pol->mode : MPOL_DEFAULT;
David Rientjesf5b087b2008-04-28 02:12:27 -07001940 unsigned short flags = pol ? pol->flags : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001941
1942 switch (mode) {
1943 case MPOL_DEFAULT:
1944 nodes_clear(nodes);
1945 break;
1946
1947 case MPOL_PREFERRED:
1948 nodes_clear(nodes);
1949 node_set(pol->v.preferred_node, nodes);
1950 break;
1951
1952 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07001953 /* Fall through */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001954 case MPOL_INTERLEAVE:
1955 nodes = pol->v.nodes;
1956 break;
1957
1958 default:
1959 BUG();
1960 return -EFAULT;
1961 }
1962
1963 l = strlen(policy_types[mode]);
1964 if (buffer + maxlen < p + l + 1)
1965 return -ENOSPC;
1966
1967 strcpy(p, policy_types[mode]);
1968 p += l;
1969
David Rientjesf5b087b2008-04-28 02:12:27 -07001970 if (flags) {
1971 int need_bar = 0;
1972
1973 if (buffer + maxlen < p + 2)
1974 return -ENOSPC;
1975 *p++ = '=';
1976
1977 if (flags & MPOL_F_STATIC_NODES)
1978 p += sprintf(p, "%sstatic", need_bar++ ? "|" : "");
David Rientjes4c50bc02008-04-28 02:12:30 -07001979 if (flags & MPOL_F_RELATIVE_NODES)
1980 p += sprintf(p, "%srelative", need_bar++ ? "|" : "");
David Rientjesf5b087b2008-04-28 02:12:27 -07001981 }
1982
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001983 if (!nodes_empty(nodes)) {
1984 if (buffer + maxlen < p + 2)
1985 return -ENOSPC;
1986 *p++ = '=';
1987 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
1988 }
1989 return p - buffer;
1990}
1991
1992struct numa_maps {
1993 unsigned long pages;
1994 unsigned long anon;
Christoph Lameter397874d2006-03-06 15:42:53 -08001995 unsigned long active;
1996 unsigned long writeback;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001997 unsigned long mapcount_max;
Christoph Lameter397874d2006-03-06 15:42:53 -08001998 unsigned long dirty;
1999 unsigned long swapcache;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002000 unsigned long node[MAX_NUMNODES];
2001};
2002
Christoph Lameter397874d2006-03-06 15:42:53 -08002003static void gather_stats(struct page *page, void *private, int pte_dirty)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002004{
2005 struct numa_maps *md = private;
2006 int count = page_mapcount(page);
2007
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002008 md->pages++;
Christoph Lameter397874d2006-03-06 15:42:53 -08002009 if (pte_dirty || PageDirty(page))
2010 md->dirty++;
2011
2012 if (PageSwapCache(page))
2013 md->swapcache++;
2014
2015 if (PageActive(page))
2016 md->active++;
2017
2018 if (PageWriteback(page))
2019 md->writeback++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002020
2021 if (PageAnon(page))
2022 md->anon++;
2023
Christoph Lameter397874d2006-03-06 15:42:53 -08002024 if (count > md->mapcount_max)
2025 md->mapcount_max = count;
2026
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002027 md->node[page_to_nid(page)]++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002028}
2029
Andrew Morton7f709ed2006-03-07 21:55:22 -08002030#ifdef CONFIG_HUGETLB_PAGE
Christoph Lameter397874d2006-03-06 15:42:53 -08002031static void check_huge_range(struct vm_area_struct *vma,
2032 unsigned long start, unsigned long end,
2033 struct numa_maps *md)
2034{
2035 unsigned long addr;
2036 struct page *page;
2037
2038 for (addr = start; addr < end; addr += HPAGE_SIZE) {
2039 pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
2040 pte_t pte;
2041
2042 if (!ptep)
2043 continue;
2044
2045 pte = *ptep;
2046 if (pte_none(pte))
2047 continue;
2048
2049 page = pte_page(pte);
2050 if (!page)
2051 continue;
2052
2053 gather_stats(page, md, pte_dirty(*ptep));
2054 }
2055}
Andrew Morton7f709ed2006-03-07 21:55:22 -08002056#else
2057static inline void check_huge_range(struct vm_area_struct *vma,
2058 unsigned long start, unsigned long end,
2059 struct numa_maps *md)
2060{
2061}
2062#endif
Christoph Lameter397874d2006-03-06 15:42:53 -08002063
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002064int show_numa_map(struct seq_file *m, void *v)
2065{
Eric W. Biederman99f89552006-06-26 00:25:55 -07002066 struct proc_maps_private *priv = m->private;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002067 struct vm_area_struct *vma = v;
2068 struct numa_maps *md;
Christoph Lameter397874d2006-03-06 15:42:53 -08002069 struct file *file = vma->vm_file;
2070 struct mm_struct *mm = vma->vm_mm;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002071 struct mempolicy *pol;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002072 int n;
2073 char buffer[50];
2074
Christoph Lameter397874d2006-03-06 15:42:53 -08002075 if (!mm)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002076 return 0;
2077
2078 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
2079 if (!md)
2080 return 0;
2081
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002082 pol = get_vma_policy(priv->task, vma, vma->vm_start);
2083 mpol_to_str(buffer, sizeof(buffer), pol);
2084 /*
2085 * unref shared or other task's mempolicy
2086 */
2087 if (pol != &default_policy && pol != current->mempolicy)
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07002088 __mpol_put(pol);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002089
Christoph Lameter397874d2006-03-06 15:42:53 -08002090 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002091
Christoph Lameter397874d2006-03-06 15:42:53 -08002092 if (file) {
2093 seq_printf(m, " file=");
Jan Blunckc32c2f62008-02-14 19:38:43 -08002094 seq_path(m, &file->f_path, "\n\t= ");
Christoph Lameter397874d2006-03-06 15:42:53 -08002095 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
2096 seq_printf(m, " heap");
2097 } else if (vma->vm_start <= mm->start_stack &&
2098 vma->vm_end >= mm->start_stack) {
2099 seq_printf(m, " stack");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002100 }
Christoph Lameter397874d2006-03-06 15:42:53 -08002101
2102 if (is_vm_hugetlb_page(vma)) {
2103 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
2104 seq_printf(m, " huge");
2105 } else {
2106 check_pgd_range(vma, vma->vm_start, vma->vm_end,
Christoph Lameter56bbd652007-10-16 01:25:35 -07002107 &node_states[N_HIGH_MEMORY], MPOL_MF_STATS, md);
Christoph Lameter397874d2006-03-06 15:42:53 -08002108 }
2109
2110 if (!md->pages)
2111 goto out;
2112
2113 if (md->anon)
2114 seq_printf(m," anon=%lu",md->anon);
2115
2116 if (md->dirty)
2117 seq_printf(m," dirty=%lu",md->dirty);
2118
2119 if (md->pages != md->anon && md->pages != md->dirty)
2120 seq_printf(m, " mapped=%lu", md->pages);
2121
2122 if (md->mapcount_max > 1)
2123 seq_printf(m, " mapmax=%lu", md->mapcount_max);
2124
2125 if (md->swapcache)
2126 seq_printf(m," swapcache=%lu", md->swapcache);
2127
2128 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
2129 seq_printf(m," active=%lu", md->active);
2130
2131 if (md->writeback)
2132 seq_printf(m," writeback=%lu", md->writeback);
2133
Christoph Lameter56bbd652007-10-16 01:25:35 -07002134 for_each_node_state(n, N_HIGH_MEMORY)
Christoph Lameter397874d2006-03-06 15:42:53 -08002135 if (md->node[n])
2136 seq_printf(m, " N%d=%lu", n, md->node[n]);
2137out:
2138 seq_putc(m, '\n');
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002139 kfree(md);
2140
2141 if (m->count < m->size)
Eric W. Biederman99f89552006-06-26 00:25:55 -07002142 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002143 return 0;
2144}