blob: da9463946556311e9e0945583fd76cdd432298e5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
66 could replace all the switch()es with a mempolicy_ops structure.
67*/
68
69#include <linux/mempolicy.h>
70#include <linux/mm.h>
71#include <linux/highmem.h>
72#include <linux/hugetlb.h>
73#include <linux/kernel.h>
74#include <linux/sched.h>
75#include <linux/mm.h>
76#include <linux/nodemask.h>
77#include <linux/cpuset.h>
78#include <linux/gfp.h>
79#include <linux/slab.h>
80#include <linux/string.h>
81#include <linux/module.h>
82#include <linux/interrupt.h>
83#include <linux/init.h>
84#include <linux/compat.h>
85#include <linux/mempolicy.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080086#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080087#include <linux/seq_file.h>
88#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080089#include <linux/migrate.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070090#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070091#include <linux/security.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080092
Linus Torvalds1da177e2005-04-16 15:20:36 -070093#include <asm/tlbflush.h>
94#include <asm/uaccess.h>
95
Christoph Lameter38e35862006-01-08 01:01:01 -080096/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080097#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -080098#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080099#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800100
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800101static struct kmem_cache *policy_cache;
102static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
104#define PDprintk(fmt...)
105
106/* Highest zone. An specific allocation for a zone below that is not
107 policied. */
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700108enum zone_type policy_zone = ZONE_DMA;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
Andi Kleend42c6992005-07-06 19:56:03 +0200110struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 .refcnt = ATOMIC_INIT(1), /* never free it */
112 .policy = MPOL_DEFAULT,
113};
114
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115/* Do sanity checking on a policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700116static int mpol_check_policy(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700118 int empty = nodes_empty(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
120 switch (mode) {
121 case MPOL_DEFAULT:
122 if (!empty)
123 return -EINVAL;
124 break;
125 case MPOL_BIND:
126 case MPOL_INTERLEAVE:
127 /* Preferred will only use the first bit, but allow
128 more for now. */
129 if (empty)
130 return -EINVAL;
131 break;
132 }
Andi Kleendfcd3c02005-10-29 18:15:48 -0700133 return nodes_subset(*nodes, node_online_map) ? 0 : -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134}
Andi Kleendd942ae2006-02-17 01:39:16 +0100135
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136/* Generate a custom zonelist for the BIND policy. */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700137static struct zonelist *bind_zonelist(nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138{
139 struct zonelist *zl;
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700140 int num, max, nd;
141 enum zone_type k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
Andi Kleendfcd3c02005-10-29 18:15:48 -0700143 max = 1 + MAX_NR_ZONES * nodes_weight(*nodes);
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800144 max++; /* space for zlcache_ptr (see mmzone.h) */
Andi Kleendd942ae2006-02-17 01:39:16 +0100145 zl = kmalloc(sizeof(struct zone *) * max, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 if (!zl)
147 return NULL;
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800148 zl->zlcache_ptr = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 num = 0;
Andi Kleendd942ae2006-02-17 01:39:16 +0100150 /* First put in the highest zones from all nodes, then all the next
151 lower zones etc. Avoid empty zones because the memory allocator
152 doesn't like them. If you implement node hot removal you
153 have to fix that. */
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700154 k = policy_zone;
155 while (1) {
Andi Kleendd942ae2006-02-17 01:39:16 +0100156 for_each_node_mask(nd, *nodes) {
157 struct zone *z = &NODE_DATA(nd)->node_zones[k];
158 if (z->present_pages > 0)
159 zl->zones[num++] = z;
160 }
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700161 if (k == 0)
162 break;
163 k--;
Andi Kleendd942ae2006-02-17 01:39:16 +0100164 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 zl->zones[num] = NULL;
166 return zl;
167}
168
169/* Create a new policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700170static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171{
172 struct mempolicy *policy;
173
Andi Kleendfcd3c02005-10-29 18:15:48 -0700174 PDprintk("setting mode %d nodes[0] %lx\n", mode, nodes_addr(*nodes)[0]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 if (mode == MPOL_DEFAULT)
176 return NULL;
177 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
178 if (!policy)
179 return ERR_PTR(-ENOMEM);
180 atomic_set(&policy->refcnt, 1);
181 switch (mode) {
182 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700183 policy->v.nodes = *nodes;
Andi Kleen8f493d72006-01-03 00:07:28 +0100184 if (nodes_weight(*nodes) == 0) {
185 kmem_cache_free(policy_cache, policy);
186 return ERR_PTR(-EINVAL);
187 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 break;
189 case MPOL_PREFERRED:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700190 policy->v.preferred_node = first_node(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 if (policy->v.preferred_node >= MAX_NUMNODES)
192 policy->v.preferred_node = -1;
193 break;
194 case MPOL_BIND:
195 policy->v.zonelist = bind_zonelist(nodes);
196 if (policy->v.zonelist == NULL) {
197 kmem_cache_free(policy_cache, policy);
198 return ERR_PTR(-ENOMEM);
199 }
200 break;
201 }
202 policy->policy = mode;
Paul Jackson74cb2152006-01-08 01:01:56 -0800203 policy->cpuset_mems_allowed = cpuset_mems_allowed(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 return policy;
205}
206
Christoph Lameter397874d2006-03-06 15:42:53 -0800207static void gather_stats(struct page *, void *, int pte_dirty);
Christoph Lameterfc301282006-01-18 17:42:29 -0800208static void migrate_page_add(struct page *page, struct list_head *pagelist,
209 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800210
Christoph Lameter38e35862006-01-08 01:01:01 -0800211/* Scan through pages checking if pages follow certain conditions. */
Nick Pigginb5810032005-10-29 18:16:12 -0700212static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800213 unsigned long addr, unsigned long end,
214 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800215 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216{
Hugh Dickins91612e02005-06-21 17:15:07 -0700217 pte_t *orig_pte;
218 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700219 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700220
Hugh Dickins705e87c2005-10-29 18:16:27 -0700221 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700222 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800223 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800224 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700225
226 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800228 page = vm_normal_page(vma, addr, *pte);
229 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800231 /*
232 * The check for PageReserved here is important to avoid
233 * handling zero pages and other pages that may have been
234 * marked special by the system.
235 *
236 * If the PageReserved would not be checked here then f.e.
237 * the location of the zero page could have an influence
238 * on MPOL_MF_STRICT, zero pages would be counted for
239 * the per node stats, and there would be useless attempts
240 * to put zero pages on the migration list.
241 */
Christoph Lameterf4598c82006-01-12 01:05:20 -0800242 if (PageReserved(page))
243 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800244 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800245 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
246 continue;
247
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800248 if (flags & MPOL_MF_STATS)
Christoph Lameter397874d2006-03-06 15:42:53 -0800249 gather_stats(page, private, pte_dirty(*pte));
Nick Piggin053837f2006-01-18 17:42:27 -0800250 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800251 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800252 else
253 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700254 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700255 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700256 return addr != end;
257}
258
Nick Pigginb5810032005-10-29 18:16:12 -0700259static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800260 unsigned long addr, unsigned long end,
261 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800262 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700263{
264 pmd_t *pmd;
265 unsigned long next;
266
267 pmd = pmd_offset(pud, addr);
268 do {
269 next = pmd_addr_end(addr, end);
270 if (pmd_none_or_clear_bad(pmd))
271 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800272 if (check_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800273 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700274 return -EIO;
275 } while (pmd++, addr = next, addr != end);
276 return 0;
277}
278
Nick Pigginb5810032005-10-29 18:16:12 -0700279static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800280 unsigned long addr, unsigned long end,
281 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800282 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700283{
284 pud_t *pud;
285 unsigned long next;
286
287 pud = pud_offset(pgd, addr);
288 do {
289 next = pud_addr_end(addr, end);
290 if (pud_none_or_clear_bad(pud))
291 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800292 if (check_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800293 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700294 return -EIO;
295 } while (pud++, addr = next, addr != end);
296 return 0;
297}
298
Nick Pigginb5810032005-10-29 18:16:12 -0700299static inline int check_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800300 unsigned long addr, unsigned long end,
301 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800302 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700303{
304 pgd_t *pgd;
305 unsigned long next;
306
Nick Pigginb5810032005-10-29 18:16:12 -0700307 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700308 do {
309 next = pgd_addr_end(addr, end);
310 if (pgd_none_or_clear_bad(pgd))
311 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800312 if (check_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800313 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700314 return -EIO;
315 } while (pgd++, addr = next, addr != end);
316 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317}
318
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800319/* Check if a vma is migratable */
320static inline int vma_migratable(struct vm_area_struct *vma)
321{
322 if (vma->vm_flags & (
Christoph Lameterf4598c82006-01-12 01:05:20 -0800323 VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP|VM_RESERVED))
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800324 return 0;
325 return 1;
326}
327
328/*
329 * Check if all pages in a range are on a set of nodes.
330 * If pagelist != NULL then isolate pages from the LRU and
331 * put them on the pagelist.
332 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333static struct vm_area_struct *
334check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800335 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336{
337 int err;
338 struct vm_area_struct *first, *vma, *prev;
339
Christoph Lameter90036ee2006-03-16 23:03:59 -0800340 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
Christoph Lameter90036ee2006-03-16 23:03:59 -0800341
Christoph Lameterb20a3502006-03-22 00:09:12 -0800342 err = migrate_prep();
343 if (err)
344 return ERR_PTR(err);
Christoph Lameter90036ee2006-03-16 23:03:59 -0800345 }
Nick Piggin053837f2006-01-18 17:42:27 -0800346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 first = find_vma(mm, start);
348 if (!first)
349 return ERR_PTR(-EFAULT);
350 prev = NULL;
351 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800352 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
353 if (!vma->vm_next && vma->vm_end < end)
354 return ERR_PTR(-EFAULT);
355 if (prev && prev->vm_end < vma->vm_start)
356 return ERR_PTR(-EFAULT);
357 }
358 if (!is_vm_hugetlb_page(vma) &&
359 ((flags & MPOL_MF_STRICT) ||
360 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
361 vma_migratable(vma)))) {
Andi Kleen5b952b32005-09-13 01:25:08 -0700362 unsigned long endvma = vma->vm_end;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800363
Andi Kleen5b952b32005-09-13 01:25:08 -0700364 if (endvma > end)
365 endvma = end;
366 if (vma->vm_start > start)
367 start = vma->vm_start;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800368 err = check_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800369 flags, private);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 if (err) {
371 first = ERR_PTR(err);
372 break;
373 }
374 }
375 prev = vma;
376 }
377 return first;
378}
379
380/* Apply policy to a single VMA */
381static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
382{
383 int err = 0;
384 struct mempolicy *old = vma->vm_policy;
385
386 PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
387 vma->vm_start, vma->vm_end, vma->vm_pgoff,
388 vma->vm_ops, vma->vm_file,
389 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
390
391 if (vma->vm_ops && vma->vm_ops->set_policy)
392 err = vma->vm_ops->set_policy(vma, new);
393 if (!err) {
394 mpol_get(new);
395 vma->vm_policy = new;
396 mpol_free(old);
397 }
398 return err;
399}
400
401/* Step 2: apply policy to a range and do splits. */
402static int mbind_range(struct vm_area_struct *vma, unsigned long start,
403 unsigned long end, struct mempolicy *new)
404{
405 struct vm_area_struct *next;
406 int err;
407
408 err = 0;
409 for (; vma && vma->vm_start < end; vma = next) {
410 next = vma->vm_next;
411 if (vma->vm_start < start)
412 err = split_vma(vma->vm_mm, vma, start, 1);
413 if (!err && vma->vm_end > end)
414 err = split_vma(vma->vm_mm, vma, end, 0);
415 if (!err)
416 err = policy_vma(vma, new);
417 if (err)
418 break;
419 }
420 return err;
421}
422
Christoph Lameter8bccd852005-10-29 18:16:59 -0700423static int contextualize_policy(int mode, nodemask_t *nodes)
424{
425 if (!nodes)
426 return 0;
427
Paul Jacksoncf2a473c2006-01-08 01:01:54 -0800428 cpuset_update_task_memory_state();
Paul Jackson59665142006-01-08 01:01:47 -0800429 if (!cpuset_nodes_subset_current_mems_allowed(*nodes))
430 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700431 return mpol_check_policy(mode, nodes);
432}
433
Paul Jacksonc61afb12006-03-24 03:16:08 -0800434
435/*
436 * Update task->flags PF_MEMPOLICY bit: set iff non-default
437 * mempolicy. Allows more rapid checking of this (combined perhaps
438 * with other PF_* flag bits) on memory allocation hot code paths.
439 *
440 * If called from outside this file, the task 'p' should -only- be
441 * a newly forked child not yet visible on the task list, because
442 * manipulating the task flags of a visible task is not safe.
443 *
444 * The above limitation is why this routine has the funny name
445 * mpol_fix_fork_child_flag().
446 *
447 * It is also safe to call this with a task pointer of current,
448 * which the static wrapper mpol_set_task_struct_flag() does,
449 * for use within this file.
450 */
451
452void mpol_fix_fork_child_flag(struct task_struct *p)
453{
454 if (p->mempolicy)
455 p->flags |= PF_MEMPOLICY;
456 else
457 p->flags &= ~PF_MEMPOLICY;
458}
459
460static void mpol_set_task_struct_flag(void)
461{
462 mpol_fix_fork_child_flag(current);
463}
464
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465/* Set the process memory policy */
Christoph Lameter8bccd852005-10-29 18:16:59 -0700466long do_set_mempolicy(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 struct mempolicy *new;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
Christoph Lameter8bccd852005-10-29 18:16:59 -0700470 if (contextualize_policy(mode, nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700472 new = mpol_new(mode, nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 if (IS_ERR(new))
474 return PTR_ERR(new);
475 mpol_free(current->mempolicy);
476 current->mempolicy = new;
Paul Jacksonc61afb12006-03-24 03:16:08 -0800477 mpol_set_task_struct_flag();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 if (new && new->policy == MPOL_INTERLEAVE)
Andi Kleendfcd3c02005-10-29 18:15:48 -0700479 current->il_next = first_node(new->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 return 0;
481}
482
483/* Fill a zone bitmap for a policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700484static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485{
486 int i;
487
Andi Kleendfcd3c02005-10-29 18:15:48 -0700488 nodes_clear(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 switch (p->policy) {
490 case MPOL_BIND:
491 for (i = 0; p->v.zonelist->zones[i]; i++)
Christoph Lameter89fa3022006-09-25 23:31:55 -0700492 node_set(zone_to_nid(p->v.zonelist->zones[i]),
Christoph Lameter8bccd852005-10-29 18:16:59 -0700493 *nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 break;
495 case MPOL_DEFAULT:
496 break;
497 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700498 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 break;
500 case MPOL_PREFERRED:
501 /* or use current node instead of online map? */
502 if (p->v.preferred_node < 0)
Andi Kleendfcd3c02005-10-29 18:15:48 -0700503 *nodes = node_online_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 else
Andi Kleendfcd3c02005-10-29 18:15:48 -0700505 node_set(p->v.preferred_node, *nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 break;
507 default:
508 BUG();
509 }
510}
511
512static int lookup_node(struct mm_struct *mm, unsigned long addr)
513{
514 struct page *p;
515 int err;
516
517 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
518 if (err >= 0) {
519 err = page_to_nid(p);
520 put_page(p);
521 }
522 return err;
523}
524
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525/* Retrieve NUMA policy */
Christoph Lameter8bccd852005-10-29 18:16:59 -0700526long do_get_mempolicy(int *policy, nodemask_t *nmask,
527 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700529 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 struct mm_struct *mm = current->mm;
531 struct vm_area_struct *vma = NULL;
532 struct mempolicy *pol = current->mempolicy;
533
Paul Jacksoncf2a473c2006-01-08 01:01:54 -0800534 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR))
536 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 if (flags & MPOL_F_ADDR) {
538 down_read(&mm->mmap_sem);
539 vma = find_vma_intersection(mm, addr, addr+1);
540 if (!vma) {
541 up_read(&mm->mmap_sem);
542 return -EFAULT;
543 }
544 if (vma->vm_ops && vma->vm_ops->get_policy)
545 pol = vma->vm_ops->get_policy(vma, addr);
546 else
547 pol = vma->vm_policy;
548 } else if (addr)
549 return -EINVAL;
550
551 if (!pol)
552 pol = &default_policy;
553
554 if (flags & MPOL_F_NODE) {
555 if (flags & MPOL_F_ADDR) {
556 err = lookup_node(mm, addr);
557 if (err < 0)
558 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700559 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 } else if (pol == current->mempolicy &&
561 pol->policy == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700562 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 } else {
564 err = -EINVAL;
565 goto out;
566 }
567 } else
Christoph Lameter8bccd852005-10-29 18:16:59 -0700568 *policy = pol->policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569
570 if (vma) {
571 up_read(&current->mm->mmap_sem);
572 vma = NULL;
573 }
574
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 err = 0;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700576 if (nmask)
577 get_zonemask(pol, nmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578
579 out:
580 if (vma)
581 up_read(&current->mm->mmap_sem);
582 return err;
583}
584
Christoph Lameterb20a3502006-03-22 00:09:12 -0800585#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700586/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800587 * page migration
588 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800589static void migrate_page_add(struct page *page, struct list_head *pagelist,
590 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800591{
592 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800593 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800594 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800595 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
596 isolate_lru_page(page, pagelist);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800597}
598
Christoph Lameter742755a2006-06-23 02:03:55 -0700599static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700600{
601 return alloc_pages_node(node, GFP_HIGHUSER, 0);
602}
603
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800604/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800605 * Migrate pages from one node to a target node.
606 * Returns error or the number of pages not migrated.
607 */
608int migrate_to_node(struct mm_struct *mm, int source, int dest, int flags)
609{
610 nodemask_t nmask;
611 LIST_HEAD(pagelist);
612 int err = 0;
613
614 nodes_clear(nmask);
615 node_set(source, nmask);
616
617 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
618 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
619
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700620 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700621 err = migrate_pages(&pagelist, new_node_page, dest);
622
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800623 return err;
624}
625
626/*
627 * Move pages between the two nodesets so as to preserve the physical
628 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -0800629 *
630 * Returns the number of page that could not be moved.
631 */
632int do_migrate_pages(struct mm_struct *mm,
633 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
634{
635 LIST_HEAD(pagelist);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800636 int busy = 0;
637 int err = 0;
638 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -0800639
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800640 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -0800641
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700642 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
643 if (err)
644 goto out;
645
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800646/*
647 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
648 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
649 * bit in 'tmp', and return that <source, dest> pair for migration.
650 * The pair of nodemasks 'to' and 'from' define the map.
651 *
652 * If no pair of bits is found that way, fallback to picking some
653 * pair of 'source' and 'dest' bits that are not the same. If the
654 * 'source' and 'dest' bits are the same, this represents a node
655 * that will be migrating to itself, so no pages need move.
656 *
657 * If no bits are left in 'tmp', or if all remaining bits left
658 * in 'tmp' correspond to the same bit in 'to', return false
659 * (nothing left to migrate).
660 *
661 * This lets us pick a pair of nodes to migrate between, such that
662 * if possible the dest node is not already occupied by some other
663 * source node, minimizing the risk of overloading the memory on a
664 * node that would happen if we migrated incoming memory to a node
665 * before migrating outgoing memory source that same node.
666 *
667 * A single scan of tmp is sufficient. As we go, we remember the
668 * most recent <s, d> pair that moved (s != d). If we find a pair
669 * that not only moved, but what's better, moved to an empty slot
670 * (d is not set in tmp), then we break out then, with that pair.
671 * Otherwise when we finish scannng from_tmp, we at least have the
672 * most recent <s, d> pair that moved. If we get all the way through
673 * the scan of tmp without finding any node that moved, much less
674 * moved to an empty node, then there is nothing left worth migrating.
675 */
Christoph Lameterd4984712006-01-08 01:00:55 -0800676
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800677 tmp = *from_nodes;
678 while (!nodes_empty(tmp)) {
679 int s,d;
680 int source = -1;
681 int dest = 0;
682
683 for_each_node_mask(s, tmp) {
684 d = node_remap(s, *from_nodes, *to_nodes);
685 if (s == d)
686 continue;
687
688 source = s; /* Node moved. Memorize */
689 dest = d;
690
691 /* dest not in remaining from nodes? */
692 if (!node_isset(dest, tmp))
693 break;
694 }
695 if (source == -1)
696 break;
697
698 node_clear(source, tmp);
699 err = migrate_to_node(mm, source, dest, flags);
700 if (err > 0)
701 busy += err;
702 if (err < 0)
703 break;
Christoph Lameter39743882006-01-08 01:00:51 -0800704 }
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700705out:
Christoph Lameter39743882006-01-08 01:00:51 -0800706 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800707 if (err < 0)
708 return err;
709 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800710
Christoph Lameter39743882006-01-08 01:00:51 -0800711}
712
Christoph Lameter742755a2006-06-23 02:03:55 -0700713static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700714{
715 struct vm_area_struct *vma = (struct vm_area_struct *)private;
716
717 return alloc_page_vma(GFP_HIGHUSER, vma, page_address_in_vma(page, vma));
718}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800719#else
720
721static void migrate_page_add(struct page *page, struct list_head *pagelist,
722 unsigned long flags)
723{
724}
725
726int do_migrate_pages(struct mm_struct *mm,
727 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
728{
729 return -ENOSYS;
730}
Christoph Lameter95a402c2006-06-23 02:03:53 -0700731
Keith Owens69939742006-10-11 01:21:28 -0700732static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700733{
734 return NULL;
735}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800736#endif
737
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800738long do_mbind(unsigned long start, unsigned long len,
739 unsigned long mode, nodemask_t *nmask, unsigned long flags)
740{
741 struct vm_area_struct *vma;
742 struct mm_struct *mm = current->mm;
743 struct mempolicy *new;
744 unsigned long end;
745 int err;
746 LIST_HEAD(pagelist);
747
748 if ((flags & ~(unsigned long)(MPOL_MF_STRICT |
749 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
750 || mode > MPOL_MAX)
751 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -0800752 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800753 return -EPERM;
754
755 if (start & ~PAGE_MASK)
756 return -EINVAL;
757
758 if (mode == MPOL_DEFAULT)
759 flags &= ~MPOL_MF_STRICT;
760
761 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
762 end = start + len;
763
764 if (end < start)
765 return -EINVAL;
766 if (end == start)
767 return 0;
768
769 if (mpol_check_policy(mode, nmask))
770 return -EINVAL;
771
772 new = mpol_new(mode, nmask);
773 if (IS_ERR(new))
774 return PTR_ERR(new);
775
776 /*
777 * If we are using the default policy then operation
778 * on discontinuous address spaces is okay after all
779 */
780 if (!new)
781 flags |= MPOL_MF_DISCONTIG_OK;
782
783 PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len,
784 mode,nodes_addr(nodes)[0]);
785
786 down_write(&mm->mmap_sem);
787 vma = check_range(mm, start, end, nmask,
788 flags | MPOL_MF_INVERT, &pagelist);
789
790 err = PTR_ERR(vma);
791 if (!IS_ERR(vma)) {
792 int nr_failed = 0;
793
794 err = mbind_range(vma, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800795
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800796 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700797 nr_failed = migrate_pages(&pagelist, new_vma_page,
798 (unsigned long)vma);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800799
800 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
801 err = -EIO;
802 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800803
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800804 up_write(&mm->mmap_sem);
805 mpol_free(new);
806 return err;
807}
808
Christoph Lameter39743882006-01-08 01:00:51 -0800809/*
Christoph Lameter8bccd852005-10-29 18:16:59 -0700810 * User space interface with variable sized bitmaps for nodelists.
811 */
812
813/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -0800814static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -0700815 unsigned long maxnode)
816{
817 unsigned long k;
818 unsigned long nlongs;
819 unsigned long endmask;
820
821 --maxnode;
822 nodes_clear(*nodes);
823 if (maxnode == 0 || !nmask)
824 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -0800825 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -0800826 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700827
828 nlongs = BITS_TO_LONGS(maxnode);
829 if ((maxnode % BITS_PER_LONG) == 0)
830 endmask = ~0UL;
831 else
832 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
833
834 /* When the user specified more nodes than supported just check
835 if the non supported part is all zero. */
836 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
837 if (nlongs > PAGE_SIZE/sizeof(long))
838 return -EINVAL;
839 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
840 unsigned long t;
841 if (get_user(t, nmask + k))
842 return -EFAULT;
843 if (k == nlongs - 1) {
844 if (t & endmask)
845 return -EINVAL;
846 } else if (t)
847 return -EINVAL;
848 }
849 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
850 endmask = ~0UL;
851 }
852
853 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
854 return -EFAULT;
855 nodes_addr(*nodes)[nlongs-1] &= endmask;
856 return 0;
857}
858
859/* Copy a kernel node mask to user space */
860static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
861 nodemask_t *nodes)
862{
863 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
864 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
865
866 if (copy > nbytes) {
867 if (copy > PAGE_SIZE)
868 return -EINVAL;
869 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
870 return -EFAULT;
871 copy = nbytes;
872 }
873 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
874}
875
876asmlinkage long sys_mbind(unsigned long start, unsigned long len,
877 unsigned long mode,
878 unsigned long __user *nmask, unsigned long maxnode,
879 unsigned flags)
880{
881 nodemask_t nodes;
882 int err;
883
884 err = get_nodes(&nodes, nmask, maxnode);
885 if (err)
886 return err;
887 return do_mbind(start, len, mode, &nodes, flags);
888}
889
890/* Set the process memory policy */
891asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
892 unsigned long maxnode)
893{
894 int err;
895 nodemask_t nodes;
896
897 if (mode < 0 || mode > MPOL_MAX)
898 return -EINVAL;
899 err = get_nodes(&nodes, nmask, maxnode);
900 if (err)
901 return err;
902 return do_set_mempolicy(mode, &nodes);
903}
904
Christoph Lameter39743882006-01-08 01:00:51 -0800905asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
906 const unsigned long __user *old_nodes,
907 const unsigned long __user *new_nodes)
908{
909 struct mm_struct *mm;
910 struct task_struct *task;
911 nodemask_t old;
912 nodemask_t new;
913 nodemask_t task_nodes;
914 int err;
915
916 err = get_nodes(&old, old_nodes, maxnode);
917 if (err)
918 return err;
919
920 err = get_nodes(&new, new_nodes, maxnode);
921 if (err)
922 return err;
923
924 /* Find the mm_struct */
925 read_lock(&tasklist_lock);
926 task = pid ? find_task_by_pid(pid) : current;
927 if (!task) {
928 read_unlock(&tasklist_lock);
929 return -ESRCH;
930 }
931 mm = get_task_mm(task);
932 read_unlock(&tasklist_lock);
933
934 if (!mm)
935 return -EINVAL;
936
937 /*
938 * Check if this process has the right to modify the specified
939 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -0800940 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -0800941 * userid as the target process.
942 */
943 if ((current->euid != task->suid) && (current->euid != task->uid) &&
944 (current->uid != task->suid) && (current->uid != task->uid) &&
Christoph Lameter74c00242006-03-14 19:50:21 -0800945 !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -0800946 err = -EPERM;
947 goto out;
948 }
949
950 task_nodes = cpuset_mems_allowed(task);
951 /* Is the user allowed to access the target nodes? */
Christoph Lameter74c00242006-03-14 19:50:21 -0800952 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -0800953 err = -EPERM;
954 goto out;
955 }
956
David Quigley86c3a762006-06-23 02:04:02 -0700957 err = security_task_movememory(task);
958 if (err)
959 goto out;
960
Christoph Lameter511030b2006-02-28 16:58:57 -0800961 err = do_migrate_pages(mm, &old, &new,
Christoph Lameter74c00242006-03-14 19:50:21 -0800962 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter39743882006-01-08 01:00:51 -0800963out:
964 mmput(mm);
965 return err;
966}
967
968
Christoph Lameter8bccd852005-10-29 18:16:59 -0700969/* Retrieve NUMA policy */
970asmlinkage long sys_get_mempolicy(int __user *policy,
971 unsigned long __user *nmask,
972 unsigned long maxnode,
973 unsigned long addr, unsigned long flags)
974{
975 int err, pval;
976 nodemask_t nodes;
977
978 if (nmask != NULL && maxnode < MAX_NUMNODES)
979 return -EINVAL;
980
981 err = do_get_mempolicy(&pval, &nodes, addr, flags);
982
983 if (err)
984 return err;
985
986 if (policy && put_user(pval, policy))
987 return -EFAULT;
988
989 if (nmask)
990 err = copy_nodes_to_user(nmask, maxnode, &nodes);
991
992 return err;
993}
994
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995#ifdef CONFIG_COMPAT
996
997asmlinkage long compat_sys_get_mempolicy(int __user *policy,
998 compat_ulong_t __user *nmask,
999 compat_ulong_t maxnode,
1000 compat_ulong_t addr, compat_ulong_t flags)
1001{
1002 long err;
1003 unsigned long __user *nm = NULL;
1004 unsigned long nr_bits, alloc_size;
1005 DECLARE_BITMAP(bm, MAX_NUMNODES);
1006
1007 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1008 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1009
1010 if (nmask)
1011 nm = compat_alloc_user_space(alloc_size);
1012
1013 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1014
1015 if (!err && nmask) {
1016 err = copy_from_user(bm, nm, alloc_size);
1017 /* ensure entire bitmap is zeroed */
1018 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1019 err |= compat_put_bitmap(nmask, bm, nr_bits);
1020 }
1021
1022 return err;
1023}
1024
1025asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1026 compat_ulong_t maxnode)
1027{
1028 long err = 0;
1029 unsigned long __user *nm = NULL;
1030 unsigned long nr_bits, alloc_size;
1031 DECLARE_BITMAP(bm, MAX_NUMNODES);
1032
1033 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1034 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1035
1036 if (nmask) {
1037 err = compat_get_bitmap(bm, nmask, nr_bits);
1038 nm = compat_alloc_user_space(alloc_size);
1039 err |= copy_to_user(nm, bm, alloc_size);
1040 }
1041
1042 if (err)
1043 return -EFAULT;
1044
1045 return sys_set_mempolicy(mode, nm, nr_bits+1);
1046}
1047
1048asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1049 compat_ulong_t mode, compat_ulong_t __user *nmask,
1050 compat_ulong_t maxnode, compat_ulong_t flags)
1051{
1052 long err = 0;
1053 unsigned long __user *nm = NULL;
1054 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001055 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056
1057 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1058 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1059
1060 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001061 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001063 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064 }
1065
1066 if (err)
1067 return -EFAULT;
1068
1069 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1070}
1071
1072#endif
1073
1074/* Return effective policy for a VMA */
Christoph Lameter48fce342006-01-08 01:01:03 -08001075static struct mempolicy * get_vma_policy(struct task_struct *task,
1076 struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001078 struct mempolicy *pol = task->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079
1080 if (vma) {
1081 if (vma->vm_ops && vma->vm_ops->get_policy)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001082 pol = vma->vm_ops->get_policy(vma, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 else if (vma->vm_policy &&
1084 vma->vm_policy->policy != MPOL_DEFAULT)
1085 pol = vma->vm_policy;
1086 }
1087 if (!pol)
1088 pol = &default_policy;
1089 return pol;
1090}
1091
1092/* Return a zonelist representing a mempolicy */
Al Virodd0fc662005-10-07 07:46:04 +01001093static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094{
1095 int nd;
1096
1097 switch (policy->policy) {
1098 case MPOL_PREFERRED:
1099 nd = policy->v.preferred_node;
1100 if (nd < 0)
1101 nd = numa_node_id();
1102 break;
1103 case MPOL_BIND:
1104 /* Lower zones don't get a policy applied */
1105 /* Careful: current->mems_allowed might have moved */
Christoph Lameter19655d32006-09-25 23:31:19 -07001106 if (gfp_zone(gfp) >= policy_zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107 if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist))
1108 return policy->v.zonelist;
1109 /*FALL THROUGH*/
1110 case MPOL_INTERLEAVE: /* should not happen */
1111 case MPOL_DEFAULT:
1112 nd = numa_node_id();
1113 break;
1114 default:
1115 nd = 0;
1116 BUG();
1117 }
Al Viroaf4ca452005-10-21 02:55:38 -04001118 return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119}
1120
1121/* Do dynamic interleaving for a process */
1122static unsigned interleave_nodes(struct mempolicy *policy)
1123{
1124 unsigned nid, next;
1125 struct task_struct *me = current;
1126
1127 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001128 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001130 next = first_node(policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131 me->il_next = next;
1132 return nid;
1133}
1134
Christoph Lameterdc85da12006-01-18 17:42:36 -08001135/*
1136 * Depending on the memory policy provide a node from which to allocate the
1137 * next slab entry.
1138 */
1139unsigned slab_node(struct mempolicy *policy)
1140{
Christoph Lameter765c4502006-09-27 01:50:08 -07001141 int pol = policy ? policy->policy : MPOL_DEFAULT;
1142
1143 switch (pol) {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001144 case MPOL_INTERLEAVE:
1145 return interleave_nodes(policy);
1146
1147 case MPOL_BIND:
1148 /*
1149 * Follow bind policy behavior and start allocation at the
1150 * first node.
1151 */
Christoph Lameter89fa3022006-09-25 23:31:55 -07001152 return zone_to_nid(policy->v.zonelist->zones[0]);
Christoph Lameterdc85da12006-01-18 17:42:36 -08001153
1154 case MPOL_PREFERRED:
1155 if (policy->v.preferred_node >= 0)
1156 return policy->v.preferred_node;
1157 /* Fall through */
1158
1159 default:
1160 return numa_node_id();
1161 }
1162}
1163
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164/* Do static interleaving for a VMA with known offset. */
1165static unsigned offset_il_node(struct mempolicy *pol,
1166 struct vm_area_struct *vma, unsigned long off)
1167{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001168 unsigned nnodes = nodes_weight(pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001169 unsigned target = (unsigned)off % nnodes;
1170 int c;
1171 int nid = -1;
1172
1173 c = 0;
1174 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001175 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176 c++;
1177 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 return nid;
1179}
1180
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001181/* Determine a node number for interleave */
1182static inline unsigned interleave_nid(struct mempolicy *pol,
1183 struct vm_area_struct *vma, unsigned long addr, int shift)
1184{
1185 if (vma) {
1186 unsigned long off;
1187
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001188 /*
1189 * for small pages, there is no difference between
1190 * shift and PAGE_SHIFT, so the bit-shift is safe.
1191 * for huge pages, since vm_pgoff is in units of small
1192 * pages, we need to shift off the always 0 bits to get
1193 * a useful offset.
1194 */
1195 BUG_ON(shift < PAGE_SHIFT);
1196 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001197 off += (addr - vma->vm_start) >> shift;
1198 return offset_il_node(pol, vma, off);
1199 } else
1200 return interleave_nodes(pol);
1201}
1202
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001203#ifdef CONFIG_HUGETLBFS
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001204/* Return a zonelist suitable for a huge page allocation. */
1205struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr)
1206{
1207 struct mempolicy *pol = get_vma_policy(current, vma, addr);
1208
1209 if (pol->policy == MPOL_INTERLEAVE) {
1210 unsigned nid;
1211
1212 nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
1213 return NODE_DATA(nid)->node_zonelists + gfp_zone(GFP_HIGHUSER);
1214 }
1215 return zonelist_policy(GFP_HIGHUSER, pol);
1216}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001217#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001218
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219/* Allocate a page in interleaved policy.
1220 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001221static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1222 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223{
1224 struct zonelist *zl;
1225 struct page *page;
1226
Al Viroaf4ca452005-10-21 02:55:38 -04001227 zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 page = __alloc_pages(gfp, order, zl);
Christoph Lameterca889e62006-06-30 01:55:44 -07001229 if (page && page_zone(page) == zl->zones[0])
1230 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 return page;
1232}
1233
1234/**
1235 * alloc_page_vma - Allocate a page for a VMA.
1236 *
1237 * @gfp:
1238 * %GFP_USER user allocation.
1239 * %GFP_KERNEL kernel allocations,
1240 * %GFP_HIGHMEM highmem/user allocations,
1241 * %GFP_FS allocation should not call back into a file system.
1242 * %GFP_ATOMIC don't sleep.
1243 *
1244 * @vma: Pointer to VMA or NULL if not available.
1245 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1246 *
1247 * This function allocates a page from the kernel page pool and applies
1248 * a NUMA policy associated with the VMA or the current process.
1249 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1250 * mm_struct of the VMA to prevent it from going away. Should be used for
1251 * all allocations for pages that will be mapped into
1252 * user space. Returns NULL when no page can be allocated.
1253 *
1254 * Should be called with the mm_sem of the vma hold.
1255 */
1256struct page *
Al Virodd0fc662005-10-07 07:46:04 +01001257alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001259 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260
Paul Jacksoncf2a473c2006-01-08 01:01:54 -08001261 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262
1263 if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
1264 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001265
1266 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267 return alloc_page_interleave(gfp, 0, nid);
1268 }
1269 return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol));
1270}
1271
1272/**
1273 * alloc_pages_current - Allocate pages.
1274 *
1275 * @gfp:
1276 * %GFP_USER user allocation,
1277 * %GFP_KERNEL kernel allocation,
1278 * %GFP_HIGHMEM highmem allocation,
1279 * %GFP_FS don't call back into a file system.
1280 * %GFP_ATOMIC don't sleep.
1281 * @order: Power of two of allocation size in pages. 0 is a single page.
1282 *
1283 * Allocate a page from the kernel page pool. When not in
1284 * interrupt context and apply the current process NUMA policy.
1285 * Returns NULL when no page can be allocated.
1286 *
Paul Jacksoncf2a473c2006-01-08 01:01:54 -08001287 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 * 1) it's ok to take cpuset_sem (can WAIT), and
1289 * 2) allocating for current task (not interrupt).
1290 */
Al Virodd0fc662005-10-07 07:46:04 +01001291struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292{
1293 struct mempolicy *pol = current->mempolicy;
1294
1295 if ((gfp & __GFP_WAIT) && !in_interrupt())
Paul Jacksoncf2a473c2006-01-08 01:01:54 -08001296 cpuset_update_task_memory_state();
Christoph Lameter9b819d22006-09-25 23:31:40 -07001297 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001298 pol = &default_policy;
1299 if (pol->policy == MPOL_INTERLEAVE)
1300 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
1301 return __alloc_pages(gfp, order, zonelist_policy(gfp, pol));
1302}
1303EXPORT_SYMBOL(alloc_pages_current);
1304
Paul Jackson42253992006-01-08 01:01:59 -08001305/*
1306 * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
1307 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1308 * with the mems_allowed returned by cpuset_mems_allowed(). This
1309 * keeps mempolicies cpuset relative after its cpuset moves. See
1310 * further kernel/cpuset.c update_nodemask().
1311 */
1312void *cpuset_being_rebound;
1313
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314/* Slow path of a mempolicy copy */
1315struct mempolicy *__mpol_copy(struct mempolicy *old)
1316{
1317 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1318
1319 if (!new)
1320 return ERR_PTR(-ENOMEM);
Paul Jackson42253992006-01-08 01:01:59 -08001321 if (current_cpuset_is_being_rebound()) {
1322 nodemask_t mems = cpuset_mems_allowed(current);
1323 mpol_rebind_policy(old, &mems);
1324 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 *new = *old;
1326 atomic_set(&new->refcnt, 1);
1327 if (new->policy == MPOL_BIND) {
1328 int sz = ksize(old->v.zonelist);
Christoph Lametere94b1762006-12-06 20:33:17 -08001329 new->v.zonelist = kmemdup(old->v.zonelist, sz, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330 if (!new->v.zonelist) {
1331 kmem_cache_free(policy_cache, new);
1332 return ERR_PTR(-ENOMEM);
1333 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334 }
1335 return new;
1336}
1337
1338/* Slow path of a mempolicy comparison */
1339int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1340{
1341 if (!a || !b)
1342 return 0;
1343 if (a->policy != b->policy)
1344 return 0;
1345 switch (a->policy) {
1346 case MPOL_DEFAULT:
1347 return 1;
1348 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -07001349 return nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 case MPOL_PREFERRED:
1351 return a->v.preferred_node == b->v.preferred_node;
1352 case MPOL_BIND: {
1353 int i;
1354 for (i = 0; a->v.zonelist->zones[i]; i++)
1355 if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i])
1356 return 0;
1357 return b->v.zonelist->zones[i] == NULL;
1358 }
1359 default:
1360 BUG();
1361 return 0;
1362 }
1363}
1364
1365/* Slow path of a mpol destructor. */
1366void __mpol_free(struct mempolicy *p)
1367{
1368 if (!atomic_dec_and_test(&p->refcnt))
1369 return;
1370 if (p->policy == MPOL_BIND)
1371 kfree(p->v.zonelist);
1372 p->policy = MPOL_DEFAULT;
1373 kmem_cache_free(policy_cache, p);
1374}
1375
1376/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377 * Shared memory backing store policy support.
1378 *
1379 * Remember policies even when nobody has shared memory mapped.
1380 * The policies are kept in Red-Black tree linked from the inode.
1381 * They are protected by the sp->lock spinlock, which should be held
1382 * for any accesses to the tree.
1383 */
1384
1385/* lookup first element intersecting start-end */
1386/* Caller holds sp->lock */
1387static struct sp_node *
1388sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1389{
1390 struct rb_node *n = sp->root.rb_node;
1391
1392 while (n) {
1393 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1394
1395 if (start >= p->end)
1396 n = n->rb_right;
1397 else if (end <= p->start)
1398 n = n->rb_left;
1399 else
1400 break;
1401 }
1402 if (!n)
1403 return NULL;
1404 for (;;) {
1405 struct sp_node *w = NULL;
1406 struct rb_node *prev = rb_prev(n);
1407 if (!prev)
1408 break;
1409 w = rb_entry(prev, struct sp_node, nd);
1410 if (w->end <= start)
1411 break;
1412 n = prev;
1413 }
1414 return rb_entry(n, struct sp_node, nd);
1415}
1416
1417/* Insert a new shared policy into the list. */
1418/* Caller holds sp->lock */
1419static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1420{
1421 struct rb_node **p = &sp->root.rb_node;
1422 struct rb_node *parent = NULL;
1423 struct sp_node *nd;
1424
1425 while (*p) {
1426 parent = *p;
1427 nd = rb_entry(parent, struct sp_node, nd);
1428 if (new->start < nd->start)
1429 p = &(*p)->rb_left;
1430 else if (new->end > nd->end)
1431 p = &(*p)->rb_right;
1432 else
1433 BUG();
1434 }
1435 rb_link_node(&new->nd, parent, p);
1436 rb_insert_color(&new->nd, &sp->root);
1437 PDprintk("inserting %lx-%lx: %d\n", new->start, new->end,
1438 new->policy ? new->policy->policy : 0);
1439}
1440
1441/* Find shared policy intersecting idx */
1442struct mempolicy *
1443mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1444{
1445 struct mempolicy *pol = NULL;
1446 struct sp_node *sn;
1447
1448 if (!sp->root.rb_node)
1449 return NULL;
1450 spin_lock(&sp->lock);
1451 sn = sp_lookup(sp, idx, idx+1);
1452 if (sn) {
1453 mpol_get(sn->policy);
1454 pol = sn->policy;
1455 }
1456 spin_unlock(&sp->lock);
1457 return pol;
1458}
1459
1460static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1461{
1462 PDprintk("deleting %lx-l%x\n", n->start, n->end);
1463 rb_erase(&n->nd, &sp->root);
1464 mpol_free(n->policy);
1465 kmem_cache_free(sn_cache, n);
1466}
1467
1468struct sp_node *
1469sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol)
1470{
1471 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1472
1473 if (!n)
1474 return NULL;
1475 n->start = start;
1476 n->end = end;
1477 mpol_get(pol);
1478 n->policy = pol;
1479 return n;
1480}
1481
1482/* Replace a policy range. */
1483static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1484 unsigned long end, struct sp_node *new)
1485{
1486 struct sp_node *n, *new2 = NULL;
1487
1488restart:
1489 spin_lock(&sp->lock);
1490 n = sp_lookup(sp, start, end);
1491 /* Take care of old policies in the same range. */
1492 while (n && n->start < end) {
1493 struct rb_node *next = rb_next(&n->nd);
1494 if (n->start >= start) {
1495 if (n->end <= end)
1496 sp_delete(sp, n);
1497 else
1498 n->start = end;
1499 } else {
1500 /* Old policy spanning whole new range. */
1501 if (n->end > end) {
1502 if (!new2) {
1503 spin_unlock(&sp->lock);
1504 new2 = sp_alloc(end, n->end, n->policy);
1505 if (!new2)
1506 return -ENOMEM;
1507 goto restart;
1508 }
1509 n->end = start;
1510 sp_insert(sp, new2);
1511 new2 = NULL;
1512 break;
1513 } else
1514 n->end = start;
1515 }
1516 if (!next)
1517 break;
1518 n = rb_entry(next, struct sp_node, nd);
1519 }
1520 if (new)
1521 sp_insert(sp, new);
1522 spin_unlock(&sp->lock);
1523 if (new2) {
1524 mpol_free(new2->policy);
1525 kmem_cache_free(sn_cache, new2);
1526 }
1527 return 0;
1528}
1529
Robin Holt7339ff82006-01-14 13:20:48 -08001530void mpol_shared_policy_init(struct shared_policy *info, int policy,
1531 nodemask_t *policy_nodes)
1532{
1533 info->root = RB_ROOT;
1534 spin_lock_init(&info->lock);
1535
1536 if (policy != MPOL_DEFAULT) {
1537 struct mempolicy *newpol;
1538
1539 /* Falls back to MPOL_DEFAULT on any error */
1540 newpol = mpol_new(policy, policy_nodes);
1541 if (!IS_ERR(newpol)) {
1542 /* Create pseudo-vma that contains just the policy */
1543 struct vm_area_struct pvma;
1544
1545 memset(&pvma, 0, sizeof(struct vm_area_struct));
1546 /* Policy covers entire file */
1547 pvma.vm_end = TASK_SIZE;
1548 mpol_set_shared_policy(info, &pvma, newpol);
1549 mpol_free(newpol);
1550 }
1551 }
1552}
1553
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554int mpol_set_shared_policy(struct shared_policy *info,
1555 struct vm_area_struct *vma, struct mempolicy *npol)
1556{
1557 int err;
1558 struct sp_node *new = NULL;
1559 unsigned long sz = vma_pages(vma);
1560
1561 PDprintk("set_shared_policy %lx sz %lu %d %lx\n",
1562 vma->vm_pgoff,
1563 sz, npol? npol->policy : -1,
Andi Kleendfcd3c02005-10-29 18:15:48 -07001564 npol ? nodes_addr(npol->v.nodes)[0] : -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565
1566 if (npol) {
1567 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1568 if (!new)
1569 return -ENOMEM;
1570 }
1571 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1572 if (err && new)
1573 kmem_cache_free(sn_cache, new);
1574 return err;
1575}
1576
1577/* Free a backing policy store on inode delete. */
1578void mpol_free_shared_policy(struct shared_policy *p)
1579{
1580 struct sp_node *n;
1581 struct rb_node *next;
1582
1583 if (!p->root.rb_node)
1584 return;
1585 spin_lock(&p->lock);
1586 next = rb_first(&p->root);
1587 while (next) {
1588 n = rb_entry(next, struct sp_node, nd);
1589 next = rb_next(&n->nd);
Andi Kleen90c50292005-07-27 11:43:50 -07001590 rb_erase(&n->nd, &p->root);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591 mpol_free(n->policy);
1592 kmem_cache_free(sn_cache, n);
1593 }
1594 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595}
1596
1597/* assumes fs == KERNEL_DS */
1598void __init numa_policy_init(void)
1599{
1600 policy_cache = kmem_cache_create("numa_policy",
1601 sizeof(struct mempolicy),
1602 0, SLAB_PANIC, NULL, NULL);
1603
1604 sn_cache = kmem_cache_create("shared_policy_node",
1605 sizeof(struct sp_node),
1606 0, SLAB_PANIC, NULL, NULL);
1607
1608 /* Set interleaving policy for system init. This way not all
1609 the data structures allocated at system boot end up in node zero. */
1610
Christoph Lameter8bccd852005-10-29 18:16:59 -07001611 if (do_set_mempolicy(MPOL_INTERLEAVE, &node_online_map))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612 printk("numa_policy_init: interleaving failed\n");
1613}
1614
Christoph Lameter8bccd852005-10-29 18:16:59 -07001615/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616void numa_default_policy(void)
1617{
Christoph Lameter8bccd852005-10-29 18:16:59 -07001618 do_set_mempolicy(MPOL_DEFAULT, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619}
Paul Jackson68860ec2005-10-30 15:02:36 -08001620
1621/* Migrate a policy to a different set of nodes */
Paul Jackson74cb2152006-01-08 01:01:56 -08001622void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
Paul Jackson68860ec2005-10-30 15:02:36 -08001623{
Paul Jackson74cb2152006-01-08 01:01:56 -08001624 nodemask_t *mpolmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001625 nodemask_t tmp;
1626
1627 if (!pol)
1628 return;
Paul Jackson74cb2152006-01-08 01:01:56 -08001629 mpolmask = &pol->cpuset_mems_allowed;
1630 if (nodes_equal(*mpolmask, *newmask))
1631 return;
Paul Jackson68860ec2005-10-30 15:02:36 -08001632
1633 switch (pol->policy) {
1634 case MPOL_DEFAULT:
1635 break;
1636 case MPOL_INTERLEAVE:
Paul Jackson74cb2152006-01-08 01:01:56 -08001637 nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001638 pol->v.nodes = tmp;
Paul Jackson74cb2152006-01-08 01:01:56 -08001639 *mpolmask = *newmask;
1640 current->il_next = node_remap(current->il_next,
1641 *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001642 break;
1643 case MPOL_PREFERRED:
1644 pol->v.preferred_node = node_remap(pol->v.preferred_node,
Paul Jackson74cb2152006-01-08 01:01:56 -08001645 *mpolmask, *newmask);
1646 *mpolmask = *newmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001647 break;
1648 case MPOL_BIND: {
1649 nodemask_t nodes;
1650 struct zone **z;
1651 struct zonelist *zonelist;
1652
1653 nodes_clear(nodes);
1654 for (z = pol->v.zonelist->zones; *z; z++)
Christoph Lameter89fa3022006-09-25 23:31:55 -07001655 node_set(zone_to_nid(*z), nodes);
Paul Jackson74cb2152006-01-08 01:01:56 -08001656 nodes_remap(tmp, nodes, *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001657 nodes = tmp;
1658
1659 zonelist = bind_zonelist(&nodes);
1660
1661 /* If no mem, then zonelist is NULL and we keep old zonelist.
1662 * If that old zonelist has no remaining mems_allowed nodes,
1663 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
1664 */
1665
1666 if (zonelist) {
1667 /* Good - got mem - substitute new zonelist */
1668 kfree(pol->v.zonelist);
1669 pol->v.zonelist = zonelist;
1670 }
Paul Jackson74cb2152006-01-08 01:01:56 -08001671 *mpolmask = *newmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001672 break;
1673 }
1674 default:
1675 BUG();
1676 break;
1677 }
1678}
1679
1680/*
Paul Jackson74cb2152006-01-08 01:01:56 -08001681 * Wrapper for mpol_rebind_policy() that just requires task
1682 * pointer, and updates task mempolicy.
Paul Jackson68860ec2005-10-30 15:02:36 -08001683 */
Paul Jackson74cb2152006-01-08 01:01:56 -08001684
1685void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
Paul Jackson68860ec2005-10-30 15:02:36 -08001686{
Paul Jackson74cb2152006-01-08 01:01:56 -08001687 mpol_rebind_policy(tsk->mempolicy, new);
Paul Jackson68860ec2005-10-30 15:02:36 -08001688}
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001689
1690/*
Paul Jackson42253992006-01-08 01:01:59 -08001691 * Rebind each vma in mm to new nodemask.
1692 *
1693 * Call holding a reference to mm. Takes mm->mmap_sem during call.
1694 */
1695
1696void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
1697{
1698 struct vm_area_struct *vma;
1699
1700 down_write(&mm->mmap_sem);
1701 for (vma = mm->mmap; vma; vma = vma->vm_next)
1702 mpol_rebind_policy(vma->vm_policy, new);
1703 up_write(&mm->mmap_sem);
1704}
1705
1706/*
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001707 * Display pages allocated per node and memory policy via /proc.
1708 */
1709
Helge Deller15ad7cd2006-12-06 20:40:36 -08001710static const char * const policy_types[] =
1711 { "default", "prefer", "bind", "interleave" };
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001712
1713/*
1714 * Convert a mempolicy into a string.
1715 * Returns the number of characters in buffer (if positive)
1716 * or an error (negative)
1717 */
1718static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
1719{
1720 char *p = buffer;
1721 int l;
1722 nodemask_t nodes;
1723 int mode = pol ? pol->policy : MPOL_DEFAULT;
1724
1725 switch (mode) {
1726 case MPOL_DEFAULT:
1727 nodes_clear(nodes);
1728 break;
1729
1730 case MPOL_PREFERRED:
1731 nodes_clear(nodes);
1732 node_set(pol->v.preferred_node, nodes);
1733 break;
1734
1735 case MPOL_BIND:
1736 get_zonemask(pol, &nodes);
1737 break;
1738
1739 case MPOL_INTERLEAVE:
1740 nodes = pol->v.nodes;
1741 break;
1742
1743 default:
1744 BUG();
1745 return -EFAULT;
1746 }
1747
1748 l = strlen(policy_types[mode]);
1749 if (buffer + maxlen < p + l + 1)
1750 return -ENOSPC;
1751
1752 strcpy(p, policy_types[mode]);
1753 p += l;
1754
1755 if (!nodes_empty(nodes)) {
1756 if (buffer + maxlen < p + 2)
1757 return -ENOSPC;
1758 *p++ = '=';
1759 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
1760 }
1761 return p - buffer;
1762}
1763
1764struct numa_maps {
1765 unsigned long pages;
1766 unsigned long anon;
Christoph Lameter397874d2006-03-06 15:42:53 -08001767 unsigned long active;
1768 unsigned long writeback;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001769 unsigned long mapcount_max;
Christoph Lameter397874d2006-03-06 15:42:53 -08001770 unsigned long dirty;
1771 unsigned long swapcache;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001772 unsigned long node[MAX_NUMNODES];
1773};
1774
Christoph Lameter397874d2006-03-06 15:42:53 -08001775static void gather_stats(struct page *page, void *private, int pte_dirty)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001776{
1777 struct numa_maps *md = private;
1778 int count = page_mapcount(page);
1779
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001780 md->pages++;
Christoph Lameter397874d2006-03-06 15:42:53 -08001781 if (pte_dirty || PageDirty(page))
1782 md->dirty++;
1783
1784 if (PageSwapCache(page))
1785 md->swapcache++;
1786
1787 if (PageActive(page))
1788 md->active++;
1789
1790 if (PageWriteback(page))
1791 md->writeback++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001792
1793 if (PageAnon(page))
1794 md->anon++;
1795
Christoph Lameter397874d2006-03-06 15:42:53 -08001796 if (count > md->mapcount_max)
1797 md->mapcount_max = count;
1798
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001799 md->node[page_to_nid(page)]++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001800}
1801
Andrew Morton7f709ed2006-03-07 21:55:22 -08001802#ifdef CONFIG_HUGETLB_PAGE
Christoph Lameter397874d2006-03-06 15:42:53 -08001803static void check_huge_range(struct vm_area_struct *vma,
1804 unsigned long start, unsigned long end,
1805 struct numa_maps *md)
1806{
1807 unsigned long addr;
1808 struct page *page;
1809
1810 for (addr = start; addr < end; addr += HPAGE_SIZE) {
1811 pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
1812 pte_t pte;
1813
1814 if (!ptep)
1815 continue;
1816
1817 pte = *ptep;
1818 if (pte_none(pte))
1819 continue;
1820
1821 page = pte_page(pte);
1822 if (!page)
1823 continue;
1824
1825 gather_stats(page, md, pte_dirty(*ptep));
1826 }
1827}
Andrew Morton7f709ed2006-03-07 21:55:22 -08001828#else
1829static inline void check_huge_range(struct vm_area_struct *vma,
1830 unsigned long start, unsigned long end,
1831 struct numa_maps *md)
1832{
1833}
1834#endif
Christoph Lameter397874d2006-03-06 15:42:53 -08001835
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001836int show_numa_map(struct seq_file *m, void *v)
1837{
Eric W. Biederman99f89552006-06-26 00:25:55 -07001838 struct proc_maps_private *priv = m->private;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001839 struct vm_area_struct *vma = v;
1840 struct numa_maps *md;
Christoph Lameter397874d2006-03-06 15:42:53 -08001841 struct file *file = vma->vm_file;
1842 struct mm_struct *mm = vma->vm_mm;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001843 int n;
1844 char buffer[50];
1845
Christoph Lameter397874d2006-03-06 15:42:53 -08001846 if (!mm)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001847 return 0;
1848
1849 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
1850 if (!md)
1851 return 0;
1852
Christoph Lameter397874d2006-03-06 15:42:53 -08001853 mpol_to_str(buffer, sizeof(buffer),
Eric W. Biederman99f89552006-06-26 00:25:55 -07001854 get_vma_policy(priv->task, vma, vma->vm_start));
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001855
Christoph Lameter397874d2006-03-06 15:42:53 -08001856 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001857
Christoph Lameter397874d2006-03-06 15:42:53 -08001858 if (file) {
1859 seq_printf(m, " file=");
Josef Sipeke9536ae2006-12-08 02:37:21 -08001860 seq_path(m, file->f_path.mnt, file->f_path.dentry, "\n\t= ");
Christoph Lameter397874d2006-03-06 15:42:53 -08001861 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1862 seq_printf(m, " heap");
1863 } else if (vma->vm_start <= mm->start_stack &&
1864 vma->vm_end >= mm->start_stack) {
1865 seq_printf(m, " stack");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001866 }
Christoph Lameter397874d2006-03-06 15:42:53 -08001867
1868 if (is_vm_hugetlb_page(vma)) {
1869 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
1870 seq_printf(m, " huge");
1871 } else {
1872 check_pgd_range(vma, vma->vm_start, vma->vm_end,
1873 &node_online_map, MPOL_MF_STATS, md);
1874 }
1875
1876 if (!md->pages)
1877 goto out;
1878
1879 if (md->anon)
1880 seq_printf(m," anon=%lu",md->anon);
1881
1882 if (md->dirty)
1883 seq_printf(m," dirty=%lu",md->dirty);
1884
1885 if (md->pages != md->anon && md->pages != md->dirty)
1886 seq_printf(m, " mapped=%lu", md->pages);
1887
1888 if (md->mapcount_max > 1)
1889 seq_printf(m, " mapmax=%lu", md->mapcount_max);
1890
1891 if (md->swapcache)
1892 seq_printf(m," swapcache=%lu", md->swapcache);
1893
1894 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1895 seq_printf(m," active=%lu", md->active);
1896
1897 if (md->writeback)
1898 seq_printf(m," writeback=%lu", md->writeback);
1899
1900 for_each_online_node(n)
1901 if (md->node[n])
1902 seq_printf(m, " N%d=%lu", n, md->node[n]);
1903out:
1904 seq_putc(m, '\n');
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001905 kfree(md);
1906
1907 if (m->count < m->size)
Eric W. Biederman99f89552006-06-26 00:25:55 -07001908 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001909 return 0;
1910}
1911