blob: bb54b88c3d5aaab2752569f046129bea8082f4ec [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
66 could replace all the switch()es with a mempolicy_ops structure.
67*/
68
69#include <linux/mempolicy.h>
70#include <linux/mm.h>
71#include <linux/highmem.h>
72#include <linux/hugetlb.h>
73#include <linux/kernel.h>
74#include <linux/sched.h>
75#include <linux/mm.h>
76#include <linux/nodemask.h>
77#include <linux/cpuset.h>
78#include <linux/gfp.h>
79#include <linux/slab.h>
80#include <linux/string.h>
81#include <linux/module.h>
82#include <linux/interrupt.h>
83#include <linux/init.h>
84#include <linux/compat.h>
85#include <linux/mempolicy.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080086#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080087#include <linux/seq_file.h>
88#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080089#include <linux/migrate.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070090#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070091#include <linux/security.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080092
Linus Torvalds1da177e2005-04-16 15:20:36 -070093#include <asm/tlbflush.h>
94#include <asm/uaccess.h>
95
Christoph Lameter38e35862006-01-08 01:01:01 -080096/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080097#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -080098#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080099#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800100
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800101static struct kmem_cache *policy_cache;
102static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104/* Highest zone. An specific allocation for a zone below that is not
105 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800106enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
Andi Kleend42c6992005-07-06 19:56:03 +0200108struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109 .refcnt = ATOMIC_INIT(1), /* never free it */
110 .policy = MPOL_DEFAULT,
111};
112
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113/* Do sanity checking on a policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700114static int mpol_check_policy(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700116 int empty = nodes_empty(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117
118 switch (mode) {
119 case MPOL_DEFAULT:
120 if (!empty)
121 return -EINVAL;
122 break;
123 case MPOL_BIND:
124 case MPOL_INTERLEAVE:
125 /* Preferred will only use the first bit, but allow
126 more for now. */
127 if (empty)
128 return -EINVAL;
129 break;
130 }
Andi Kleendfcd3c02005-10-29 18:15:48 -0700131 return nodes_subset(*nodes, node_online_map) ? 0 : -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132}
Andi Kleendd942ae2006-02-17 01:39:16 +0100133
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134/* Generate a custom zonelist for the BIND policy. */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700135static struct zonelist *bind_zonelist(nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136{
137 struct zonelist *zl;
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700138 int num, max, nd;
139 enum zone_type k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
Andi Kleendfcd3c02005-10-29 18:15:48 -0700141 max = 1 + MAX_NR_ZONES * nodes_weight(*nodes);
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800142 max++; /* space for zlcache_ptr (see mmzone.h) */
Andi Kleendd942ae2006-02-17 01:39:16 +0100143 zl = kmalloc(sizeof(struct zone *) * max, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 if (!zl)
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800145 return ERR_PTR(-ENOMEM);
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800146 zl->zlcache_ptr = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 num = 0;
Andi Kleendd942ae2006-02-17 01:39:16 +0100148 /* First put in the highest zones from all nodes, then all the next
149 lower zones etc. Avoid empty zones because the memory allocator
150 doesn't like them. If you implement node hot removal you
151 have to fix that. */
Mel Gormanb377fd32007-08-22 14:02:05 -0700152 k = MAX_NR_ZONES - 1;
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700153 while (1) {
Andi Kleendd942ae2006-02-17 01:39:16 +0100154 for_each_node_mask(nd, *nodes) {
155 struct zone *z = &NODE_DATA(nd)->node_zones[k];
156 if (z->present_pages > 0)
157 zl->zones[num++] = z;
158 }
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700159 if (k == 0)
160 break;
161 k--;
Andi Kleendd942ae2006-02-17 01:39:16 +0100162 }
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800163 if (num == 0) {
164 kfree(zl);
165 return ERR_PTR(-EINVAL);
166 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 zl->zones[num] = NULL;
168 return zl;
169}
170
171/* Create a new policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700172static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173{
174 struct mempolicy *policy;
175
Paul Mundt140d5a42007-07-15 23:38:16 -0700176 pr_debug("setting mode %d nodes[0] %lx\n",
177 mode, nodes ? nodes_addr(*nodes)[0] : -1);
178
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 if (mode == MPOL_DEFAULT)
180 return NULL;
181 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
182 if (!policy)
183 return ERR_PTR(-ENOMEM);
184 atomic_set(&policy->refcnt, 1);
185 switch (mode) {
186 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700187 policy->v.nodes = *nodes;
Andi Kleen8f493d72006-01-03 00:07:28 +0100188 if (nodes_weight(*nodes) == 0) {
189 kmem_cache_free(policy_cache, policy);
190 return ERR_PTR(-EINVAL);
191 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 break;
193 case MPOL_PREFERRED:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700194 policy->v.preferred_node = first_node(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 if (policy->v.preferred_node >= MAX_NUMNODES)
196 policy->v.preferred_node = -1;
197 break;
198 case MPOL_BIND:
199 policy->v.zonelist = bind_zonelist(nodes);
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800200 if (IS_ERR(policy->v.zonelist)) {
201 void *error_code = policy->v.zonelist;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 kmem_cache_free(policy_cache, policy);
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800203 return error_code;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 }
205 break;
206 }
207 policy->policy = mode;
Paul Jackson74cb2152006-01-08 01:01:56 -0800208 policy->cpuset_mems_allowed = cpuset_mems_allowed(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 return policy;
210}
211
Christoph Lameter397874d2006-03-06 15:42:53 -0800212static void gather_stats(struct page *, void *, int pte_dirty);
Christoph Lameterfc301282006-01-18 17:42:29 -0800213static void migrate_page_add(struct page *page, struct list_head *pagelist,
214 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800215
Christoph Lameter38e35862006-01-08 01:01:01 -0800216/* Scan through pages checking if pages follow certain conditions. */
Nick Pigginb5810032005-10-29 18:16:12 -0700217static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800218 unsigned long addr, unsigned long end,
219 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800220 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
Hugh Dickins91612e02005-06-21 17:15:07 -0700222 pte_t *orig_pte;
223 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700224 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700225
Hugh Dickins705e87c2005-10-29 18:16:27 -0700226 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700227 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800228 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800229 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700230
231 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800233 page = vm_normal_page(vma, addr, *pte);
234 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800236 /*
237 * The check for PageReserved here is important to avoid
238 * handling zero pages and other pages that may have been
239 * marked special by the system.
240 *
241 * If the PageReserved would not be checked here then f.e.
242 * the location of the zero page could have an influence
243 * on MPOL_MF_STRICT, zero pages would be counted for
244 * the per node stats, and there would be useless attempts
245 * to put zero pages on the migration list.
246 */
Christoph Lameterf4598c82006-01-12 01:05:20 -0800247 if (PageReserved(page))
248 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800249 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800250 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
251 continue;
252
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800253 if (flags & MPOL_MF_STATS)
Christoph Lameter397874d2006-03-06 15:42:53 -0800254 gather_stats(page, private, pte_dirty(*pte));
Nick Piggin053837f2006-01-18 17:42:27 -0800255 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800256 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800257 else
258 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700259 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700260 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700261 return addr != end;
262}
263
Nick Pigginb5810032005-10-29 18:16:12 -0700264static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800265 unsigned long addr, unsigned long end,
266 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800267 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700268{
269 pmd_t *pmd;
270 unsigned long next;
271
272 pmd = pmd_offset(pud, addr);
273 do {
274 next = pmd_addr_end(addr, end);
275 if (pmd_none_or_clear_bad(pmd))
276 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800277 if (check_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800278 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700279 return -EIO;
280 } while (pmd++, addr = next, addr != end);
281 return 0;
282}
283
Nick Pigginb5810032005-10-29 18:16:12 -0700284static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800285 unsigned long addr, unsigned long end,
286 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800287 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700288{
289 pud_t *pud;
290 unsigned long next;
291
292 pud = pud_offset(pgd, addr);
293 do {
294 next = pud_addr_end(addr, end);
295 if (pud_none_or_clear_bad(pud))
296 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800297 if (check_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800298 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700299 return -EIO;
300 } while (pud++, addr = next, addr != end);
301 return 0;
302}
303
Nick Pigginb5810032005-10-29 18:16:12 -0700304static inline int check_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800305 unsigned long addr, unsigned long end,
306 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800307 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700308{
309 pgd_t *pgd;
310 unsigned long next;
311
Nick Pigginb5810032005-10-29 18:16:12 -0700312 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700313 do {
314 next = pgd_addr_end(addr, end);
315 if (pgd_none_or_clear_bad(pgd))
316 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800317 if (check_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800318 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700319 return -EIO;
320 } while (pgd++, addr = next, addr != end);
321 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322}
323
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800324/*
325 * Check if all pages in a range are on a set of nodes.
326 * If pagelist != NULL then isolate pages from the LRU and
327 * put them on the pagelist.
328 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329static struct vm_area_struct *
330check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800331 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332{
333 int err;
334 struct vm_area_struct *first, *vma, *prev;
335
Christoph Lameter90036ee2006-03-16 23:03:59 -0800336 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
Christoph Lameter90036ee2006-03-16 23:03:59 -0800337
Christoph Lameterb20a3502006-03-22 00:09:12 -0800338 err = migrate_prep();
339 if (err)
340 return ERR_PTR(err);
Christoph Lameter90036ee2006-03-16 23:03:59 -0800341 }
Nick Piggin053837f2006-01-18 17:42:27 -0800342
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 first = find_vma(mm, start);
344 if (!first)
345 return ERR_PTR(-EFAULT);
346 prev = NULL;
347 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800348 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
349 if (!vma->vm_next && vma->vm_end < end)
350 return ERR_PTR(-EFAULT);
351 if (prev && prev->vm_end < vma->vm_start)
352 return ERR_PTR(-EFAULT);
353 }
354 if (!is_vm_hugetlb_page(vma) &&
355 ((flags & MPOL_MF_STRICT) ||
356 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
357 vma_migratable(vma)))) {
Andi Kleen5b952b32005-09-13 01:25:08 -0700358 unsigned long endvma = vma->vm_end;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800359
Andi Kleen5b952b32005-09-13 01:25:08 -0700360 if (endvma > end)
361 endvma = end;
362 if (vma->vm_start > start)
363 start = vma->vm_start;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800364 err = check_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800365 flags, private);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 if (err) {
367 first = ERR_PTR(err);
368 break;
369 }
370 }
371 prev = vma;
372 }
373 return first;
374}
375
376/* Apply policy to a single VMA */
377static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
378{
379 int err = 0;
380 struct mempolicy *old = vma->vm_policy;
381
Paul Mundt140d5a42007-07-15 23:38:16 -0700382 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 vma->vm_start, vma->vm_end, vma->vm_pgoff,
384 vma->vm_ops, vma->vm_file,
385 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
386
387 if (vma->vm_ops && vma->vm_ops->set_policy)
388 err = vma->vm_ops->set_policy(vma, new);
389 if (!err) {
390 mpol_get(new);
391 vma->vm_policy = new;
392 mpol_free(old);
393 }
394 return err;
395}
396
397/* Step 2: apply policy to a range and do splits. */
398static int mbind_range(struct vm_area_struct *vma, unsigned long start,
399 unsigned long end, struct mempolicy *new)
400{
401 struct vm_area_struct *next;
402 int err;
403
404 err = 0;
405 for (; vma && vma->vm_start < end; vma = next) {
406 next = vma->vm_next;
407 if (vma->vm_start < start)
408 err = split_vma(vma->vm_mm, vma, start, 1);
409 if (!err && vma->vm_end > end)
410 err = split_vma(vma->vm_mm, vma, end, 0);
411 if (!err)
412 err = policy_vma(vma, new);
413 if (err)
414 break;
415 }
416 return err;
417}
418
Christoph Lameter8bccd852005-10-29 18:16:59 -0700419static int contextualize_policy(int mode, nodemask_t *nodes)
420{
421 if (!nodes)
422 return 0;
423
Paul Jacksoncf2a4732006-01-08 01:01:54 -0800424 cpuset_update_task_memory_state();
Paul Jackson59665142006-01-08 01:01:47 -0800425 if (!cpuset_nodes_subset_current_mems_allowed(*nodes))
426 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700427 return mpol_check_policy(mode, nodes);
428}
429
Paul Jacksonc61afb12006-03-24 03:16:08 -0800430
431/*
432 * Update task->flags PF_MEMPOLICY bit: set iff non-default
433 * mempolicy. Allows more rapid checking of this (combined perhaps
434 * with other PF_* flag bits) on memory allocation hot code paths.
435 *
436 * If called from outside this file, the task 'p' should -only- be
437 * a newly forked child not yet visible on the task list, because
438 * manipulating the task flags of a visible task is not safe.
439 *
440 * The above limitation is why this routine has the funny name
441 * mpol_fix_fork_child_flag().
442 *
443 * It is also safe to call this with a task pointer of current,
444 * which the static wrapper mpol_set_task_struct_flag() does,
445 * for use within this file.
446 */
447
448void mpol_fix_fork_child_flag(struct task_struct *p)
449{
450 if (p->mempolicy)
451 p->flags |= PF_MEMPOLICY;
452 else
453 p->flags &= ~PF_MEMPOLICY;
454}
455
456static void mpol_set_task_struct_flag(void)
457{
458 mpol_fix_fork_child_flag(current);
459}
460
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461/* Set the process memory policy */
Christoph Lameter8bccd852005-10-29 18:16:59 -0700462long do_set_mempolicy(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 struct mempolicy *new;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
Christoph Lameter8bccd852005-10-29 18:16:59 -0700466 if (contextualize_policy(mode, nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700468 new = mpol_new(mode, nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 if (IS_ERR(new))
470 return PTR_ERR(new);
471 mpol_free(current->mempolicy);
472 current->mempolicy = new;
Paul Jacksonc61afb12006-03-24 03:16:08 -0800473 mpol_set_task_struct_flag();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 if (new && new->policy == MPOL_INTERLEAVE)
Andi Kleendfcd3c02005-10-29 18:15:48 -0700475 current->il_next = first_node(new->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 return 0;
477}
478
479/* Fill a zone bitmap for a policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700480static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481{
482 int i;
483
Andi Kleendfcd3c02005-10-29 18:15:48 -0700484 nodes_clear(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 switch (p->policy) {
486 case MPOL_BIND:
487 for (i = 0; p->v.zonelist->zones[i]; i++)
Christoph Lameter89fa3022006-09-25 23:31:55 -0700488 node_set(zone_to_nid(p->v.zonelist->zones[i]),
Christoph Lameter8bccd852005-10-29 18:16:59 -0700489 *nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 break;
491 case MPOL_DEFAULT:
492 break;
493 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700494 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 break;
496 case MPOL_PREFERRED:
497 /* or use current node instead of online map? */
498 if (p->v.preferred_node < 0)
Andi Kleendfcd3c02005-10-29 18:15:48 -0700499 *nodes = node_online_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 else
Andi Kleendfcd3c02005-10-29 18:15:48 -0700501 node_set(p->v.preferred_node, *nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 break;
503 default:
504 BUG();
505 }
506}
507
508static int lookup_node(struct mm_struct *mm, unsigned long addr)
509{
510 struct page *p;
511 int err;
512
513 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
514 if (err >= 0) {
515 err = page_to_nid(p);
516 put_page(p);
517 }
518 return err;
519}
520
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521/* Retrieve NUMA policy */
Christoph Lameter8bccd852005-10-29 18:16:59 -0700522long do_get_mempolicy(int *policy, nodemask_t *nmask,
523 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700525 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 struct mm_struct *mm = current->mm;
527 struct vm_area_struct *vma = NULL;
528 struct mempolicy *pol = current->mempolicy;
529
Paul Jacksoncf2a4732006-01-08 01:01:54 -0800530 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR))
532 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 if (flags & MPOL_F_ADDR) {
534 down_read(&mm->mmap_sem);
535 vma = find_vma_intersection(mm, addr, addr+1);
536 if (!vma) {
537 up_read(&mm->mmap_sem);
538 return -EFAULT;
539 }
540 if (vma->vm_ops && vma->vm_ops->get_policy)
541 pol = vma->vm_ops->get_policy(vma, addr);
542 else
543 pol = vma->vm_policy;
544 } else if (addr)
545 return -EINVAL;
546
547 if (!pol)
548 pol = &default_policy;
549
550 if (flags & MPOL_F_NODE) {
551 if (flags & MPOL_F_ADDR) {
552 err = lookup_node(mm, addr);
553 if (err < 0)
554 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700555 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 } else if (pol == current->mempolicy &&
557 pol->policy == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700558 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 } else {
560 err = -EINVAL;
561 goto out;
562 }
563 } else
Christoph Lameter8bccd852005-10-29 18:16:59 -0700564 *policy = pol->policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565
566 if (vma) {
567 up_read(&current->mm->mmap_sem);
568 vma = NULL;
569 }
570
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 err = 0;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700572 if (nmask)
573 get_zonemask(pol, nmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574
575 out:
576 if (vma)
577 up_read(&current->mm->mmap_sem);
578 return err;
579}
580
Christoph Lameterb20a3502006-03-22 00:09:12 -0800581#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700582/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800583 * page migration
584 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800585static void migrate_page_add(struct page *page, struct list_head *pagelist,
586 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800587{
588 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800589 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800590 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800591 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
592 isolate_lru_page(page, pagelist);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800593}
594
Christoph Lameter742755a2006-06-23 02:03:55 -0700595static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700596{
Mel Gorman769848c2007-07-17 04:03:05 -0700597 return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700598}
599
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800600/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800601 * Migrate pages from one node to a target node.
602 * Returns error or the number of pages not migrated.
603 */
604int migrate_to_node(struct mm_struct *mm, int source, int dest, int flags)
605{
606 nodemask_t nmask;
607 LIST_HEAD(pagelist);
608 int err = 0;
609
610 nodes_clear(nmask);
611 node_set(source, nmask);
612
613 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
614 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
615
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700616 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700617 err = migrate_pages(&pagelist, new_node_page, dest);
618
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800619 return err;
620}
621
622/*
623 * Move pages between the two nodesets so as to preserve the physical
624 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -0800625 *
626 * Returns the number of page that could not be moved.
627 */
628int do_migrate_pages(struct mm_struct *mm,
629 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
630{
631 LIST_HEAD(pagelist);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800632 int busy = 0;
633 int err = 0;
634 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -0800635
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800636 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -0800637
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700638 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
639 if (err)
640 goto out;
641
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800642/*
643 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
644 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
645 * bit in 'tmp', and return that <source, dest> pair for migration.
646 * The pair of nodemasks 'to' and 'from' define the map.
647 *
648 * If no pair of bits is found that way, fallback to picking some
649 * pair of 'source' and 'dest' bits that are not the same. If the
650 * 'source' and 'dest' bits are the same, this represents a node
651 * that will be migrating to itself, so no pages need move.
652 *
653 * If no bits are left in 'tmp', or if all remaining bits left
654 * in 'tmp' correspond to the same bit in 'to', return false
655 * (nothing left to migrate).
656 *
657 * This lets us pick a pair of nodes to migrate between, such that
658 * if possible the dest node is not already occupied by some other
659 * source node, minimizing the risk of overloading the memory on a
660 * node that would happen if we migrated incoming memory to a node
661 * before migrating outgoing memory source that same node.
662 *
663 * A single scan of tmp is sufficient. As we go, we remember the
664 * most recent <s, d> pair that moved (s != d). If we find a pair
665 * that not only moved, but what's better, moved to an empty slot
666 * (d is not set in tmp), then we break out then, with that pair.
667 * Otherwise when we finish scannng from_tmp, we at least have the
668 * most recent <s, d> pair that moved. If we get all the way through
669 * the scan of tmp without finding any node that moved, much less
670 * moved to an empty node, then there is nothing left worth migrating.
671 */
Christoph Lameterd4984712006-01-08 01:00:55 -0800672
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800673 tmp = *from_nodes;
674 while (!nodes_empty(tmp)) {
675 int s,d;
676 int source = -1;
677 int dest = 0;
678
679 for_each_node_mask(s, tmp) {
680 d = node_remap(s, *from_nodes, *to_nodes);
681 if (s == d)
682 continue;
683
684 source = s; /* Node moved. Memorize */
685 dest = d;
686
687 /* dest not in remaining from nodes? */
688 if (!node_isset(dest, tmp))
689 break;
690 }
691 if (source == -1)
692 break;
693
694 node_clear(source, tmp);
695 err = migrate_to_node(mm, source, dest, flags);
696 if (err > 0)
697 busy += err;
698 if (err < 0)
699 break;
Christoph Lameter39743882006-01-08 01:00:51 -0800700 }
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700701out:
Christoph Lameter39743882006-01-08 01:00:51 -0800702 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800703 if (err < 0)
704 return err;
705 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800706
Christoph Lameter39743882006-01-08 01:00:51 -0800707}
708
Christoph Lameter742755a2006-06-23 02:03:55 -0700709static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700710{
711 struct vm_area_struct *vma = (struct vm_area_struct *)private;
712
Mel Gorman769848c2007-07-17 04:03:05 -0700713 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
714 page_address_in_vma(page, vma));
Christoph Lameter95a402c2006-06-23 02:03:53 -0700715}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800716#else
717
718static void migrate_page_add(struct page *page, struct list_head *pagelist,
719 unsigned long flags)
720{
721}
722
723int do_migrate_pages(struct mm_struct *mm,
724 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
725{
726 return -ENOSYS;
727}
Christoph Lameter95a402c2006-06-23 02:03:53 -0700728
Keith Owens69939742006-10-11 01:21:28 -0700729static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700730{
731 return NULL;
732}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800733#endif
734
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800735long do_mbind(unsigned long start, unsigned long len,
736 unsigned long mode, nodemask_t *nmask, unsigned long flags)
737{
738 struct vm_area_struct *vma;
739 struct mm_struct *mm = current->mm;
740 struct mempolicy *new;
741 unsigned long end;
742 int err;
743 LIST_HEAD(pagelist);
744
745 if ((flags & ~(unsigned long)(MPOL_MF_STRICT |
746 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
747 || mode > MPOL_MAX)
748 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -0800749 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800750 return -EPERM;
751
752 if (start & ~PAGE_MASK)
753 return -EINVAL;
754
755 if (mode == MPOL_DEFAULT)
756 flags &= ~MPOL_MF_STRICT;
757
758 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
759 end = start + len;
760
761 if (end < start)
762 return -EINVAL;
763 if (end == start)
764 return 0;
765
766 if (mpol_check_policy(mode, nmask))
767 return -EINVAL;
768
769 new = mpol_new(mode, nmask);
770 if (IS_ERR(new))
771 return PTR_ERR(new);
772
773 /*
774 * If we are using the default policy then operation
775 * on discontinuous address spaces is okay after all
776 */
777 if (!new)
778 flags |= MPOL_MF_DISCONTIG_OK;
779
Paul Mundt140d5a42007-07-15 23:38:16 -0700780 pr_debug("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len,
781 mode, nmask ? nodes_addr(*nmask)[0] : -1);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800782
783 down_write(&mm->mmap_sem);
784 vma = check_range(mm, start, end, nmask,
785 flags | MPOL_MF_INVERT, &pagelist);
786
787 err = PTR_ERR(vma);
788 if (!IS_ERR(vma)) {
789 int nr_failed = 0;
790
791 err = mbind_range(vma, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800792
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800793 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700794 nr_failed = migrate_pages(&pagelist, new_vma_page,
795 (unsigned long)vma);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800796
797 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
798 err = -EIO;
799 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800800
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800801 up_write(&mm->mmap_sem);
802 mpol_free(new);
803 return err;
804}
805
Christoph Lameter39743882006-01-08 01:00:51 -0800806/*
Christoph Lameter8bccd852005-10-29 18:16:59 -0700807 * User space interface with variable sized bitmaps for nodelists.
808 */
809
810/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -0800811static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -0700812 unsigned long maxnode)
813{
814 unsigned long k;
815 unsigned long nlongs;
816 unsigned long endmask;
817
818 --maxnode;
819 nodes_clear(*nodes);
820 if (maxnode == 0 || !nmask)
821 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -0800822 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -0800823 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700824
825 nlongs = BITS_TO_LONGS(maxnode);
826 if ((maxnode % BITS_PER_LONG) == 0)
827 endmask = ~0UL;
828 else
829 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
830
831 /* When the user specified more nodes than supported just check
832 if the non supported part is all zero. */
833 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
834 if (nlongs > PAGE_SIZE/sizeof(long))
835 return -EINVAL;
836 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
837 unsigned long t;
838 if (get_user(t, nmask + k))
839 return -EFAULT;
840 if (k == nlongs - 1) {
841 if (t & endmask)
842 return -EINVAL;
843 } else if (t)
844 return -EINVAL;
845 }
846 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
847 endmask = ~0UL;
848 }
849
850 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
851 return -EFAULT;
852 nodes_addr(*nodes)[nlongs-1] &= endmask;
853 return 0;
854}
855
856/* Copy a kernel node mask to user space */
857static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
858 nodemask_t *nodes)
859{
860 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
861 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
862
863 if (copy > nbytes) {
864 if (copy > PAGE_SIZE)
865 return -EINVAL;
866 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
867 return -EFAULT;
868 copy = nbytes;
869 }
870 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
871}
872
873asmlinkage long sys_mbind(unsigned long start, unsigned long len,
874 unsigned long mode,
875 unsigned long __user *nmask, unsigned long maxnode,
876 unsigned flags)
877{
878 nodemask_t nodes;
879 int err;
880
881 err = get_nodes(&nodes, nmask, maxnode);
882 if (err)
883 return err;
Christoph Lameter30150f82007-01-22 20:40:45 -0800884#ifdef CONFIG_CPUSETS
885 /* Restrict the nodes to the allowed nodes in the cpuset */
886 nodes_and(nodes, nodes, current->mems_allowed);
887#endif
Christoph Lameter8bccd852005-10-29 18:16:59 -0700888 return do_mbind(start, len, mode, &nodes, flags);
889}
890
891/* Set the process memory policy */
892asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
893 unsigned long maxnode)
894{
895 int err;
896 nodemask_t nodes;
897
898 if (mode < 0 || mode > MPOL_MAX)
899 return -EINVAL;
900 err = get_nodes(&nodes, nmask, maxnode);
901 if (err)
902 return err;
903 return do_set_mempolicy(mode, &nodes);
904}
905
Christoph Lameter39743882006-01-08 01:00:51 -0800906asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
907 const unsigned long __user *old_nodes,
908 const unsigned long __user *new_nodes)
909{
910 struct mm_struct *mm;
911 struct task_struct *task;
912 nodemask_t old;
913 nodemask_t new;
914 nodemask_t task_nodes;
915 int err;
916
917 err = get_nodes(&old, old_nodes, maxnode);
918 if (err)
919 return err;
920
921 err = get_nodes(&new, new_nodes, maxnode);
922 if (err)
923 return err;
924
925 /* Find the mm_struct */
926 read_lock(&tasklist_lock);
927 task = pid ? find_task_by_pid(pid) : current;
928 if (!task) {
929 read_unlock(&tasklist_lock);
930 return -ESRCH;
931 }
932 mm = get_task_mm(task);
933 read_unlock(&tasklist_lock);
934
935 if (!mm)
936 return -EINVAL;
937
938 /*
939 * Check if this process has the right to modify the specified
940 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -0800941 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -0800942 * userid as the target process.
943 */
944 if ((current->euid != task->suid) && (current->euid != task->uid) &&
945 (current->uid != task->suid) && (current->uid != task->uid) &&
Christoph Lameter74c00242006-03-14 19:50:21 -0800946 !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -0800947 err = -EPERM;
948 goto out;
949 }
950
951 task_nodes = cpuset_mems_allowed(task);
952 /* Is the user allowed to access the target nodes? */
Christoph Lameter74c00242006-03-14 19:50:21 -0800953 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -0800954 err = -EPERM;
955 goto out;
956 }
957
Christoph Lameter3b42d282007-08-31 00:12:08 -0700958 if (!nodes_subset(new, node_online_map)) {
959 err = -EINVAL;
960 goto out;
961 }
962
David Quigley86c3a762006-06-23 02:04:02 -0700963 err = security_task_movememory(task);
964 if (err)
965 goto out;
966
Christoph Lameter511030b2006-02-28 16:58:57 -0800967 err = do_migrate_pages(mm, &old, &new,
Christoph Lameter74c00242006-03-14 19:50:21 -0800968 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter39743882006-01-08 01:00:51 -0800969out:
970 mmput(mm);
971 return err;
972}
973
974
Christoph Lameter8bccd852005-10-29 18:16:59 -0700975/* Retrieve NUMA policy */
976asmlinkage long sys_get_mempolicy(int __user *policy,
977 unsigned long __user *nmask,
978 unsigned long maxnode,
979 unsigned long addr, unsigned long flags)
980{
981 int err, pval;
982 nodemask_t nodes;
983
984 if (nmask != NULL && maxnode < MAX_NUMNODES)
985 return -EINVAL;
986
987 err = do_get_mempolicy(&pval, &nodes, addr, flags);
988
989 if (err)
990 return err;
991
992 if (policy && put_user(pval, policy))
993 return -EFAULT;
994
995 if (nmask)
996 err = copy_nodes_to_user(nmask, maxnode, &nodes);
997
998 return err;
999}
1000
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001#ifdef CONFIG_COMPAT
1002
1003asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1004 compat_ulong_t __user *nmask,
1005 compat_ulong_t maxnode,
1006 compat_ulong_t addr, compat_ulong_t flags)
1007{
1008 long err;
1009 unsigned long __user *nm = NULL;
1010 unsigned long nr_bits, alloc_size;
1011 DECLARE_BITMAP(bm, MAX_NUMNODES);
1012
1013 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1014 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1015
1016 if (nmask)
1017 nm = compat_alloc_user_space(alloc_size);
1018
1019 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1020
1021 if (!err && nmask) {
1022 err = copy_from_user(bm, nm, alloc_size);
1023 /* ensure entire bitmap is zeroed */
1024 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1025 err |= compat_put_bitmap(nmask, bm, nr_bits);
1026 }
1027
1028 return err;
1029}
1030
1031asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1032 compat_ulong_t maxnode)
1033{
1034 long err = 0;
1035 unsigned long __user *nm = NULL;
1036 unsigned long nr_bits, alloc_size;
1037 DECLARE_BITMAP(bm, MAX_NUMNODES);
1038
1039 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1040 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1041
1042 if (nmask) {
1043 err = compat_get_bitmap(bm, nmask, nr_bits);
1044 nm = compat_alloc_user_space(alloc_size);
1045 err |= copy_to_user(nm, bm, alloc_size);
1046 }
1047
1048 if (err)
1049 return -EFAULT;
1050
1051 return sys_set_mempolicy(mode, nm, nr_bits+1);
1052}
1053
1054asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1055 compat_ulong_t mode, compat_ulong_t __user *nmask,
1056 compat_ulong_t maxnode, compat_ulong_t flags)
1057{
1058 long err = 0;
1059 unsigned long __user *nm = NULL;
1060 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001061 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062
1063 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1064 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1065
1066 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001067 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001069 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 }
1071
1072 if (err)
1073 return -EFAULT;
1074
1075 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1076}
1077
1078#endif
1079
1080/* Return effective policy for a VMA */
Christoph Lameter48fce342006-01-08 01:01:03 -08001081static struct mempolicy * get_vma_policy(struct task_struct *task,
1082 struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001084 struct mempolicy *pol = task->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085
1086 if (vma) {
1087 if (vma->vm_ops && vma->vm_ops->get_policy)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001088 pol = vma->vm_ops->get_policy(vma, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089 else if (vma->vm_policy &&
1090 vma->vm_policy->policy != MPOL_DEFAULT)
1091 pol = vma->vm_policy;
1092 }
1093 if (!pol)
1094 pol = &default_policy;
1095 return pol;
1096}
1097
1098/* Return a zonelist representing a mempolicy */
Al Virodd0fc662005-10-07 07:46:04 +01001099static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100{
1101 int nd;
1102
1103 switch (policy->policy) {
1104 case MPOL_PREFERRED:
1105 nd = policy->v.preferred_node;
1106 if (nd < 0)
1107 nd = numa_node_id();
1108 break;
1109 case MPOL_BIND:
1110 /* Lower zones don't get a policy applied */
1111 /* Careful: current->mems_allowed might have moved */
Christoph Lameter19655d32006-09-25 23:31:19 -07001112 if (gfp_zone(gfp) >= policy_zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist))
1114 return policy->v.zonelist;
1115 /*FALL THROUGH*/
1116 case MPOL_INTERLEAVE: /* should not happen */
1117 case MPOL_DEFAULT:
1118 nd = numa_node_id();
1119 break;
1120 default:
1121 nd = 0;
1122 BUG();
1123 }
Al Viroaf4ca452005-10-21 02:55:38 -04001124 return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125}
1126
1127/* Do dynamic interleaving for a process */
1128static unsigned interleave_nodes(struct mempolicy *policy)
1129{
1130 unsigned nid, next;
1131 struct task_struct *me = current;
1132
1133 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001134 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001136 next = first_node(policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 me->il_next = next;
1138 return nid;
1139}
1140
Christoph Lameterdc85da12006-01-18 17:42:36 -08001141/*
1142 * Depending on the memory policy provide a node from which to allocate the
1143 * next slab entry.
1144 */
1145unsigned slab_node(struct mempolicy *policy)
1146{
Christoph Lameter765c4502006-09-27 01:50:08 -07001147 int pol = policy ? policy->policy : MPOL_DEFAULT;
1148
1149 switch (pol) {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001150 case MPOL_INTERLEAVE:
1151 return interleave_nodes(policy);
1152
1153 case MPOL_BIND:
1154 /*
1155 * Follow bind policy behavior and start allocation at the
1156 * first node.
1157 */
Christoph Lameter89fa3022006-09-25 23:31:55 -07001158 return zone_to_nid(policy->v.zonelist->zones[0]);
Christoph Lameterdc85da12006-01-18 17:42:36 -08001159
1160 case MPOL_PREFERRED:
1161 if (policy->v.preferred_node >= 0)
1162 return policy->v.preferred_node;
1163 /* Fall through */
1164
1165 default:
1166 return numa_node_id();
1167 }
1168}
1169
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170/* Do static interleaving for a VMA with known offset. */
1171static unsigned offset_il_node(struct mempolicy *pol,
1172 struct vm_area_struct *vma, unsigned long off)
1173{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001174 unsigned nnodes = nodes_weight(pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 unsigned target = (unsigned)off % nnodes;
1176 int c;
1177 int nid = -1;
1178
1179 c = 0;
1180 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001181 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 c++;
1183 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184 return nid;
1185}
1186
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001187/* Determine a node number for interleave */
1188static inline unsigned interleave_nid(struct mempolicy *pol,
1189 struct vm_area_struct *vma, unsigned long addr, int shift)
1190{
1191 if (vma) {
1192 unsigned long off;
1193
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001194 /*
1195 * for small pages, there is no difference between
1196 * shift and PAGE_SHIFT, so the bit-shift is safe.
1197 * for huge pages, since vm_pgoff is in units of small
1198 * pages, we need to shift off the always 0 bits to get
1199 * a useful offset.
1200 */
1201 BUG_ON(shift < PAGE_SHIFT);
1202 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001203 off += (addr - vma->vm_start) >> shift;
1204 return offset_il_node(pol, vma, off);
1205 } else
1206 return interleave_nodes(pol);
1207}
1208
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001209#ifdef CONFIG_HUGETLBFS
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001210/* Return a zonelist suitable for a huge page allocation. */
Mel Gorman396faf02007-07-17 04:03:13 -07001211struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
1212 gfp_t gfp_flags)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001213{
1214 struct mempolicy *pol = get_vma_policy(current, vma, addr);
1215
1216 if (pol->policy == MPOL_INTERLEAVE) {
1217 unsigned nid;
1218
1219 nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
Mel Gorman396faf02007-07-17 04:03:13 -07001220 return NODE_DATA(nid)->node_zonelists + gfp_zone(gfp_flags);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001221 }
1222 return zonelist_policy(GFP_HIGHUSER, pol);
1223}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001224#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001225
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226/* Allocate a page in interleaved policy.
1227 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001228static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1229 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230{
1231 struct zonelist *zl;
1232 struct page *page;
1233
Al Viroaf4ca452005-10-21 02:55:38 -04001234 zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235 page = __alloc_pages(gfp, order, zl);
Christoph Lameterca889e62006-06-30 01:55:44 -07001236 if (page && page_zone(page) == zl->zones[0])
1237 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238 return page;
1239}
1240
1241/**
1242 * alloc_page_vma - Allocate a page for a VMA.
1243 *
1244 * @gfp:
1245 * %GFP_USER user allocation.
1246 * %GFP_KERNEL kernel allocations,
1247 * %GFP_HIGHMEM highmem/user allocations,
1248 * %GFP_FS allocation should not call back into a file system.
1249 * %GFP_ATOMIC don't sleep.
1250 *
1251 * @vma: Pointer to VMA or NULL if not available.
1252 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1253 *
1254 * This function allocates a page from the kernel page pool and applies
1255 * a NUMA policy associated with the VMA or the current process.
1256 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1257 * mm_struct of the VMA to prevent it from going away. Should be used for
1258 * all allocations for pages that will be mapped into
1259 * user space. Returns NULL when no page can be allocated.
1260 *
1261 * Should be called with the mm_sem of the vma hold.
1262 */
1263struct page *
Al Virodd0fc662005-10-07 07:46:04 +01001264alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001266 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001268 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269
1270 if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
1271 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001272
1273 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274 return alloc_page_interleave(gfp, 0, nid);
1275 }
1276 return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol));
1277}
1278
1279/**
1280 * alloc_pages_current - Allocate pages.
1281 *
1282 * @gfp:
1283 * %GFP_USER user allocation,
1284 * %GFP_KERNEL kernel allocation,
1285 * %GFP_HIGHMEM highmem allocation,
1286 * %GFP_FS don't call back into a file system.
1287 * %GFP_ATOMIC don't sleep.
1288 * @order: Power of two of allocation size in pages. 0 is a single page.
1289 *
1290 * Allocate a page from the kernel page pool. When not in
1291 * interrupt context and apply the current process NUMA policy.
1292 * Returns NULL when no page can be allocated.
1293 *
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001294 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295 * 1) it's ok to take cpuset_sem (can WAIT), and
1296 * 2) allocating for current task (not interrupt).
1297 */
Al Virodd0fc662005-10-07 07:46:04 +01001298struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299{
1300 struct mempolicy *pol = current->mempolicy;
1301
1302 if ((gfp & __GFP_WAIT) && !in_interrupt())
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001303 cpuset_update_task_memory_state();
Christoph Lameter9b819d22006-09-25 23:31:40 -07001304 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 pol = &default_policy;
1306 if (pol->policy == MPOL_INTERLEAVE)
1307 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
1308 return __alloc_pages(gfp, order, zonelist_policy(gfp, pol));
1309}
1310EXPORT_SYMBOL(alloc_pages_current);
1311
Paul Jackson42253992006-01-08 01:01:59 -08001312/*
1313 * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
1314 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1315 * with the mems_allowed returned by cpuset_mems_allowed(). This
1316 * keeps mempolicies cpuset relative after its cpuset moves. See
1317 * further kernel/cpuset.c update_nodemask().
1318 */
1319void *cpuset_being_rebound;
1320
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321/* Slow path of a mempolicy copy */
1322struct mempolicy *__mpol_copy(struct mempolicy *old)
1323{
1324 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1325
1326 if (!new)
1327 return ERR_PTR(-ENOMEM);
Paul Jackson42253992006-01-08 01:01:59 -08001328 if (current_cpuset_is_being_rebound()) {
1329 nodemask_t mems = cpuset_mems_allowed(current);
1330 mpol_rebind_policy(old, &mems);
1331 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001332 *new = *old;
1333 atomic_set(&new->refcnt, 1);
1334 if (new->policy == MPOL_BIND) {
1335 int sz = ksize(old->v.zonelist);
Christoph Lametere94b1762006-12-06 20:33:17 -08001336 new->v.zonelist = kmemdup(old->v.zonelist, sz, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 if (!new->v.zonelist) {
1338 kmem_cache_free(policy_cache, new);
1339 return ERR_PTR(-ENOMEM);
1340 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341 }
1342 return new;
1343}
1344
1345/* Slow path of a mempolicy comparison */
1346int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1347{
1348 if (!a || !b)
1349 return 0;
1350 if (a->policy != b->policy)
1351 return 0;
1352 switch (a->policy) {
1353 case MPOL_DEFAULT:
1354 return 1;
1355 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -07001356 return nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 case MPOL_PREFERRED:
1358 return a->v.preferred_node == b->v.preferred_node;
1359 case MPOL_BIND: {
1360 int i;
1361 for (i = 0; a->v.zonelist->zones[i]; i++)
1362 if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i])
1363 return 0;
1364 return b->v.zonelist->zones[i] == NULL;
1365 }
1366 default:
1367 BUG();
1368 return 0;
1369 }
1370}
1371
1372/* Slow path of a mpol destructor. */
1373void __mpol_free(struct mempolicy *p)
1374{
1375 if (!atomic_dec_and_test(&p->refcnt))
1376 return;
1377 if (p->policy == MPOL_BIND)
1378 kfree(p->v.zonelist);
1379 p->policy = MPOL_DEFAULT;
1380 kmem_cache_free(policy_cache, p);
1381}
1382
1383/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 * Shared memory backing store policy support.
1385 *
1386 * Remember policies even when nobody has shared memory mapped.
1387 * The policies are kept in Red-Black tree linked from the inode.
1388 * They are protected by the sp->lock spinlock, which should be held
1389 * for any accesses to the tree.
1390 */
1391
1392/* lookup first element intersecting start-end */
1393/* Caller holds sp->lock */
1394static struct sp_node *
1395sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1396{
1397 struct rb_node *n = sp->root.rb_node;
1398
1399 while (n) {
1400 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1401
1402 if (start >= p->end)
1403 n = n->rb_right;
1404 else if (end <= p->start)
1405 n = n->rb_left;
1406 else
1407 break;
1408 }
1409 if (!n)
1410 return NULL;
1411 for (;;) {
1412 struct sp_node *w = NULL;
1413 struct rb_node *prev = rb_prev(n);
1414 if (!prev)
1415 break;
1416 w = rb_entry(prev, struct sp_node, nd);
1417 if (w->end <= start)
1418 break;
1419 n = prev;
1420 }
1421 return rb_entry(n, struct sp_node, nd);
1422}
1423
1424/* Insert a new shared policy into the list. */
1425/* Caller holds sp->lock */
1426static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1427{
1428 struct rb_node **p = &sp->root.rb_node;
1429 struct rb_node *parent = NULL;
1430 struct sp_node *nd;
1431
1432 while (*p) {
1433 parent = *p;
1434 nd = rb_entry(parent, struct sp_node, nd);
1435 if (new->start < nd->start)
1436 p = &(*p)->rb_left;
1437 else if (new->end > nd->end)
1438 p = &(*p)->rb_right;
1439 else
1440 BUG();
1441 }
1442 rb_link_node(&new->nd, parent, p);
1443 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07001444 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 new->policy ? new->policy->policy : 0);
1446}
1447
1448/* Find shared policy intersecting idx */
1449struct mempolicy *
1450mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1451{
1452 struct mempolicy *pol = NULL;
1453 struct sp_node *sn;
1454
1455 if (!sp->root.rb_node)
1456 return NULL;
1457 spin_lock(&sp->lock);
1458 sn = sp_lookup(sp, idx, idx+1);
1459 if (sn) {
1460 mpol_get(sn->policy);
1461 pol = sn->policy;
1462 }
1463 spin_unlock(&sp->lock);
1464 return pol;
1465}
1466
1467static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1468{
Paul Mundt140d5a42007-07-15 23:38:16 -07001469 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470 rb_erase(&n->nd, &sp->root);
1471 mpol_free(n->policy);
1472 kmem_cache_free(sn_cache, n);
1473}
1474
1475struct sp_node *
1476sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol)
1477{
1478 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1479
1480 if (!n)
1481 return NULL;
1482 n->start = start;
1483 n->end = end;
1484 mpol_get(pol);
1485 n->policy = pol;
1486 return n;
1487}
1488
1489/* Replace a policy range. */
1490static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1491 unsigned long end, struct sp_node *new)
1492{
1493 struct sp_node *n, *new2 = NULL;
1494
1495restart:
1496 spin_lock(&sp->lock);
1497 n = sp_lookup(sp, start, end);
1498 /* Take care of old policies in the same range. */
1499 while (n && n->start < end) {
1500 struct rb_node *next = rb_next(&n->nd);
1501 if (n->start >= start) {
1502 if (n->end <= end)
1503 sp_delete(sp, n);
1504 else
1505 n->start = end;
1506 } else {
1507 /* Old policy spanning whole new range. */
1508 if (n->end > end) {
1509 if (!new2) {
1510 spin_unlock(&sp->lock);
1511 new2 = sp_alloc(end, n->end, n->policy);
1512 if (!new2)
1513 return -ENOMEM;
1514 goto restart;
1515 }
1516 n->end = start;
1517 sp_insert(sp, new2);
1518 new2 = NULL;
1519 break;
1520 } else
1521 n->end = start;
1522 }
1523 if (!next)
1524 break;
1525 n = rb_entry(next, struct sp_node, nd);
1526 }
1527 if (new)
1528 sp_insert(sp, new);
1529 spin_unlock(&sp->lock);
1530 if (new2) {
1531 mpol_free(new2->policy);
1532 kmem_cache_free(sn_cache, new2);
1533 }
1534 return 0;
1535}
1536
Robin Holt7339ff82006-01-14 13:20:48 -08001537void mpol_shared_policy_init(struct shared_policy *info, int policy,
1538 nodemask_t *policy_nodes)
1539{
1540 info->root = RB_ROOT;
1541 spin_lock_init(&info->lock);
1542
1543 if (policy != MPOL_DEFAULT) {
1544 struct mempolicy *newpol;
1545
1546 /* Falls back to MPOL_DEFAULT on any error */
1547 newpol = mpol_new(policy, policy_nodes);
1548 if (!IS_ERR(newpol)) {
1549 /* Create pseudo-vma that contains just the policy */
1550 struct vm_area_struct pvma;
1551
1552 memset(&pvma, 0, sizeof(struct vm_area_struct));
1553 /* Policy covers entire file */
1554 pvma.vm_end = TASK_SIZE;
1555 mpol_set_shared_policy(info, &pvma, newpol);
1556 mpol_free(newpol);
1557 }
1558 }
1559}
1560
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561int mpol_set_shared_policy(struct shared_policy *info,
1562 struct vm_area_struct *vma, struct mempolicy *npol)
1563{
1564 int err;
1565 struct sp_node *new = NULL;
1566 unsigned long sz = vma_pages(vma);
1567
Paul Mundt140d5a42007-07-15 23:38:16 -07001568 pr_debug("set_shared_policy %lx sz %lu %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569 vma->vm_pgoff,
1570 sz, npol? npol->policy : -1,
Paul Mundt140d5a42007-07-15 23:38:16 -07001571 npol ? nodes_addr(npol->v.nodes)[0] : -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572
1573 if (npol) {
1574 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1575 if (!new)
1576 return -ENOMEM;
1577 }
1578 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1579 if (err && new)
1580 kmem_cache_free(sn_cache, new);
1581 return err;
1582}
1583
1584/* Free a backing policy store on inode delete. */
1585void mpol_free_shared_policy(struct shared_policy *p)
1586{
1587 struct sp_node *n;
1588 struct rb_node *next;
1589
1590 if (!p->root.rb_node)
1591 return;
1592 spin_lock(&p->lock);
1593 next = rb_first(&p->root);
1594 while (next) {
1595 n = rb_entry(next, struct sp_node, nd);
1596 next = rb_next(&n->nd);
Andi Kleen90c50292005-07-27 11:43:50 -07001597 rb_erase(&n->nd, &p->root);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598 mpol_free(n->policy);
1599 kmem_cache_free(sn_cache, n);
1600 }
1601 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602}
1603
1604/* assumes fs == KERNEL_DS */
1605void __init numa_policy_init(void)
1606{
Paul Mundtb71636e2007-07-15 23:38:15 -07001607 nodemask_t interleave_nodes;
1608 unsigned long largest = 0;
1609 int nid, prefer = 0;
1610
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611 policy_cache = kmem_cache_create("numa_policy",
1612 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09001613 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614
1615 sn_cache = kmem_cache_create("shared_policy_node",
1616 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09001617 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618
Paul Mundtb71636e2007-07-15 23:38:15 -07001619 /*
1620 * Set interleaving policy for system init. Interleaving is only
1621 * enabled across suitably sized nodes (default is >= 16MB), or
1622 * fall back to the largest node if they're all smaller.
1623 */
1624 nodes_clear(interleave_nodes);
1625 for_each_online_node(nid) {
1626 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627
Paul Mundtb71636e2007-07-15 23:38:15 -07001628 /* Preserve the largest node */
1629 if (largest < total_pages) {
1630 largest = total_pages;
1631 prefer = nid;
1632 }
1633
1634 /* Interleave this node? */
1635 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
1636 node_set(nid, interleave_nodes);
1637 }
1638
1639 /* All too small, use the largest */
1640 if (unlikely(nodes_empty(interleave_nodes)))
1641 node_set(prefer, interleave_nodes);
1642
1643 if (do_set_mempolicy(MPOL_INTERLEAVE, &interleave_nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001644 printk("numa_policy_init: interleaving failed\n");
1645}
1646
Christoph Lameter8bccd852005-10-29 18:16:59 -07001647/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648void numa_default_policy(void)
1649{
Christoph Lameter8bccd852005-10-29 18:16:59 -07001650 do_set_mempolicy(MPOL_DEFAULT, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651}
Paul Jackson68860ec2005-10-30 15:02:36 -08001652
1653/* Migrate a policy to a different set of nodes */
Paul Jackson74cb2152006-01-08 01:01:56 -08001654void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
Paul Jackson68860ec2005-10-30 15:02:36 -08001655{
Paul Jackson74cb2152006-01-08 01:01:56 -08001656 nodemask_t *mpolmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001657 nodemask_t tmp;
1658
1659 if (!pol)
1660 return;
Paul Jackson74cb2152006-01-08 01:01:56 -08001661 mpolmask = &pol->cpuset_mems_allowed;
1662 if (nodes_equal(*mpolmask, *newmask))
1663 return;
Paul Jackson68860ec2005-10-30 15:02:36 -08001664
1665 switch (pol->policy) {
1666 case MPOL_DEFAULT:
1667 break;
1668 case MPOL_INTERLEAVE:
Paul Jackson74cb2152006-01-08 01:01:56 -08001669 nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001670 pol->v.nodes = tmp;
Paul Jackson74cb2152006-01-08 01:01:56 -08001671 *mpolmask = *newmask;
1672 current->il_next = node_remap(current->il_next,
1673 *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001674 break;
1675 case MPOL_PREFERRED:
1676 pol->v.preferred_node = node_remap(pol->v.preferred_node,
Paul Jackson74cb2152006-01-08 01:01:56 -08001677 *mpolmask, *newmask);
1678 *mpolmask = *newmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001679 break;
1680 case MPOL_BIND: {
1681 nodemask_t nodes;
1682 struct zone **z;
1683 struct zonelist *zonelist;
1684
1685 nodes_clear(nodes);
1686 for (z = pol->v.zonelist->zones; *z; z++)
Christoph Lameter89fa3022006-09-25 23:31:55 -07001687 node_set(zone_to_nid(*z), nodes);
Paul Jackson74cb2152006-01-08 01:01:56 -08001688 nodes_remap(tmp, nodes, *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001689 nodes = tmp;
1690
1691 zonelist = bind_zonelist(&nodes);
1692
1693 /* If no mem, then zonelist is NULL and we keep old zonelist.
1694 * If that old zonelist has no remaining mems_allowed nodes,
1695 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
1696 */
1697
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -08001698 if (!IS_ERR(zonelist)) {
Paul Jackson68860ec2005-10-30 15:02:36 -08001699 /* Good - got mem - substitute new zonelist */
1700 kfree(pol->v.zonelist);
1701 pol->v.zonelist = zonelist;
1702 }
Paul Jackson74cb2152006-01-08 01:01:56 -08001703 *mpolmask = *newmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001704 break;
1705 }
1706 default:
1707 BUG();
1708 break;
1709 }
1710}
1711
1712/*
Paul Jackson74cb2152006-01-08 01:01:56 -08001713 * Wrapper for mpol_rebind_policy() that just requires task
1714 * pointer, and updates task mempolicy.
Paul Jackson68860ec2005-10-30 15:02:36 -08001715 */
Paul Jackson74cb2152006-01-08 01:01:56 -08001716
1717void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
Paul Jackson68860ec2005-10-30 15:02:36 -08001718{
Paul Jackson74cb2152006-01-08 01:01:56 -08001719 mpol_rebind_policy(tsk->mempolicy, new);
Paul Jackson68860ec2005-10-30 15:02:36 -08001720}
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001721
1722/*
Paul Jackson42253992006-01-08 01:01:59 -08001723 * Rebind each vma in mm to new nodemask.
1724 *
1725 * Call holding a reference to mm. Takes mm->mmap_sem during call.
1726 */
1727
1728void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
1729{
1730 struct vm_area_struct *vma;
1731
1732 down_write(&mm->mmap_sem);
1733 for (vma = mm->mmap; vma; vma = vma->vm_next)
1734 mpol_rebind_policy(vma->vm_policy, new);
1735 up_write(&mm->mmap_sem);
1736}
1737
1738/*
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001739 * Display pages allocated per node and memory policy via /proc.
1740 */
1741
Helge Deller15ad7cd2006-12-06 20:40:36 -08001742static const char * const policy_types[] =
1743 { "default", "prefer", "bind", "interleave" };
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001744
1745/*
1746 * Convert a mempolicy into a string.
1747 * Returns the number of characters in buffer (if positive)
1748 * or an error (negative)
1749 */
1750static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
1751{
1752 char *p = buffer;
1753 int l;
1754 nodemask_t nodes;
1755 int mode = pol ? pol->policy : MPOL_DEFAULT;
1756
1757 switch (mode) {
1758 case MPOL_DEFAULT:
1759 nodes_clear(nodes);
1760 break;
1761
1762 case MPOL_PREFERRED:
1763 nodes_clear(nodes);
1764 node_set(pol->v.preferred_node, nodes);
1765 break;
1766
1767 case MPOL_BIND:
1768 get_zonemask(pol, &nodes);
1769 break;
1770
1771 case MPOL_INTERLEAVE:
1772 nodes = pol->v.nodes;
1773 break;
1774
1775 default:
1776 BUG();
1777 return -EFAULT;
1778 }
1779
1780 l = strlen(policy_types[mode]);
1781 if (buffer + maxlen < p + l + 1)
1782 return -ENOSPC;
1783
1784 strcpy(p, policy_types[mode]);
1785 p += l;
1786
1787 if (!nodes_empty(nodes)) {
1788 if (buffer + maxlen < p + 2)
1789 return -ENOSPC;
1790 *p++ = '=';
1791 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
1792 }
1793 return p - buffer;
1794}
1795
1796struct numa_maps {
1797 unsigned long pages;
1798 unsigned long anon;
Christoph Lameter397874d2006-03-06 15:42:53 -08001799 unsigned long active;
1800 unsigned long writeback;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001801 unsigned long mapcount_max;
Christoph Lameter397874d2006-03-06 15:42:53 -08001802 unsigned long dirty;
1803 unsigned long swapcache;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001804 unsigned long node[MAX_NUMNODES];
1805};
1806
Christoph Lameter397874d2006-03-06 15:42:53 -08001807static void gather_stats(struct page *page, void *private, int pte_dirty)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001808{
1809 struct numa_maps *md = private;
1810 int count = page_mapcount(page);
1811
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001812 md->pages++;
Christoph Lameter397874d2006-03-06 15:42:53 -08001813 if (pte_dirty || PageDirty(page))
1814 md->dirty++;
1815
1816 if (PageSwapCache(page))
1817 md->swapcache++;
1818
1819 if (PageActive(page))
1820 md->active++;
1821
1822 if (PageWriteback(page))
1823 md->writeback++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001824
1825 if (PageAnon(page))
1826 md->anon++;
1827
Christoph Lameter397874d2006-03-06 15:42:53 -08001828 if (count > md->mapcount_max)
1829 md->mapcount_max = count;
1830
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001831 md->node[page_to_nid(page)]++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001832}
1833
Andrew Morton7f709ed2006-03-07 21:55:22 -08001834#ifdef CONFIG_HUGETLB_PAGE
Christoph Lameter397874d2006-03-06 15:42:53 -08001835static void check_huge_range(struct vm_area_struct *vma,
1836 unsigned long start, unsigned long end,
1837 struct numa_maps *md)
1838{
1839 unsigned long addr;
1840 struct page *page;
1841
1842 for (addr = start; addr < end; addr += HPAGE_SIZE) {
1843 pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
1844 pte_t pte;
1845
1846 if (!ptep)
1847 continue;
1848
1849 pte = *ptep;
1850 if (pte_none(pte))
1851 continue;
1852
1853 page = pte_page(pte);
1854 if (!page)
1855 continue;
1856
1857 gather_stats(page, md, pte_dirty(*ptep));
1858 }
1859}
Andrew Morton7f709ed2006-03-07 21:55:22 -08001860#else
1861static inline void check_huge_range(struct vm_area_struct *vma,
1862 unsigned long start, unsigned long end,
1863 struct numa_maps *md)
1864{
1865}
1866#endif
Christoph Lameter397874d2006-03-06 15:42:53 -08001867
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001868int show_numa_map(struct seq_file *m, void *v)
1869{
Eric W. Biederman99f89552006-06-26 00:25:55 -07001870 struct proc_maps_private *priv = m->private;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001871 struct vm_area_struct *vma = v;
1872 struct numa_maps *md;
Christoph Lameter397874d2006-03-06 15:42:53 -08001873 struct file *file = vma->vm_file;
1874 struct mm_struct *mm = vma->vm_mm;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001875 int n;
1876 char buffer[50];
1877
Christoph Lameter397874d2006-03-06 15:42:53 -08001878 if (!mm)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001879 return 0;
1880
1881 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
1882 if (!md)
1883 return 0;
1884
Christoph Lameter397874d2006-03-06 15:42:53 -08001885 mpol_to_str(buffer, sizeof(buffer),
Eric W. Biederman99f89552006-06-26 00:25:55 -07001886 get_vma_policy(priv->task, vma, vma->vm_start));
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001887
Christoph Lameter397874d2006-03-06 15:42:53 -08001888 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001889
Christoph Lameter397874d2006-03-06 15:42:53 -08001890 if (file) {
1891 seq_printf(m, " file=");
Josef Sipeke9536ae2006-12-08 02:37:21 -08001892 seq_path(m, file->f_path.mnt, file->f_path.dentry, "\n\t= ");
Christoph Lameter397874d2006-03-06 15:42:53 -08001893 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1894 seq_printf(m, " heap");
1895 } else if (vma->vm_start <= mm->start_stack &&
1896 vma->vm_end >= mm->start_stack) {
1897 seq_printf(m, " stack");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001898 }
Christoph Lameter397874d2006-03-06 15:42:53 -08001899
1900 if (is_vm_hugetlb_page(vma)) {
1901 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
1902 seq_printf(m, " huge");
1903 } else {
1904 check_pgd_range(vma, vma->vm_start, vma->vm_end,
1905 &node_online_map, MPOL_MF_STATS, md);
1906 }
1907
1908 if (!md->pages)
1909 goto out;
1910
1911 if (md->anon)
1912 seq_printf(m," anon=%lu",md->anon);
1913
1914 if (md->dirty)
1915 seq_printf(m," dirty=%lu",md->dirty);
1916
1917 if (md->pages != md->anon && md->pages != md->dirty)
1918 seq_printf(m, " mapped=%lu", md->pages);
1919
1920 if (md->mapcount_max > 1)
1921 seq_printf(m, " mapmax=%lu", md->mapcount_max);
1922
1923 if (md->swapcache)
1924 seq_printf(m," swapcache=%lu", md->swapcache);
1925
1926 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1927 seq_printf(m," active=%lu", md->active);
1928
1929 if (md->writeback)
1930 seq_printf(m," writeback=%lu", md->writeback);
1931
1932 for_each_online_node(n)
1933 if (md->node[n])
1934 seq_printf(m, " N%d=%lu", n, md->node[n]);
1935out:
1936 seq_putc(m, '\n');
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001937 kfree(md);
1938
1939 if (m->count < m->size)
Eric W. Biederman99f89552006-06-26 00:25:55 -07001940 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001941 return 0;
1942}
1943