blob: 8d246c3b340f5cd635e8e7177d23abbb38efec64 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
66 could replace all the switch()es with a mempolicy_ops structure.
67*/
68
69#include <linux/mempolicy.h>
70#include <linux/mm.h>
71#include <linux/highmem.h>
72#include <linux/hugetlb.h>
73#include <linux/kernel.h>
74#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#include <linux/nodemask.h>
76#include <linux/cpuset.h>
77#include <linux/gfp.h>
78#include <linux/slab.h>
79#include <linux/string.h>
80#include <linux/module.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070081#include <linux/nsproxy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070082#include <linux/interrupt.h>
83#include <linux/init.h>
84#include <linux/compat.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080085#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080086#include <linux/seq_file.h>
87#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080088#include <linux/migrate.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070089#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070090#include <linux/security.h>
Adrian Bunkdbcb0f12007-10-16 01:26:26 -070091#include <linux/syscalls.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080092
Linus Torvalds1da177e2005-04-16 15:20:36 -070093#include <asm/tlbflush.h>
94#include <asm/uaccess.h>
95
Christoph Lameter38e35862006-01-08 01:01:01 -080096/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080097#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -080098#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080099#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800100
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800101static struct kmem_cache *policy_cache;
102static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104/* Highest zone. An specific allocation for a zone below that is not
105 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800106enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
Andi Kleend42c6992005-07-06 19:56:03 +0200108struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109 .refcnt = ATOMIC_INIT(1), /* never free it */
110 .policy = MPOL_DEFAULT,
111};
112
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700113static void mpol_rebind_policy(struct mempolicy *pol,
114 const nodemask_t *newmask);
115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116/* Do sanity checking on a policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700117static int mpol_check_policy(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118{
KOSAKI Motohiro31f1de42008-02-12 13:30:22 +0900119 int was_empty, is_empty;
120
121 if (!nodes)
122 return 0;
123
124 /*
125 * "Contextualize" the in-coming nodemast for cpusets:
126 * Remember whether in-coming nodemask was empty, If not,
127 * restrict the nodes to the allowed nodes in the cpuset.
128 * This is guaranteed to be a subset of nodes with memory.
129 */
130 cpuset_update_task_memory_state();
131 is_empty = was_empty = nodes_empty(*nodes);
132 if (!was_empty) {
133 nodes_and(*nodes, *nodes, cpuset_current_mems_allowed);
134 is_empty = nodes_empty(*nodes); /* after "contextualization" */
135 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
137 switch (mode) {
138 case MPOL_DEFAULT:
KOSAKI Motohiro31f1de42008-02-12 13:30:22 +0900139 /*
140 * require caller to specify an empty nodemask
141 * before "contextualization"
142 */
143 if (!was_empty)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 return -EINVAL;
145 break;
146 case MPOL_BIND:
147 case MPOL_INTERLEAVE:
KOSAKI Motohiro31f1de42008-02-12 13:30:22 +0900148 /*
149 * require at least 1 valid node after "contextualization"
150 */
151 if (is_empty)
152 return -EINVAL;
153 break;
154 case MPOL_PREFERRED:
155 /*
156 * Did caller specify invalid nodes?
157 * Don't silently accept this as "local allocation".
158 */
159 if (!was_empty && is_empty)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 return -EINVAL;
161 break;
162 }
KOSAKI Motohiro31f1de42008-02-12 13:30:22 +0900163 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164}
Andi Kleendd942ae2006-02-17 01:39:16 +0100165
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166/* Generate a custom zonelist for the BIND policy. */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700167static struct zonelist *bind_zonelist(nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168{
169 struct zonelist *zl;
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700170 int num, max, nd;
171 enum zone_type k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
Andi Kleendfcd3c02005-10-29 18:15:48 -0700173 max = 1 + MAX_NR_ZONES * nodes_weight(*nodes);
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800174 max++; /* space for zlcache_ptr (see mmzone.h) */
Andi Kleendd942ae2006-02-17 01:39:16 +0100175 zl = kmalloc(sizeof(struct zone *) * max, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176 if (!zl)
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800177 return ERR_PTR(-ENOMEM);
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800178 zl->zlcache_ptr = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 num = 0;
Andi Kleendd942ae2006-02-17 01:39:16 +0100180 /* First put in the highest zones from all nodes, then all the next
181 lower zones etc. Avoid empty zones because the memory allocator
182 doesn't like them. If you implement node hot removal you
183 have to fix that. */
Mel Gormanb377fd32007-08-22 14:02:05 -0700184 k = MAX_NR_ZONES - 1;
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700185 while (1) {
Andi Kleendd942ae2006-02-17 01:39:16 +0100186 for_each_node_mask(nd, *nodes) {
187 struct zone *z = &NODE_DATA(nd)->node_zones[k];
188 if (z->present_pages > 0)
189 zl->zones[num++] = z;
190 }
Christoph Lameter2f6726e2006-09-25 23:31:18 -0700191 if (k == 0)
192 break;
193 k--;
Andi Kleendd942ae2006-02-17 01:39:16 +0100194 }
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800195 if (num == 0) {
196 kfree(zl);
197 return ERR_PTR(-EINVAL);
198 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 zl->zones[num] = NULL;
200 return zl;
201}
202
203/* Create a new policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700204static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205{
206 struct mempolicy *policy;
207
Paul Mundt140d5a42007-07-15 23:38:16 -0700208 pr_debug("setting mode %d nodes[0] %lx\n",
209 mode, nodes ? nodes_addr(*nodes)[0] : -1);
210
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 if (mode == MPOL_DEFAULT)
212 return NULL;
213 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
214 if (!policy)
215 return ERR_PTR(-ENOMEM);
216 atomic_set(&policy->refcnt, 1);
217 switch (mode) {
218 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700219 policy->v.nodes = *nodes;
Christoph Lameter6eaf8062007-10-16 01:25:30 -0700220 if (nodes_weight(policy->v.nodes) == 0) {
Andi Kleen8f493d72006-01-03 00:07:28 +0100221 kmem_cache_free(policy_cache, policy);
222 return ERR_PTR(-EINVAL);
223 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 break;
225 case MPOL_PREFERRED:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700226 policy->v.preferred_node = first_node(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 if (policy->v.preferred_node >= MAX_NUMNODES)
228 policy->v.preferred_node = -1;
229 break;
230 case MPOL_BIND:
231 policy->v.zonelist = bind_zonelist(nodes);
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800232 if (IS_ERR(policy->v.zonelist)) {
233 void *error_code = policy->v.zonelist;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 kmem_cache_free(policy_cache, policy);
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -0800235 return error_code;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 }
237 break;
238 }
239 policy->policy = mode;
Paul Jackson74cb2152006-01-08 01:01:56 -0800240 policy->cpuset_mems_allowed = cpuset_mems_allowed(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 return policy;
242}
243
Christoph Lameter397874d2006-03-06 15:42:53 -0800244static void gather_stats(struct page *, void *, int pte_dirty);
Christoph Lameterfc301282006-01-18 17:42:29 -0800245static void migrate_page_add(struct page *page, struct list_head *pagelist,
246 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800247
Christoph Lameter38e35862006-01-08 01:01:01 -0800248/* Scan through pages checking if pages follow certain conditions. */
Nick Pigginb5810032005-10-29 18:16:12 -0700249static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800250 unsigned long addr, unsigned long end,
251 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800252 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253{
Hugh Dickins91612e02005-06-21 17:15:07 -0700254 pte_t *orig_pte;
255 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700256 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700257
Hugh Dickins705e87c2005-10-29 18:16:27 -0700258 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700259 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800260 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800261 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700262
263 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800265 page = vm_normal_page(vma, addr, *pte);
266 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800268 /*
269 * The check for PageReserved here is important to avoid
270 * handling zero pages and other pages that may have been
271 * marked special by the system.
272 *
273 * If the PageReserved would not be checked here then f.e.
274 * the location of the zero page could have an influence
275 * on MPOL_MF_STRICT, zero pages would be counted for
276 * the per node stats, and there would be useless attempts
277 * to put zero pages on the migration list.
278 */
Christoph Lameterf4598c82006-01-12 01:05:20 -0800279 if (PageReserved(page))
280 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800281 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800282 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
283 continue;
284
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800285 if (flags & MPOL_MF_STATS)
Christoph Lameter397874d2006-03-06 15:42:53 -0800286 gather_stats(page, private, pte_dirty(*pte));
Nick Piggin053837f2006-01-18 17:42:27 -0800287 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800288 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800289 else
290 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700291 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700292 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700293 return addr != end;
294}
295
Nick Pigginb5810032005-10-29 18:16:12 -0700296static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800297 unsigned long addr, unsigned long end,
298 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800299 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700300{
301 pmd_t *pmd;
302 unsigned long next;
303
304 pmd = pmd_offset(pud, addr);
305 do {
306 next = pmd_addr_end(addr, end);
307 if (pmd_none_or_clear_bad(pmd))
308 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800309 if (check_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800310 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700311 return -EIO;
312 } while (pmd++, addr = next, addr != end);
313 return 0;
314}
315
Nick Pigginb5810032005-10-29 18:16:12 -0700316static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800317 unsigned long addr, unsigned long end,
318 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800319 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700320{
321 pud_t *pud;
322 unsigned long next;
323
324 pud = pud_offset(pgd, addr);
325 do {
326 next = pud_addr_end(addr, end);
327 if (pud_none_or_clear_bad(pud))
328 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800329 if (check_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800330 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700331 return -EIO;
332 } while (pud++, addr = next, addr != end);
333 return 0;
334}
335
Nick Pigginb5810032005-10-29 18:16:12 -0700336static inline int check_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800337 unsigned long addr, unsigned long end,
338 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800339 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700340{
341 pgd_t *pgd;
342 unsigned long next;
343
Nick Pigginb5810032005-10-29 18:16:12 -0700344 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700345 do {
346 next = pgd_addr_end(addr, end);
347 if (pgd_none_or_clear_bad(pgd))
348 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800349 if (check_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800350 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700351 return -EIO;
352 } while (pgd++, addr = next, addr != end);
353 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354}
355
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800356/*
357 * Check if all pages in a range are on a set of nodes.
358 * If pagelist != NULL then isolate pages from the LRU and
359 * put them on the pagelist.
360 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361static struct vm_area_struct *
362check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800363 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364{
365 int err;
366 struct vm_area_struct *first, *vma, *prev;
367
Christoph Lameter90036ee2006-03-16 23:03:59 -0800368 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
Christoph Lameter90036ee2006-03-16 23:03:59 -0800369
Christoph Lameterb20a3502006-03-22 00:09:12 -0800370 err = migrate_prep();
371 if (err)
372 return ERR_PTR(err);
Christoph Lameter90036ee2006-03-16 23:03:59 -0800373 }
Nick Piggin053837f2006-01-18 17:42:27 -0800374
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375 first = find_vma(mm, start);
376 if (!first)
377 return ERR_PTR(-EFAULT);
378 prev = NULL;
379 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800380 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
381 if (!vma->vm_next && vma->vm_end < end)
382 return ERR_PTR(-EFAULT);
383 if (prev && prev->vm_end < vma->vm_start)
384 return ERR_PTR(-EFAULT);
385 }
386 if (!is_vm_hugetlb_page(vma) &&
387 ((flags & MPOL_MF_STRICT) ||
388 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
389 vma_migratable(vma)))) {
Andi Kleen5b952b32005-09-13 01:25:08 -0700390 unsigned long endvma = vma->vm_end;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800391
Andi Kleen5b952b32005-09-13 01:25:08 -0700392 if (endvma > end)
393 endvma = end;
394 if (vma->vm_start > start)
395 start = vma->vm_start;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800396 err = check_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800397 flags, private);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 if (err) {
399 first = ERR_PTR(err);
400 break;
401 }
402 }
403 prev = vma;
404 }
405 return first;
406}
407
408/* Apply policy to a single VMA */
409static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
410{
411 int err = 0;
412 struct mempolicy *old = vma->vm_policy;
413
Paul Mundt140d5a42007-07-15 23:38:16 -0700414 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 vma->vm_start, vma->vm_end, vma->vm_pgoff,
416 vma->vm_ops, vma->vm_file,
417 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
418
419 if (vma->vm_ops && vma->vm_ops->set_policy)
420 err = vma->vm_ops->set_policy(vma, new);
421 if (!err) {
422 mpol_get(new);
423 vma->vm_policy = new;
424 mpol_free(old);
425 }
426 return err;
427}
428
429/* Step 2: apply policy to a range and do splits. */
430static int mbind_range(struct vm_area_struct *vma, unsigned long start,
431 unsigned long end, struct mempolicy *new)
432{
433 struct vm_area_struct *next;
434 int err;
435
436 err = 0;
437 for (; vma && vma->vm_start < end; vma = next) {
438 next = vma->vm_next;
439 if (vma->vm_start < start)
440 err = split_vma(vma->vm_mm, vma, start, 1);
441 if (!err && vma->vm_end > end)
442 err = split_vma(vma->vm_mm, vma, end, 0);
443 if (!err)
444 err = policy_vma(vma, new);
445 if (err)
446 break;
447 }
448 return err;
449}
450
Paul Jacksonc61afb12006-03-24 03:16:08 -0800451/*
452 * Update task->flags PF_MEMPOLICY bit: set iff non-default
453 * mempolicy. Allows more rapid checking of this (combined perhaps
454 * with other PF_* flag bits) on memory allocation hot code paths.
455 *
456 * If called from outside this file, the task 'p' should -only- be
457 * a newly forked child not yet visible on the task list, because
458 * manipulating the task flags of a visible task is not safe.
459 *
460 * The above limitation is why this routine has the funny name
461 * mpol_fix_fork_child_flag().
462 *
463 * It is also safe to call this with a task pointer of current,
464 * which the static wrapper mpol_set_task_struct_flag() does,
465 * for use within this file.
466 */
467
468void mpol_fix_fork_child_flag(struct task_struct *p)
469{
470 if (p->mempolicy)
471 p->flags |= PF_MEMPOLICY;
472 else
473 p->flags &= ~PF_MEMPOLICY;
474}
475
476static void mpol_set_task_struct_flag(void)
477{
478 mpol_fix_fork_child_flag(current);
479}
480
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481/* Set the process memory policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700482static long do_set_mempolicy(int mode, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 struct mempolicy *new;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485
KOSAKI Motohiro31f1de42008-02-12 13:30:22 +0900486 if (mpol_check_policy(mode, nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700488 new = mpol_new(mode, nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 if (IS_ERR(new))
490 return PTR_ERR(new);
491 mpol_free(current->mempolicy);
492 current->mempolicy = new;
Paul Jacksonc61afb12006-03-24 03:16:08 -0800493 mpol_set_task_struct_flag();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 if (new && new->policy == MPOL_INTERLEAVE)
Andi Kleendfcd3c02005-10-29 18:15:48 -0700495 current->il_next = first_node(new->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 return 0;
497}
498
499/* Fill a zone bitmap for a policy */
Andi Kleendfcd3c02005-10-29 18:15:48 -0700500static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501{
502 int i;
503
Andi Kleendfcd3c02005-10-29 18:15:48 -0700504 nodes_clear(*nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 switch (p->policy) {
506 case MPOL_BIND:
507 for (i = 0; p->v.zonelist->zones[i]; i++)
Christoph Lameter89fa3022006-09-25 23:31:55 -0700508 node_set(zone_to_nid(p->v.zonelist->zones[i]),
Christoph Lameter8bccd852005-10-29 18:16:59 -0700509 *nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 break;
511 case MPOL_DEFAULT:
512 break;
513 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700514 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 break;
516 case MPOL_PREFERRED:
Christoph Lameter56bbd652007-10-16 01:25:35 -0700517 /* or use current node instead of memory_map? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 if (p->v.preferred_node < 0)
Christoph Lameter56bbd652007-10-16 01:25:35 -0700519 *nodes = node_states[N_HIGH_MEMORY];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 else
Andi Kleendfcd3c02005-10-29 18:15:48 -0700521 node_set(p->v.preferred_node, *nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 break;
523 default:
524 BUG();
525 }
526}
527
528static int lookup_node(struct mm_struct *mm, unsigned long addr)
529{
530 struct page *p;
531 int err;
532
533 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
534 if (err >= 0) {
535 err = page_to_nid(p);
536 put_page(p);
537 }
538 return err;
539}
540
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541/* Retrieve NUMA policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700542static long do_get_mempolicy(int *policy, nodemask_t *nmask,
543 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700545 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546 struct mm_struct *mm = current->mm;
547 struct vm_area_struct *vma = NULL;
548 struct mempolicy *pol = current->mempolicy;
549
Paul Jacksoncf2a4732006-01-08 01:01:54 -0800550 cpuset_update_task_memory_state();
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700551 if (flags &
552 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 return -EINVAL;
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700554
555 if (flags & MPOL_F_MEMS_ALLOWED) {
556 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
557 return -EINVAL;
558 *policy = 0; /* just so it's initialized */
559 *nmask = cpuset_current_mems_allowed;
560 return 0;
561 }
562
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 if (flags & MPOL_F_ADDR) {
564 down_read(&mm->mmap_sem);
565 vma = find_vma_intersection(mm, addr, addr+1);
566 if (!vma) {
567 up_read(&mm->mmap_sem);
568 return -EFAULT;
569 }
570 if (vma->vm_ops && vma->vm_ops->get_policy)
571 pol = vma->vm_ops->get_policy(vma, addr);
572 else
573 pol = vma->vm_policy;
574 } else if (addr)
575 return -EINVAL;
576
577 if (!pol)
578 pol = &default_policy;
579
580 if (flags & MPOL_F_NODE) {
581 if (flags & MPOL_F_ADDR) {
582 err = lookup_node(mm, addr);
583 if (err < 0)
584 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700585 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 } else if (pol == current->mempolicy &&
587 pol->policy == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700588 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 } else {
590 err = -EINVAL;
591 goto out;
592 }
593 } else
Christoph Lameter8bccd852005-10-29 18:16:59 -0700594 *policy = pol->policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
596 if (vma) {
597 up_read(&current->mm->mmap_sem);
598 vma = NULL;
599 }
600
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 err = 0;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700602 if (nmask)
603 get_zonemask(pol, nmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604
605 out:
606 if (vma)
607 up_read(&current->mm->mmap_sem);
608 return err;
609}
610
Christoph Lameterb20a3502006-03-22 00:09:12 -0800611#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700612/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800613 * page migration
614 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800615static void migrate_page_add(struct page *page, struct list_head *pagelist,
616 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800617{
618 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800619 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800620 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800621 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
622 isolate_lru_page(page, pagelist);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800623}
624
Christoph Lameter742755a2006-06-23 02:03:55 -0700625static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700626{
Mel Gorman769848c2007-07-17 04:03:05 -0700627 return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700628}
629
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800630/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800631 * Migrate pages from one node to a target node.
632 * Returns error or the number of pages not migrated.
633 */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700634static int migrate_to_node(struct mm_struct *mm, int source, int dest,
635 int flags)
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800636{
637 nodemask_t nmask;
638 LIST_HEAD(pagelist);
639 int err = 0;
640
641 nodes_clear(nmask);
642 node_set(source, nmask);
643
644 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
645 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
646
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700647 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700648 err = migrate_pages(&pagelist, new_node_page, dest);
649
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800650 return err;
651}
652
653/*
654 * Move pages between the two nodesets so as to preserve the physical
655 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -0800656 *
657 * Returns the number of page that could not be moved.
658 */
659int do_migrate_pages(struct mm_struct *mm,
660 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
661{
662 LIST_HEAD(pagelist);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800663 int busy = 0;
664 int err = 0;
665 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -0800666
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800667 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -0800668
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700669 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
670 if (err)
671 goto out;
672
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800673/*
674 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
675 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
676 * bit in 'tmp', and return that <source, dest> pair for migration.
677 * The pair of nodemasks 'to' and 'from' define the map.
678 *
679 * If no pair of bits is found that way, fallback to picking some
680 * pair of 'source' and 'dest' bits that are not the same. If the
681 * 'source' and 'dest' bits are the same, this represents a node
682 * that will be migrating to itself, so no pages need move.
683 *
684 * If no bits are left in 'tmp', or if all remaining bits left
685 * in 'tmp' correspond to the same bit in 'to', return false
686 * (nothing left to migrate).
687 *
688 * This lets us pick a pair of nodes to migrate between, such that
689 * if possible the dest node is not already occupied by some other
690 * source node, minimizing the risk of overloading the memory on a
691 * node that would happen if we migrated incoming memory to a node
692 * before migrating outgoing memory source that same node.
693 *
694 * A single scan of tmp is sufficient. As we go, we remember the
695 * most recent <s, d> pair that moved (s != d). If we find a pair
696 * that not only moved, but what's better, moved to an empty slot
697 * (d is not set in tmp), then we break out then, with that pair.
698 * Otherwise when we finish scannng from_tmp, we at least have the
699 * most recent <s, d> pair that moved. If we get all the way through
700 * the scan of tmp without finding any node that moved, much less
701 * moved to an empty node, then there is nothing left worth migrating.
702 */
Christoph Lameterd4984712006-01-08 01:00:55 -0800703
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800704 tmp = *from_nodes;
705 while (!nodes_empty(tmp)) {
706 int s,d;
707 int source = -1;
708 int dest = 0;
709
710 for_each_node_mask(s, tmp) {
711 d = node_remap(s, *from_nodes, *to_nodes);
712 if (s == d)
713 continue;
714
715 source = s; /* Node moved. Memorize */
716 dest = d;
717
718 /* dest not in remaining from nodes? */
719 if (!node_isset(dest, tmp))
720 break;
721 }
722 if (source == -1)
723 break;
724
725 node_clear(source, tmp);
726 err = migrate_to_node(mm, source, dest, flags);
727 if (err > 0)
728 busy += err;
729 if (err < 0)
730 break;
Christoph Lameter39743882006-01-08 01:00:51 -0800731 }
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700732out:
Christoph Lameter39743882006-01-08 01:00:51 -0800733 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800734 if (err < 0)
735 return err;
736 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800737
Christoph Lameter39743882006-01-08 01:00:51 -0800738}
739
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800740/*
741 * Allocate a new page for page migration based on vma policy.
742 * Start assuming that page is mapped by vma pointed to by @private.
743 * Search forward from there, if not. N.B., this assumes that the
744 * list of pages handed to migrate_pages()--which is how we get here--
745 * is in virtual address order.
746 */
Christoph Lameter742755a2006-06-23 02:03:55 -0700747static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700748{
749 struct vm_area_struct *vma = (struct vm_area_struct *)private;
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800750 unsigned long uninitialized_var(address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700751
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800752 while (vma) {
753 address = page_address_in_vma(page, vma);
754 if (address != -EFAULT)
755 break;
756 vma = vma->vm_next;
757 }
758
759 /*
760 * if !vma, alloc_page_vma() will use task or system default policy
761 */
762 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700763}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800764#else
765
766static void migrate_page_add(struct page *page, struct list_head *pagelist,
767 unsigned long flags)
768{
769}
770
771int do_migrate_pages(struct mm_struct *mm,
772 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
773{
774 return -ENOSYS;
775}
Christoph Lameter95a402c2006-06-23 02:03:53 -0700776
Keith Owens69939742006-10-11 01:21:28 -0700777static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700778{
779 return NULL;
780}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800781#endif
782
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700783static long do_mbind(unsigned long start, unsigned long len,
784 unsigned long mode, nodemask_t *nmask,
785 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800786{
787 struct vm_area_struct *vma;
788 struct mm_struct *mm = current->mm;
789 struct mempolicy *new;
790 unsigned long end;
791 int err;
792 LIST_HEAD(pagelist);
793
794 if ((flags & ~(unsigned long)(MPOL_MF_STRICT |
795 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
796 || mode > MPOL_MAX)
797 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -0800798 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800799 return -EPERM;
800
801 if (start & ~PAGE_MASK)
802 return -EINVAL;
803
804 if (mode == MPOL_DEFAULT)
805 flags &= ~MPOL_MF_STRICT;
806
807 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
808 end = start + len;
809
810 if (end < start)
811 return -EINVAL;
812 if (end == start)
813 return 0;
814
815 if (mpol_check_policy(mode, nmask))
816 return -EINVAL;
817
818 new = mpol_new(mode, nmask);
819 if (IS_ERR(new))
820 return PTR_ERR(new);
821
822 /*
823 * If we are using the default policy then operation
824 * on discontinuous address spaces is okay after all
825 */
826 if (!new)
827 flags |= MPOL_MF_DISCONTIG_OK;
828
Paul Mundt140d5a42007-07-15 23:38:16 -0700829 pr_debug("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len,
830 mode, nmask ? nodes_addr(*nmask)[0] : -1);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800831
832 down_write(&mm->mmap_sem);
833 vma = check_range(mm, start, end, nmask,
834 flags | MPOL_MF_INVERT, &pagelist);
835
836 err = PTR_ERR(vma);
837 if (!IS_ERR(vma)) {
838 int nr_failed = 0;
839
840 err = mbind_range(vma, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800841
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800842 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700843 nr_failed = migrate_pages(&pagelist, new_vma_page,
844 (unsigned long)vma);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800845
846 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
847 err = -EIO;
848 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800849
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800850 up_write(&mm->mmap_sem);
851 mpol_free(new);
852 return err;
853}
854
Christoph Lameter39743882006-01-08 01:00:51 -0800855/*
Christoph Lameter8bccd852005-10-29 18:16:59 -0700856 * User space interface with variable sized bitmaps for nodelists.
857 */
858
859/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -0800860static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -0700861 unsigned long maxnode)
862{
863 unsigned long k;
864 unsigned long nlongs;
865 unsigned long endmask;
866
867 --maxnode;
868 nodes_clear(*nodes);
869 if (maxnode == 0 || !nmask)
870 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -0800871 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -0800872 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700873
874 nlongs = BITS_TO_LONGS(maxnode);
875 if ((maxnode % BITS_PER_LONG) == 0)
876 endmask = ~0UL;
877 else
878 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
879
880 /* When the user specified more nodes than supported just check
881 if the non supported part is all zero. */
882 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
883 if (nlongs > PAGE_SIZE/sizeof(long))
884 return -EINVAL;
885 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
886 unsigned long t;
887 if (get_user(t, nmask + k))
888 return -EFAULT;
889 if (k == nlongs - 1) {
890 if (t & endmask)
891 return -EINVAL;
892 } else if (t)
893 return -EINVAL;
894 }
895 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
896 endmask = ~0UL;
897 }
898
899 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
900 return -EFAULT;
901 nodes_addr(*nodes)[nlongs-1] &= endmask;
902 return 0;
903}
904
905/* Copy a kernel node mask to user space */
906static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
907 nodemask_t *nodes)
908{
909 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
910 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
911
912 if (copy > nbytes) {
913 if (copy > PAGE_SIZE)
914 return -EINVAL;
915 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
916 return -EFAULT;
917 copy = nbytes;
918 }
919 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
920}
921
922asmlinkage long sys_mbind(unsigned long start, unsigned long len,
923 unsigned long mode,
924 unsigned long __user *nmask, unsigned long maxnode,
925 unsigned flags)
926{
927 nodemask_t nodes;
928 int err;
929
930 err = get_nodes(&nodes, nmask, maxnode);
931 if (err)
932 return err;
933 return do_mbind(start, len, mode, &nodes, flags);
934}
935
936/* Set the process memory policy */
937asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
938 unsigned long maxnode)
939{
940 int err;
941 nodemask_t nodes;
942
943 if (mode < 0 || mode > MPOL_MAX)
944 return -EINVAL;
945 err = get_nodes(&nodes, nmask, maxnode);
946 if (err)
947 return err;
948 return do_set_mempolicy(mode, &nodes);
949}
950
Christoph Lameter39743882006-01-08 01:00:51 -0800951asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
952 const unsigned long __user *old_nodes,
953 const unsigned long __user *new_nodes)
954{
955 struct mm_struct *mm;
956 struct task_struct *task;
957 nodemask_t old;
958 nodemask_t new;
959 nodemask_t task_nodes;
960 int err;
961
962 err = get_nodes(&old, old_nodes, maxnode);
963 if (err)
964 return err;
965
966 err = get_nodes(&new, new_nodes, maxnode);
967 if (err)
968 return err;
969
970 /* Find the mm_struct */
971 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -0700972 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter39743882006-01-08 01:00:51 -0800973 if (!task) {
974 read_unlock(&tasklist_lock);
975 return -ESRCH;
976 }
977 mm = get_task_mm(task);
978 read_unlock(&tasklist_lock);
979
980 if (!mm)
981 return -EINVAL;
982
983 /*
984 * Check if this process has the right to modify the specified
985 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -0800986 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -0800987 * userid as the target process.
988 */
989 if ((current->euid != task->suid) && (current->euid != task->uid) &&
990 (current->uid != task->suid) && (current->uid != task->uid) &&
Christoph Lameter74c00242006-03-14 19:50:21 -0800991 !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -0800992 err = -EPERM;
993 goto out;
994 }
995
996 task_nodes = cpuset_mems_allowed(task);
997 /* Is the user allowed to access the target nodes? */
Christoph Lameter74c00242006-03-14 19:50:21 -0800998 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -0800999 err = -EPERM;
1000 goto out;
1001 }
1002
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07001003 if (!nodes_subset(new, node_states[N_HIGH_MEMORY])) {
Christoph Lameter3b42d282007-08-31 00:12:08 -07001004 err = -EINVAL;
1005 goto out;
1006 }
1007
David Quigley86c3a762006-06-23 02:04:02 -07001008 err = security_task_movememory(task);
1009 if (err)
1010 goto out;
1011
Christoph Lameter511030b2006-02-28 16:58:57 -08001012 err = do_migrate_pages(mm, &old, &new,
Christoph Lameter74c00242006-03-14 19:50:21 -08001013 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter39743882006-01-08 01:00:51 -08001014out:
1015 mmput(mm);
1016 return err;
1017}
1018
1019
Christoph Lameter8bccd852005-10-29 18:16:59 -07001020/* Retrieve NUMA policy */
1021asmlinkage long sys_get_mempolicy(int __user *policy,
1022 unsigned long __user *nmask,
1023 unsigned long maxnode,
1024 unsigned long addr, unsigned long flags)
1025{
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001026 int err;
1027 int uninitialized_var(pval);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001028 nodemask_t nodes;
1029
1030 if (nmask != NULL && maxnode < MAX_NUMNODES)
1031 return -EINVAL;
1032
1033 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1034
1035 if (err)
1036 return err;
1037
1038 if (policy && put_user(pval, policy))
1039 return -EFAULT;
1040
1041 if (nmask)
1042 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1043
1044 return err;
1045}
1046
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047#ifdef CONFIG_COMPAT
1048
1049asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1050 compat_ulong_t __user *nmask,
1051 compat_ulong_t maxnode,
1052 compat_ulong_t addr, compat_ulong_t flags)
1053{
1054 long err;
1055 unsigned long __user *nm = NULL;
1056 unsigned long nr_bits, alloc_size;
1057 DECLARE_BITMAP(bm, MAX_NUMNODES);
1058
1059 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1060 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1061
1062 if (nmask)
1063 nm = compat_alloc_user_space(alloc_size);
1064
1065 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1066
1067 if (!err && nmask) {
1068 err = copy_from_user(bm, nm, alloc_size);
1069 /* ensure entire bitmap is zeroed */
1070 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1071 err |= compat_put_bitmap(nmask, bm, nr_bits);
1072 }
1073
1074 return err;
1075}
1076
1077asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1078 compat_ulong_t maxnode)
1079{
1080 long err = 0;
1081 unsigned long __user *nm = NULL;
1082 unsigned long nr_bits, alloc_size;
1083 DECLARE_BITMAP(bm, MAX_NUMNODES);
1084
1085 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1086 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1087
1088 if (nmask) {
1089 err = compat_get_bitmap(bm, nmask, nr_bits);
1090 nm = compat_alloc_user_space(alloc_size);
1091 err |= copy_to_user(nm, bm, alloc_size);
1092 }
1093
1094 if (err)
1095 return -EFAULT;
1096
1097 return sys_set_mempolicy(mode, nm, nr_bits+1);
1098}
1099
1100asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1101 compat_ulong_t mode, compat_ulong_t __user *nmask,
1102 compat_ulong_t maxnode, compat_ulong_t flags)
1103{
1104 long err = 0;
1105 unsigned long __user *nm = NULL;
1106 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001107 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
1109 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1110 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1111
1112 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001113 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001115 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116 }
1117
1118 if (err)
1119 return -EFAULT;
1120
1121 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1122}
1123
1124#endif
1125
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001126/*
1127 * get_vma_policy(@task, @vma, @addr)
1128 * @task - task for fallback if vma policy == default
1129 * @vma - virtual memory area whose policy is sought
1130 * @addr - address in @vma for shared policy lookup
1131 *
1132 * Returns effective policy for a VMA at specified address.
1133 * Falls back to @task or system default policy, as necessary.
1134 * Returned policy has extra reference count if shared, vma,
1135 * or some other task's policy [show_numa_maps() can pass
1136 * @task != current]. It is the caller's responsibility to
1137 * free the reference in these cases.
1138 */
Christoph Lameter48fce342006-01-08 01:01:03 -08001139static struct mempolicy * get_vma_policy(struct task_struct *task,
1140 struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001141{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001142 struct mempolicy *pol = task->mempolicy;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001143 int shared_pol = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144
1145 if (vma) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001146 if (vma->vm_ops && vma->vm_ops->get_policy) {
Christoph Lameter8bccd852005-10-29 18:16:59 -07001147 pol = vma->vm_ops->get_policy(vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001148 shared_pol = 1; /* if pol non-NULL, add ref below */
1149 } else if (vma->vm_policy &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150 vma->vm_policy->policy != MPOL_DEFAULT)
1151 pol = vma->vm_policy;
1152 }
1153 if (!pol)
1154 pol = &default_policy;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001155 else if (!shared_pol && pol != current->mempolicy)
1156 mpol_get(pol); /* vma or other task's policy */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157 return pol;
1158}
1159
1160/* Return a zonelist representing a mempolicy */
Al Virodd0fc662005-10-07 07:46:04 +01001161static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162{
1163 int nd;
1164
1165 switch (policy->policy) {
1166 case MPOL_PREFERRED:
1167 nd = policy->v.preferred_node;
1168 if (nd < 0)
1169 nd = numa_node_id();
1170 break;
1171 case MPOL_BIND:
1172 /* Lower zones don't get a policy applied */
1173 /* Careful: current->mems_allowed might have moved */
Christoph Lameter19655d32006-09-25 23:31:19 -07001174 if (gfp_zone(gfp) >= policy_zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist))
1176 return policy->v.zonelist;
1177 /*FALL THROUGH*/
1178 case MPOL_INTERLEAVE: /* should not happen */
1179 case MPOL_DEFAULT:
1180 nd = numa_node_id();
1181 break;
1182 default:
1183 nd = 0;
1184 BUG();
1185 }
Al Viroaf4ca452005-10-21 02:55:38 -04001186 return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187}
1188
1189/* Do dynamic interleaving for a process */
1190static unsigned interleave_nodes(struct mempolicy *policy)
1191{
1192 unsigned nid, next;
1193 struct task_struct *me = current;
1194
1195 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001196 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001198 next = first_node(policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 me->il_next = next;
1200 return nid;
1201}
1202
Christoph Lameterdc85da12006-01-18 17:42:36 -08001203/*
1204 * Depending on the memory policy provide a node from which to allocate the
1205 * next slab entry.
1206 */
1207unsigned slab_node(struct mempolicy *policy)
1208{
Christoph Lameter765c4502006-09-27 01:50:08 -07001209 int pol = policy ? policy->policy : MPOL_DEFAULT;
1210
1211 switch (pol) {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001212 case MPOL_INTERLEAVE:
1213 return interleave_nodes(policy);
1214
1215 case MPOL_BIND:
1216 /*
1217 * Follow bind policy behavior and start allocation at the
1218 * first node.
1219 */
Christoph Lameter89fa3022006-09-25 23:31:55 -07001220 return zone_to_nid(policy->v.zonelist->zones[0]);
Christoph Lameterdc85da12006-01-18 17:42:36 -08001221
1222 case MPOL_PREFERRED:
1223 if (policy->v.preferred_node >= 0)
1224 return policy->v.preferred_node;
1225 /* Fall through */
1226
1227 default:
1228 return numa_node_id();
1229 }
1230}
1231
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232/* Do static interleaving for a VMA with known offset. */
1233static unsigned offset_il_node(struct mempolicy *pol,
1234 struct vm_area_struct *vma, unsigned long off)
1235{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001236 unsigned nnodes = nodes_weight(pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237 unsigned target = (unsigned)off % nnodes;
1238 int c;
1239 int nid = -1;
1240
1241 c = 0;
1242 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001243 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244 c++;
1245 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246 return nid;
1247}
1248
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001249/* Determine a node number for interleave */
1250static inline unsigned interleave_nid(struct mempolicy *pol,
1251 struct vm_area_struct *vma, unsigned long addr, int shift)
1252{
1253 if (vma) {
1254 unsigned long off;
1255
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001256 /*
1257 * for small pages, there is no difference between
1258 * shift and PAGE_SHIFT, so the bit-shift is safe.
1259 * for huge pages, since vm_pgoff is in units of small
1260 * pages, we need to shift off the always 0 bits to get
1261 * a useful offset.
1262 */
1263 BUG_ON(shift < PAGE_SHIFT);
1264 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001265 off += (addr - vma->vm_start) >> shift;
1266 return offset_il_node(pol, vma, off);
1267 } else
1268 return interleave_nodes(pol);
1269}
1270
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001271#ifdef CONFIG_HUGETLBFS
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001272/*
1273 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1274 * @vma = virtual memory area whose policy is sought
1275 * @addr = address in @vma for shared policy lookup and interleave policy
1276 * @gfp_flags = for requested zone
1277 * @mpol = pointer to mempolicy pointer for reference counted 'BIND policy
1278 *
1279 * Returns a zonelist suitable for a huge page allocation.
1280 * If the effective policy is 'BIND, returns pointer to policy's zonelist.
1281 * If it is also a policy for which get_vma_policy() returns an extra
1282 * reference, we must hold that reference until after allocation.
1283 * In that case, return policy via @mpol so hugetlb allocation can drop
1284 * the reference. For non-'BIND referenced policies, we can/do drop the
1285 * reference here, so the caller doesn't need to know about the special case
1286 * for default and current task policy.
1287 */
Mel Gorman396faf02007-07-17 04:03:13 -07001288struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001289 gfp_t gfp_flags, struct mempolicy **mpol)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001290{
1291 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001292 struct zonelist *zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001293
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001294 *mpol = NULL; /* probably no unref needed */
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001295 if (pol->policy == MPOL_INTERLEAVE) {
1296 unsigned nid;
1297
1298 nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001299 __mpol_free(pol); /* finished with pol */
Mel Gorman396faf02007-07-17 04:03:13 -07001300 return NODE_DATA(nid)->node_zonelists + gfp_zone(gfp_flags);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001301 }
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001302
1303 zl = zonelist_policy(GFP_HIGHUSER, pol);
1304 if (unlikely(pol != &default_policy && pol != current->mempolicy)) {
1305 if (pol->policy != MPOL_BIND)
1306 __mpol_free(pol); /* finished with pol */
1307 else
1308 *mpol = pol; /* unref needed after allocation */
1309 }
1310 return zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001311}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001312#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001313
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314/* Allocate a page in interleaved policy.
1315 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001316static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1317 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318{
1319 struct zonelist *zl;
1320 struct page *page;
1321
Al Viroaf4ca452005-10-21 02:55:38 -04001322 zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323 page = __alloc_pages(gfp, order, zl);
Christoph Lameterca889e62006-06-30 01:55:44 -07001324 if (page && page_zone(page) == zl->zones[0])
1325 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326 return page;
1327}
1328
1329/**
1330 * alloc_page_vma - Allocate a page for a VMA.
1331 *
1332 * @gfp:
1333 * %GFP_USER user allocation.
1334 * %GFP_KERNEL kernel allocations,
1335 * %GFP_HIGHMEM highmem/user allocations,
1336 * %GFP_FS allocation should not call back into a file system.
1337 * %GFP_ATOMIC don't sleep.
1338 *
1339 * @vma: Pointer to VMA or NULL if not available.
1340 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1341 *
1342 * This function allocates a page from the kernel page pool and applies
1343 * a NUMA policy associated with the VMA or the current process.
1344 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1345 * mm_struct of the VMA to prevent it from going away. Should be used for
1346 * all allocations for pages that will be mapped into
1347 * user space. Returns NULL when no page can be allocated.
1348 *
1349 * Should be called with the mm_sem of the vma hold.
1350 */
1351struct page *
Al Virodd0fc662005-10-07 07:46:04 +01001352alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001354 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001355 struct zonelist *zl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001356
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001357 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358
1359 if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
1360 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001361
1362 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 return alloc_page_interleave(gfp, 0, nid);
1364 }
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001365 zl = zonelist_policy(gfp, pol);
1366 if (pol != &default_policy && pol != current->mempolicy) {
1367 /*
1368 * slow path: ref counted policy -- shared or vma
1369 */
1370 struct page *page = __alloc_pages(gfp, 0, zl);
1371 __mpol_free(pol);
1372 return page;
1373 }
1374 /*
1375 * fast path: default or task policy
1376 */
1377 return __alloc_pages(gfp, 0, zl);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378}
1379
1380/**
1381 * alloc_pages_current - Allocate pages.
1382 *
1383 * @gfp:
1384 * %GFP_USER user allocation,
1385 * %GFP_KERNEL kernel allocation,
1386 * %GFP_HIGHMEM highmem allocation,
1387 * %GFP_FS don't call back into a file system.
1388 * %GFP_ATOMIC don't sleep.
1389 * @order: Power of two of allocation size in pages. 0 is a single page.
1390 *
1391 * Allocate a page from the kernel page pool. When not in
1392 * interrupt context and apply the current process NUMA policy.
1393 * Returns NULL when no page can be allocated.
1394 *
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001395 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 * 1) it's ok to take cpuset_sem (can WAIT), and
1397 * 2) allocating for current task (not interrupt).
1398 */
Al Virodd0fc662005-10-07 07:46:04 +01001399struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400{
1401 struct mempolicy *pol = current->mempolicy;
1402
1403 if ((gfp & __GFP_WAIT) && !in_interrupt())
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001404 cpuset_update_task_memory_state();
Christoph Lameter9b819d22006-09-25 23:31:40 -07001405 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406 pol = &default_policy;
1407 if (pol->policy == MPOL_INTERLEAVE)
1408 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
1409 return __alloc_pages(gfp, order, zonelist_policy(gfp, pol));
1410}
1411EXPORT_SYMBOL(alloc_pages_current);
1412
Paul Jackson42253992006-01-08 01:01:59 -08001413/*
1414 * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
1415 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1416 * with the mems_allowed returned by cpuset_mems_allowed(). This
1417 * keeps mempolicies cpuset relative after its cpuset moves. See
1418 * further kernel/cpuset.c update_nodemask().
1419 */
Paul Jackson42253992006-01-08 01:01:59 -08001420
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421/* Slow path of a mempolicy copy */
1422struct mempolicy *__mpol_copy(struct mempolicy *old)
1423{
1424 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1425
1426 if (!new)
1427 return ERR_PTR(-ENOMEM);
Paul Jackson42253992006-01-08 01:01:59 -08001428 if (current_cpuset_is_being_rebound()) {
1429 nodemask_t mems = cpuset_mems_allowed(current);
1430 mpol_rebind_policy(old, &mems);
1431 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432 *new = *old;
1433 atomic_set(&new->refcnt, 1);
1434 if (new->policy == MPOL_BIND) {
1435 int sz = ksize(old->v.zonelist);
Christoph Lametere94b1762006-12-06 20:33:17 -08001436 new->v.zonelist = kmemdup(old->v.zonelist, sz, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001437 if (!new->v.zonelist) {
1438 kmem_cache_free(policy_cache, new);
1439 return ERR_PTR(-ENOMEM);
1440 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441 }
1442 return new;
1443}
1444
1445/* Slow path of a mempolicy comparison */
1446int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1447{
1448 if (!a || !b)
1449 return 0;
1450 if (a->policy != b->policy)
1451 return 0;
1452 switch (a->policy) {
1453 case MPOL_DEFAULT:
1454 return 1;
1455 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -07001456 return nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457 case MPOL_PREFERRED:
1458 return a->v.preferred_node == b->v.preferred_node;
1459 case MPOL_BIND: {
1460 int i;
1461 for (i = 0; a->v.zonelist->zones[i]; i++)
1462 if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i])
1463 return 0;
1464 return b->v.zonelist->zones[i] == NULL;
1465 }
1466 default:
1467 BUG();
1468 return 0;
1469 }
1470}
1471
1472/* Slow path of a mpol destructor. */
1473void __mpol_free(struct mempolicy *p)
1474{
1475 if (!atomic_dec_and_test(&p->refcnt))
1476 return;
1477 if (p->policy == MPOL_BIND)
1478 kfree(p->v.zonelist);
1479 p->policy = MPOL_DEFAULT;
1480 kmem_cache_free(policy_cache, p);
1481}
1482
1483/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 * Shared memory backing store policy support.
1485 *
1486 * Remember policies even when nobody has shared memory mapped.
1487 * The policies are kept in Red-Black tree linked from the inode.
1488 * They are protected by the sp->lock spinlock, which should be held
1489 * for any accesses to the tree.
1490 */
1491
1492/* lookup first element intersecting start-end */
1493/* Caller holds sp->lock */
1494static struct sp_node *
1495sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1496{
1497 struct rb_node *n = sp->root.rb_node;
1498
1499 while (n) {
1500 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1501
1502 if (start >= p->end)
1503 n = n->rb_right;
1504 else if (end <= p->start)
1505 n = n->rb_left;
1506 else
1507 break;
1508 }
1509 if (!n)
1510 return NULL;
1511 for (;;) {
1512 struct sp_node *w = NULL;
1513 struct rb_node *prev = rb_prev(n);
1514 if (!prev)
1515 break;
1516 w = rb_entry(prev, struct sp_node, nd);
1517 if (w->end <= start)
1518 break;
1519 n = prev;
1520 }
1521 return rb_entry(n, struct sp_node, nd);
1522}
1523
1524/* Insert a new shared policy into the list. */
1525/* Caller holds sp->lock */
1526static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1527{
1528 struct rb_node **p = &sp->root.rb_node;
1529 struct rb_node *parent = NULL;
1530 struct sp_node *nd;
1531
1532 while (*p) {
1533 parent = *p;
1534 nd = rb_entry(parent, struct sp_node, nd);
1535 if (new->start < nd->start)
1536 p = &(*p)->rb_left;
1537 else if (new->end > nd->end)
1538 p = &(*p)->rb_right;
1539 else
1540 BUG();
1541 }
1542 rb_link_node(&new->nd, parent, p);
1543 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07001544 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545 new->policy ? new->policy->policy : 0);
1546}
1547
1548/* Find shared policy intersecting idx */
1549struct mempolicy *
1550mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1551{
1552 struct mempolicy *pol = NULL;
1553 struct sp_node *sn;
1554
1555 if (!sp->root.rb_node)
1556 return NULL;
1557 spin_lock(&sp->lock);
1558 sn = sp_lookup(sp, idx, idx+1);
1559 if (sn) {
1560 mpol_get(sn->policy);
1561 pol = sn->policy;
1562 }
1563 spin_unlock(&sp->lock);
1564 return pol;
1565}
1566
1567static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1568{
Paul Mundt140d5a42007-07-15 23:38:16 -07001569 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570 rb_erase(&n->nd, &sp->root);
1571 mpol_free(n->policy);
1572 kmem_cache_free(sn_cache, n);
1573}
1574
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001575static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
1576 struct mempolicy *pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577{
1578 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1579
1580 if (!n)
1581 return NULL;
1582 n->start = start;
1583 n->end = end;
1584 mpol_get(pol);
1585 n->policy = pol;
1586 return n;
1587}
1588
1589/* Replace a policy range. */
1590static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1591 unsigned long end, struct sp_node *new)
1592{
1593 struct sp_node *n, *new2 = NULL;
1594
1595restart:
1596 spin_lock(&sp->lock);
1597 n = sp_lookup(sp, start, end);
1598 /* Take care of old policies in the same range. */
1599 while (n && n->start < end) {
1600 struct rb_node *next = rb_next(&n->nd);
1601 if (n->start >= start) {
1602 if (n->end <= end)
1603 sp_delete(sp, n);
1604 else
1605 n->start = end;
1606 } else {
1607 /* Old policy spanning whole new range. */
1608 if (n->end > end) {
1609 if (!new2) {
1610 spin_unlock(&sp->lock);
1611 new2 = sp_alloc(end, n->end, n->policy);
1612 if (!new2)
1613 return -ENOMEM;
1614 goto restart;
1615 }
1616 n->end = start;
1617 sp_insert(sp, new2);
1618 new2 = NULL;
1619 break;
1620 } else
1621 n->end = start;
1622 }
1623 if (!next)
1624 break;
1625 n = rb_entry(next, struct sp_node, nd);
1626 }
1627 if (new)
1628 sp_insert(sp, new);
1629 spin_unlock(&sp->lock);
1630 if (new2) {
1631 mpol_free(new2->policy);
1632 kmem_cache_free(sn_cache, new2);
1633 }
1634 return 0;
1635}
1636
Robin Holt7339ff82006-01-14 13:20:48 -08001637void mpol_shared_policy_init(struct shared_policy *info, int policy,
1638 nodemask_t *policy_nodes)
1639{
1640 info->root = RB_ROOT;
1641 spin_lock_init(&info->lock);
1642
1643 if (policy != MPOL_DEFAULT) {
1644 struct mempolicy *newpol;
1645
1646 /* Falls back to MPOL_DEFAULT on any error */
1647 newpol = mpol_new(policy, policy_nodes);
1648 if (!IS_ERR(newpol)) {
1649 /* Create pseudo-vma that contains just the policy */
1650 struct vm_area_struct pvma;
1651
1652 memset(&pvma, 0, sizeof(struct vm_area_struct));
1653 /* Policy covers entire file */
1654 pvma.vm_end = TASK_SIZE;
1655 mpol_set_shared_policy(info, &pvma, newpol);
1656 mpol_free(newpol);
1657 }
1658 }
1659}
1660
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661int mpol_set_shared_policy(struct shared_policy *info,
1662 struct vm_area_struct *vma, struct mempolicy *npol)
1663{
1664 int err;
1665 struct sp_node *new = NULL;
1666 unsigned long sz = vma_pages(vma);
1667
Paul Mundt140d5a42007-07-15 23:38:16 -07001668 pr_debug("set_shared_policy %lx sz %lu %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 vma->vm_pgoff,
1670 sz, npol? npol->policy : -1,
Paul Mundt140d5a42007-07-15 23:38:16 -07001671 npol ? nodes_addr(npol->v.nodes)[0] : -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672
1673 if (npol) {
1674 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1675 if (!new)
1676 return -ENOMEM;
1677 }
1678 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1679 if (err && new)
1680 kmem_cache_free(sn_cache, new);
1681 return err;
1682}
1683
1684/* Free a backing policy store on inode delete. */
1685void mpol_free_shared_policy(struct shared_policy *p)
1686{
1687 struct sp_node *n;
1688 struct rb_node *next;
1689
1690 if (!p->root.rb_node)
1691 return;
1692 spin_lock(&p->lock);
1693 next = rb_first(&p->root);
1694 while (next) {
1695 n = rb_entry(next, struct sp_node, nd);
1696 next = rb_next(&n->nd);
Andi Kleen90c50292005-07-27 11:43:50 -07001697 rb_erase(&n->nd, &p->root);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698 mpol_free(n->policy);
1699 kmem_cache_free(sn_cache, n);
1700 }
1701 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702}
1703
1704/* assumes fs == KERNEL_DS */
1705void __init numa_policy_init(void)
1706{
Paul Mundtb71636e2007-07-15 23:38:15 -07001707 nodemask_t interleave_nodes;
1708 unsigned long largest = 0;
1709 int nid, prefer = 0;
1710
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711 policy_cache = kmem_cache_create("numa_policy",
1712 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09001713 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714
1715 sn_cache = kmem_cache_create("shared_policy_node",
1716 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09001717 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718
Paul Mundtb71636e2007-07-15 23:38:15 -07001719 /*
1720 * Set interleaving policy for system init. Interleaving is only
1721 * enabled across suitably sized nodes (default is >= 16MB), or
1722 * fall back to the largest node if they're all smaller.
1723 */
1724 nodes_clear(interleave_nodes);
Christoph Lameter56bbd652007-10-16 01:25:35 -07001725 for_each_node_state(nid, N_HIGH_MEMORY) {
Paul Mundtb71636e2007-07-15 23:38:15 -07001726 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727
Paul Mundtb71636e2007-07-15 23:38:15 -07001728 /* Preserve the largest node */
1729 if (largest < total_pages) {
1730 largest = total_pages;
1731 prefer = nid;
1732 }
1733
1734 /* Interleave this node? */
1735 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
1736 node_set(nid, interleave_nodes);
1737 }
1738
1739 /* All too small, use the largest */
1740 if (unlikely(nodes_empty(interleave_nodes)))
1741 node_set(prefer, interleave_nodes);
1742
1743 if (do_set_mempolicy(MPOL_INTERLEAVE, &interleave_nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744 printk("numa_policy_init: interleaving failed\n");
1745}
1746
Christoph Lameter8bccd852005-10-29 18:16:59 -07001747/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748void numa_default_policy(void)
1749{
Christoph Lameter8bccd852005-10-29 18:16:59 -07001750 do_set_mempolicy(MPOL_DEFAULT, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751}
Paul Jackson68860ec2005-10-30 15:02:36 -08001752
1753/* Migrate a policy to a different set of nodes */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001754static void mpol_rebind_policy(struct mempolicy *pol,
1755 const nodemask_t *newmask)
Paul Jackson68860ec2005-10-30 15:02:36 -08001756{
Paul Jackson74cb2152006-01-08 01:01:56 -08001757 nodemask_t *mpolmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001758 nodemask_t tmp;
1759
1760 if (!pol)
1761 return;
Paul Jackson74cb2152006-01-08 01:01:56 -08001762 mpolmask = &pol->cpuset_mems_allowed;
1763 if (nodes_equal(*mpolmask, *newmask))
1764 return;
Paul Jackson68860ec2005-10-30 15:02:36 -08001765
1766 switch (pol->policy) {
1767 case MPOL_DEFAULT:
1768 break;
1769 case MPOL_INTERLEAVE:
Paul Jackson74cb2152006-01-08 01:01:56 -08001770 nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001771 pol->v.nodes = tmp;
Paul Jackson74cb2152006-01-08 01:01:56 -08001772 *mpolmask = *newmask;
1773 current->il_next = node_remap(current->il_next,
1774 *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001775 break;
1776 case MPOL_PREFERRED:
1777 pol->v.preferred_node = node_remap(pol->v.preferred_node,
Paul Jackson74cb2152006-01-08 01:01:56 -08001778 *mpolmask, *newmask);
1779 *mpolmask = *newmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001780 break;
1781 case MPOL_BIND: {
1782 nodemask_t nodes;
1783 struct zone **z;
1784 struct zonelist *zonelist;
1785
1786 nodes_clear(nodes);
1787 for (z = pol->v.zonelist->zones; *z; z++)
Christoph Lameter89fa3022006-09-25 23:31:55 -07001788 node_set(zone_to_nid(*z), nodes);
Paul Jackson74cb2152006-01-08 01:01:56 -08001789 nodes_remap(tmp, nodes, *mpolmask, *newmask);
Paul Jackson68860ec2005-10-30 15:02:36 -08001790 nodes = tmp;
1791
1792 zonelist = bind_zonelist(&nodes);
1793
1794 /* If no mem, then zonelist is NULL and we keep old zonelist.
1795 * If that old zonelist has no remaining mems_allowed nodes,
1796 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
1797 */
1798
KAMEZAWA Hiroyuki8af5e2e2007-02-20 13:57:49 -08001799 if (!IS_ERR(zonelist)) {
Paul Jackson68860ec2005-10-30 15:02:36 -08001800 /* Good - got mem - substitute new zonelist */
1801 kfree(pol->v.zonelist);
1802 pol->v.zonelist = zonelist;
1803 }
Paul Jackson74cb2152006-01-08 01:01:56 -08001804 *mpolmask = *newmask;
Paul Jackson68860ec2005-10-30 15:02:36 -08001805 break;
1806 }
1807 default:
1808 BUG();
1809 break;
1810 }
1811}
1812
1813/*
Paul Jackson74cb2152006-01-08 01:01:56 -08001814 * Wrapper for mpol_rebind_policy() that just requires task
1815 * pointer, and updates task mempolicy.
Paul Jackson68860ec2005-10-30 15:02:36 -08001816 */
Paul Jackson74cb2152006-01-08 01:01:56 -08001817
1818void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
Paul Jackson68860ec2005-10-30 15:02:36 -08001819{
Paul Jackson74cb2152006-01-08 01:01:56 -08001820 mpol_rebind_policy(tsk->mempolicy, new);
Paul Jackson68860ec2005-10-30 15:02:36 -08001821}
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001822
1823/*
Paul Jackson42253992006-01-08 01:01:59 -08001824 * Rebind each vma in mm to new nodemask.
1825 *
1826 * Call holding a reference to mm. Takes mm->mmap_sem during call.
1827 */
1828
1829void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
1830{
1831 struct vm_area_struct *vma;
1832
1833 down_write(&mm->mmap_sem);
1834 for (vma = mm->mmap; vma; vma = vma->vm_next)
1835 mpol_rebind_policy(vma->vm_policy, new);
1836 up_write(&mm->mmap_sem);
1837}
1838
1839/*
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001840 * Display pages allocated per node and memory policy via /proc.
1841 */
1842
Helge Deller15ad7cd2006-12-06 20:40:36 -08001843static const char * const policy_types[] =
1844 { "default", "prefer", "bind", "interleave" };
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001845
1846/*
1847 * Convert a mempolicy into a string.
1848 * Returns the number of characters in buffer (if positive)
1849 * or an error (negative)
1850 */
1851static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
1852{
1853 char *p = buffer;
1854 int l;
1855 nodemask_t nodes;
1856 int mode = pol ? pol->policy : MPOL_DEFAULT;
1857
1858 switch (mode) {
1859 case MPOL_DEFAULT:
1860 nodes_clear(nodes);
1861 break;
1862
1863 case MPOL_PREFERRED:
1864 nodes_clear(nodes);
1865 node_set(pol->v.preferred_node, nodes);
1866 break;
1867
1868 case MPOL_BIND:
1869 get_zonemask(pol, &nodes);
1870 break;
1871
1872 case MPOL_INTERLEAVE:
1873 nodes = pol->v.nodes;
1874 break;
1875
1876 default:
1877 BUG();
1878 return -EFAULT;
1879 }
1880
1881 l = strlen(policy_types[mode]);
1882 if (buffer + maxlen < p + l + 1)
1883 return -ENOSPC;
1884
1885 strcpy(p, policy_types[mode]);
1886 p += l;
1887
1888 if (!nodes_empty(nodes)) {
1889 if (buffer + maxlen < p + 2)
1890 return -ENOSPC;
1891 *p++ = '=';
1892 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
1893 }
1894 return p - buffer;
1895}
1896
1897struct numa_maps {
1898 unsigned long pages;
1899 unsigned long anon;
Christoph Lameter397874d2006-03-06 15:42:53 -08001900 unsigned long active;
1901 unsigned long writeback;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001902 unsigned long mapcount_max;
Christoph Lameter397874d2006-03-06 15:42:53 -08001903 unsigned long dirty;
1904 unsigned long swapcache;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001905 unsigned long node[MAX_NUMNODES];
1906};
1907
Christoph Lameter397874d2006-03-06 15:42:53 -08001908static void gather_stats(struct page *page, void *private, int pte_dirty)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001909{
1910 struct numa_maps *md = private;
1911 int count = page_mapcount(page);
1912
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001913 md->pages++;
Christoph Lameter397874d2006-03-06 15:42:53 -08001914 if (pte_dirty || PageDirty(page))
1915 md->dirty++;
1916
1917 if (PageSwapCache(page))
1918 md->swapcache++;
1919
1920 if (PageActive(page))
1921 md->active++;
1922
1923 if (PageWriteback(page))
1924 md->writeback++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001925
1926 if (PageAnon(page))
1927 md->anon++;
1928
Christoph Lameter397874d2006-03-06 15:42:53 -08001929 if (count > md->mapcount_max)
1930 md->mapcount_max = count;
1931
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001932 md->node[page_to_nid(page)]++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001933}
1934
Andrew Morton7f709ed2006-03-07 21:55:22 -08001935#ifdef CONFIG_HUGETLB_PAGE
Christoph Lameter397874d2006-03-06 15:42:53 -08001936static void check_huge_range(struct vm_area_struct *vma,
1937 unsigned long start, unsigned long end,
1938 struct numa_maps *md)
1939{
1940 unsigned long addr;
1941 struct page *page;
1942
1943 for (addr = start; addr < end; addr += HPAGE_SIZE) {
1944 pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
1945 pte_t pte;
1946
1947 if (!ptep)
1948 continue;
1949
1950 pte = *ptep;
1951 if (pte_none(pte))
1952 continue;
1953
1954 page = pte_page(pte);
1955 if (!page)
1956 continue;
1957
1958 gather_stats(page, md, pte_dirty(*ptep));
1959 }
1960}
Andrew Morton7f709ed2006-03-07 21:55:22 -08001961#else
1962static inline void check_huge_range(struct vm_area_struct *vma,
1963 unsigned long start, unsigned long end,
1964 struct numa_maps *md)
1965{
1966}
1967#endif
Christoph Lameter397874d2006-03-06 15:42:53 -08001968
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001969int show_numa_map(struct seq_file *m, void *v)
1970{
Eric W. Biederman99f89552006-06-26 00:25:55 -07001971 struct proc_maps_private *priv = m->private;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001972 struct vm_area_struct *vma = v;
1973 struct numa_maps *md;
Christoph Lameter397874d2006-03-06 15:42:53 -08001974 struct file *file = vma->vm_file;
1975 struct mm_struct *mm = vma->vm_mm;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001976 struct mempolicy *pol;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001977 int n;
1978 char buffer[50];
1979
Christoph Lameter397874d2006-03-06 15:42:53 -08001980 if (!mm)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001981 return 0;
1982
1983 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
1984 if (!md)
1985 return 0;
1986
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001987 pol = get_vma_policy(priv->task, vma, vma->vm_start);
1988 mpol_to_str(buffer, sizeof(buffer), pol);
1989 /*
1990 * unref shared or other task's mempolicy
1991 */
1992 if (pol != &default_policy && pol != current->mempolicy)
1993 __mpol_free(pol);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001994
Christoph Lameter397874d2006-03-06 15:42:53 -08001995 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001996
Christoph Lameter397874d2006-03-06 15:42:53 -08001997 if (file) {
1998 seq_printf(m, " file=");
Josef Sipeke9536ae2006-12-08 02:37:21 -08001999 seq_path(m, file->f_path.mnt, file->f_path.dentry, "\n\t= ");
Christoph Lameter397874d2006-03-06 15:42:53 -08002000 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
2001 seq_printf(m, " heap");
2002 } else if (vma->vm_start <= mm->start_stack &&
2003 vma->vm_end >= mm->start_stack) {
2004 seq_printf(m, " stack");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002005 }
Christoph Lameter397874d2006-03-06 15:42:53 -08002006
2007 if (is_vm_hugetlb_page(vma)) {
2008 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
2009 seq_printf(m, " huge");
2010 } else {
2011 check_pgd_range(vma, vma->vm_start, vma->vm_end,
Christoph Lameter56bbd652007-10-16 01:25:35 -07002012 &node_states[N_HIGH_MEMORY], MPOL_MF_STATS, md);
Christoph Lameter397874d2006-03-06 15:42:53 -08002013 }
2014
2015 if (!md->pages)
2016 goto out;
2017
2018 if (md->anon)
2019 seq_printf(m," anon=%lu",md->anon);
2020
2021 if (md->dirty)
2022 seq_printf(m," dirty=%lu",md->dirty);
2023
2024 if (md->pages != md->anon && md->pages != md->dirty)
2025 seq_printf(m, " mapped=%lu", md->pages);
2026
2027 if (md->mapcount_max > 1)
2028 seq_printf(m, " mapmax=%lu", md->mapcount_max);
2029
2030 if (md->swapcache)
2031 seq_printf(m," swapcache=%lu", md->swapcache);
2032
2033 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
2034 seq_printf(m," active=%lu", md->active);
2035
2036 if (md->writeback)
2037 seq_printf(m," writeback=%lu", md->writeback);
2038
Christoph Lameter56bbd652007-10-16 01:25:35 -07002039 for_each_node_state(n, N_HIGH_MEMORY)
Christoph Lameter397874d2006-03-06 15:42:53 -08002040 if (md->node[n])
2041 seq_printf(m, " N%d=%lu", n, md->node[n]);
2042out:
2043 seq_putc(m, '\n');
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002044 kfree(md);
2045
2046 if (m->count < m->size)
Eric W. Biederman99f89552006-06-26 00:25:55 -07002047 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002048 return 0;
2049}