blob: a37a5034f63dda824f15b5ffb89156fea43fb4a3 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
Linus Torvalds1da177e2005-04-16 15:20:36 -070066*/
67
68#include <linux/mempolicy.h>
69#include <linux/mm.h>
70#include <linux/highmem.h>
71#include <linux/hugetlb.h>
72#include <linux/kernel.h>
73#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <linux/nodemask.h>
75#include <linux/cpuset.h>
76#include <linux/gfp.h>
77#include <linux/slab.h>
78#include <linux/string.h>
79#include <linux/module.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070080#include <linux/nsproxy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#include <linux/interrupt.h>
82#include <linux/init.h>
83#include <linux/compat.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080084#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080085#include <linux/seq_file.h>
86#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080087#include <linux/migrate.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070088#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070089#include <linux/security.h>
Adrian Bunkdbcb0f12007-10-16 01:26:26 -070090#include <linux/syscalls.h>
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -070091#include <linux/ctype.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080092
Linus Torvalds1da177e2005-04-16 15:20:36 -070093#include <asm/tlbflush.h>
94#include <asm/uaccess.h>
95
Christoph Lameter38e35862006-01-08 01:01:01 -080096/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080097#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -080098#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080099#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800100
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800101static struct kmem_cache *policy_cache;
102static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104/* Highest zone. An specific allocation for a zone below that is not
105 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800106enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700108/*
109 * run-time system-wide default policy => local allocation
110 */
Andi Kleend42c6992005-07-06 19:56:03 +0200111struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 .refcnt = ATOMIC_INIT(1), /* never free it */
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700113 .mode = MPOL_PREFERRED,
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700114 .flags = MPOL_F_LOCAL,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115};
116
David Rientjes37012942008-04-28 02:12:33 -0700117static const struct mempolicy_operations {
118 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
119 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
120} mpol_ops[MPOL_MAX];
121
Mel Gorman19770b32008-04-28 02:12:18 -0700122/* Check that the nodemask contains at least one populated zone */
David Rientjes37012942008-04-28 02:12:33 -0700123static int is_valid_nodemask(const nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124{
Mel Gorman19770b32008-04-28 02:12:18 -0700125 int nd, k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126
Mel Gorman19770b32008-04-28 02:12:18 -0700127 /* Check that there is something useful in this mask */
128 k = policy_zone;
129
130 for_each_node_mask(nd, *nodemask) {
131 struct zone *z;
132
133 for (k = 0; k <= policy_zone; k++) {
134 z = &NODE_DATA(nd)->node_zones[k];
135 if (z->present_pages > 0)
136 return 1;
Andi Kleendd942ae2006-02-17 01:39:16 +0100137 }
138 }
Mel Gorman19770b32008-04-28 02:12:18 -0700139
140 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141}
142
David Rientjesf5b087b2008-04-28 02:12:27 -0700143static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
144{
David Rientjes4c50bc02008-04-28 02:12:30 -0700145 return pol->flags & (MPOL_F_STATIC_NODES | MPOL_F_RELATIVE_NODES);
146}
147
148static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
149 const nodemask_t *rel)
150{
151 nodemask_t tmp;
152 nodes_fold(tmp, *orig, nodes_weight(*rel));
153 nodes_onto(*ret, tmp, *rel);
David Rientjesf5b087b2008-04-28 02:12:27 -0700154}
155
David Rientjes37012942008-04-28 02:12:33 -0700156static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
157{
158 if (nodes_empty(*nodes))
159 return -EINVAL;
160 pol->v.nodes = *nodes;
161 return 0;
162}
163
164static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
165{
166 if (!nodes)
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700167 pol->flags |= MPOL_F_LOCAL; /* local allocation */
David Rientjes37012942008-04-28 02:12:33 -0700168 else if (nodes_empty(*nodes))
169 return -EINVAL; /* no allowed nodes */
170 else
171 pol->v.preferred_node = first_node(*nodes);
172 return 0;
173}
174
175static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
176{
177 if (!is_valid_nodemask(nodes))
178 return -EINVAL;
179 pol->v.nodes = *nodes;
180 return 0;
181}
182
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183/* Create a new policy */
David Rientjes028fec42008-04-28 02:12:25 -0700184static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
185 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186{
187 struct mempolicy *policy;
David Rientjesf5b087b2008-04-28 02:12:27 -0700188 nodemask_t cpuset_context_nmask;
David Rientjes37012942008-04-28 02:12:33 -0700189 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
David Rientjes028fec42008-04-28 02:12:25 -0700191 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
192 mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
Paul Mundt140d5a42007-07-15 23:38:16 -0700193
David Rientjes3e1f0642008-04-28 02:12:34 -0700194 if (mode == MPOL_DEFAULT) {
195 if (nodes && !nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700196 return ERR_PTR(-EINVAL);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700197 return NULL; /* simply delete any existing policy */
David Rientjes37012942008-04-28 02:12:33 -0700198 }
David Rientjes3e1f0642008-04-28 02:12:34 -0700199 VM_BUG_ON(!nodes);
200
201 /*
202 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
203 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
204 * All other modes require a valid pointer to a non-empty nodemask.
205 */
206 if (mode == MPOL_PREFERRED) {
207 if (nodes_empty(*nodes)) {
208 if (((flags & MPOL_F_STATIC_NODES) ||
209 (flags & MPOL_F_RELATIVE_NODES)))
210 return ERR_PTR(-EINVAL);
211 nodes = NULL; /* flag local alloc */
212 }
213 } else if (nodes_empty(*nodes))
214 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
216 if (!policy)
217 return ERR_PTR(-ENOMEM);
218 atomic_set(&policy->refcnt, 1);
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700219 policy->mode = mode;
David Rientjes3e1f0642008-04-28 02:12:34 -0700220 policy->flags = flags;
David Rientjesf5b087b2008-04-28 02:12:27 -0700221
David Rientjes3e1f0642008-04-28 02:12:34 -0700222 if (nodes) {
223 /*
224 * cpuset related setup doesn't apply to local allocation
225 */
David Rientjes37012942008-04-28 02:12:33 -0700226 cpuset_update_task_memory_state();
227 if (flags & MPOL_F_RELATIVE_NODES)
228 mpol_relative_nodemask(&cpuset_context_nmask, nodes,
229 &cpuset_current_mems_allowed);
230 else
231 nodes_and(cpuset_context_nmask, *nodes,
232 cpuset_current_mems_allowed);
233 if (mpol_store_user_nodemask(policy))
234 policy->w.user_nodemask = *nodes;
235 else
236 policy->w.cpuset_mems_allowed =
237 cpuset_mems_allowed(current);
238 }
239
240 ret = mpol_ops[mode].create(policy,
David Rientjes3e1f0642008-04-28 02:12:34 -0700241 nodes ? &cpuset_context_nmask : NULL);
David Rientjes37012942008-04-28 02:12:33 -0700242 if (ret < 0) {
243 kmem_cache_free(policy_cache, policy);
244 return ERR_PTR(ret);
245 }
246 return policy;
247}
248
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700249/* Slow path of a mpol destructor. */
250void __mpol_put(struct mempolicy *p)
251{
252 if (!atomic_dec_and_test(&p->refcnt))
253 return;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700254 kmem_cache_free(policy_cache, p);
255}
256
David Rientjes37012942008-04-28 02:12:33 -0700257static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
258{
259}
260
261static void mpol_rebind_nodemask(struct mempolicy *pol,
262 const nodemask_t *nodes)
263{
264 nodemask_t tmp;
265
266 if (pol->flags & MPOL_F_STATIC_NODES)
267 nodes_and(tmp, pol->w.user_nodemask, *nodes);
268 else if (pol->flags & MPOL_F_RELATIVE_NODES)
269 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
270 else {
271 nodes_remap(tmp, pol->v.nodes, pol->w.cpuset_mems_allowed,
272 *nodes);
273 pol->w.cpuset_mems_allowed = *nodes;
274 }
275
276 pol->v.nodes = tmp;
277 if (!node_isset(current->il_next, tmp)) {
278 current->il_next = next_node(current->il_next, tmp);
279 if (current->il_next >= MAX_NUMNODES)
280 current->il_next = first_node(tmp);
281 if (current->il_next >= MAX_NUMNODES)
282 current->il_next = numa_node_id();
283 }
284}
285
286static void mpol_rebind_preferred(struct mempolicy *pol,
287 const nodemask_t *nodes)
288{
289 nodemask_t tmp;
290
David Rientjes37012942008-04-28 02:12:33 -0700291 if (pol->flags & MPOL_F_STATIC_NODES) {
292 int node = first_node(pol->w.user_nodemask);
293
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700294 if (node_isset(node, *nodes)) {
David Rientjes37012942008-04-28 02:12:33 -0700295 pol->v.preferred_node = node;
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700296 pol->flags &= ~MPOL_F_LOCAL;
297 } else
298 pol->flags |= MPOL_F_LOCAL;
David Rientjes37012942008-04-28 02:12:33 -0700299 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
300 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
301 pol->v.preferred_node = first_node(tmp);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700302 } else if (!(pol->flags & MPOL_F_LOCAL)) {
David Rientjes37012942008-04-28 02:12:33 -0700303 pol->v.preferred_node = node_remap(pol->v.preferred_node,
304 pol->w.cpuset_mems_allowed,
305 *nodes);
306 pol->w.cpuset_mems_allowed = *nodes;
307 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308}
309
David Rientjes1d0d2682008-04-28 02:12:32 -0700310/* Migrate a policy to a different set of nodes */
311static void mpol_rebind_policy(struct mempolicy *pol,
312 const nodemask_t *newmask)
313{
David Rientjes1d0d2682008-04-28 02:12:32 -0700314 if (!pol)
315 return;
David Rientjes1d0d2682008-04-28 02:12:32 -0700316 if (!mpol_store_user_nodemask(pol) &&
317 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
318 return;
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700319 mpol_ops[pol->mode].rebind(pol, newmask);
David Rientjes1d0d2682008-04-28 02:12:32 -0700320}
321
322/*
323 * Wrapper for mpol_rebind_policy() that just requires task
324 * pointer, and updates task mempolicy.
325 */
326
327void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
328{
329 mpol_rebind_policy(tsk->mempolicy, new);
330}
331
332/*
333 * Rebind each vma in mm to new nodemask.
334 *
335 * Call holding a reference to mm. Takes mm->mmap_sem during call.
336 */
337
338void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
339{
340 struct vm_area_struct *vma;
341
342 down_write(&mm->mmap_sem);
343 for (vma = mm->mmap; vma; vma = vma->vm_next)
344 mpol_rebind_policy(vma->vm_policy, new);
345 up_write(&mm->mmap_sem);
346}
347
David Rientjes37012942008-04-28 02:12:33 -0700348static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
349 [MPOL_DEFAULT] = {
350 .rebind = mpol_rebind_default,
351 },
352 [MPOL_INTERLEAVE] = {
353 .create = mpol_new_interleave,
354 .rebind = mpol_rebind_nodemask,
355 },
356 [MPOL_PREFERRED] = {
357 .create = mpol_new_preferred,
358 .rebind = mpol_rebind_preferred,
359 },
360 [MPOL_BIND] = {
361 .create = mpol_new_bind,
362 .rebind = mpol_rebind_nodemask,
363 },
364};
365
Christoph Lameter397874d2006-03-06 15:42:53 -0800366static void gather_stats(struct page *, void *, int pte_dirty);
Christoph Lameterfc301282006-01-18 17:42:29 -0800367static void migrate_page_add(struct page *page, struct list_head *pagelist,
368 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800369
Christoph Lameter38e35862006-01-08 01:01:01 -0800370/* Scan through pages checking if pages follow certain conditions. */
Nick Pigginb5810032005-10-29 18:16:12 -0700371static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800372 unsigned long addr, unsigned long end,
373 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800374 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375{
Hugh Dickins91612e02005-06-21 17:15:07 -0700376 pte_t *orig_pte;
377 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700378 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700379
Hugh Dickins705e87c2005-10-29 18:16:27 -0700380 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700381 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800382 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800383 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700384
385 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800387 page = vm_normal_page(vma, addr, *pte);
388 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800390 /*
391 * The check for PageReserved here is important to avoid
392 * handling zero pages and other pages that may have been
393 * marked special by the system.
394 *
395 * If the PageReserved would not be checked here then f.e.
396 * the location of the zero page could have an influence
397 * on MPOL_MF_STRICT, zero pages would be counted for
398 * the per node stats, and there would be useless attempts
399 * to put zero pages on the migration list.
400 */
Christoph Lameterf4598c82006-01-12 01:05:20 -0800401 if (PageReserved(page))
402 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800403 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800404 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
405 continue;
406
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800407 if (flags & MPOL_MF_STATS)
Christoph Lameter397874d2006-03-06 15:42:53 -0800408 gather_stats(page, private, pte_dirty(*pte));
Nick Piggin053837f2006-01-18 17:42:27 -0800409 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800410 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800411 else
412 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700413 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700414 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700415 return addr != end;
416}
417
Nick Pigginb5810032005-10-29 18:16:12 -0700418static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800419 unsigned long addr, unsigned long end,
420 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800421 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700422{
423 pmd_t *pmd;
424 unsigned long next;
425
426 pmd = pmd_offset(pud, addr);
427 do {
428 next = pmd_addr_end(addr, end);
429 if (pmd_none_or_clear_bad(pmd))
430 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800431 if (check_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800432 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700433 return -EIO;
434 } while (pmd++, addr = next, addr != end);
435 return 0;
436}
437
Nick Pigginb5810032005-10-29 18:16:12 -0700438static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800439 unsigned long addr, unsigned long end,
440 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800441 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700442{
443 pud_t *pud;
444 unsigned long next;
445
446 pud = pud_offset(pgd, addr);
447 do {
448 next = pud_addr_end(addr, end);
449 if (pud_none_or_clear_bad(pud))
450 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800451 if (check_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800452 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700453 return -EIO;
454 } while (pud++, addr = next, addr != end);
455 return 0;
456}
457
Nick Pigginb5810032005-10-29 18:16:12 -0700458static inline int check_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800459 unsigned long addr, unsigned long end,
460 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800461 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700462{
463 pgd_t *pgd;
464 unsigned long next;
465
Nick Pigginb5810032005-10-29 18:16:12 -0700466 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700467 do {
468 next = pgd_addr_end(addr, end);
469 if (pgd_none_or_clear_bad(pgd))
470 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800471 if (check_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800472 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700473 return -EIO;
474 } while (pgd++, addr = next, addr != end);
475 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476}
477
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800478/*
479 * Check if all pages in a range are on a set of nodes.
480 * If pagelist != NULL then isolate pages from the LRU and
481 * put them on the pagelist.
482 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483static struct vm_area_struct *
484check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800485 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486{
487 int err;
488 struct vm_area_struct *first, *vma, *prev;
489
Christoph Lameter90036ee2006-03-16 23:03:59 -0800490 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
Christoph Lameter90036ee2006-03-16 23:03:59 -0800491
Christoph Lameterb20a3502006-03-22 00:09:12 -0800492 err = migrate_prep();
493 if (err)
494 return ERR_PTR(err);
Christoph Lameter90036ee2006-03-16 23:03:59 -0800495 }
Nick Piggin053837f2006-01-18 17:42:27 -0800496
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 first = find_vma(mm, start);
498 if (!first)
499 return ERR_PTR(-EFAULT);
500 prev = NULL;
501 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800502 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
503 if (!vma->vm_next && vma->vm_end < end)
504 return ERR_PTR(-EFAULT);
505 if (prev && prev->vm_end < vma->vm_start)
506 return ERR_PTR(-EFAULT);
507 }
508 if (!is_vm_hugetlb_page(vma) &&
509 ((flags & MPOL_MF_STRICT) ||
510 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
511 vma_migratable(vma)))) {
Andi Kleen5b952b32005-09-13 01:25:08 -0700512 unsigned long endvma = vma->vm_end;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800513
Andi Kleen5b952b32005-09-13 01:25:08 -0700514 if (endvma > end)
515 endvma = end;
516 if (vma->vm_start > start)
517 start = vma->vm_start;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800518 err = check_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800519 flags, private);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 if (err) {
521 first = ERR_PTR(err);
522 break;
523 }
524 }
525 prev = vma;
526 }
527 return first;
528}
529
530/* Apply policy to a single VMA */
531static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
532{
533 int err = 0;
534 struct mempolicy *old = vma->vm_policy;
535
Paul Mundt140d5a42007-07-15 23:38:16 -0700536 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 vma->vm_start, vma->vm_end, vma->vm_pgoff,
538 vma->vm_ops, vma->vm_file,
539 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
540
541 if (vma->vm_ops && vma->vm_ops->set_policy)
542 err = vma->vm_ops->set_policy(vma, new);
543 if (!err) {
544 mpol_get(new);
545 vma->vm_policy = new;
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700546 mpol_put(old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 }
548 return err;
549}
550
551/* Step 2: apply policy to a range and do splits. */
552static int mbind_range(struct vm_area_struct *vma, unsigned long start,
553 unsigned long end, struct mempolicy *new)
554{
555 struct vm_area_struct *next;
556 int err;
557
558 err = 0;
559 for (; vma && vma->vm_start < end; vma = next) {
560 next = vma->vm_next;
561 if (vma->vm_start < start)
562 err = split_vma(vma->vm_mm, vma, start, 1);
563 if (!err && vma->vm_end > end)
564 err = split_vma(vma->vm_mm, vma, end, 0);
565 if (!err)
566 err = policy_vma(vma, new);
567 if (err)
568 break;
569 }
570 return err;
571}
572
Paul Jacksonc61afb12006-03-24 03:16:08 -0800573/*
574 * Update task->flags PF_MEMPOLICY bit: set iff non-default
575 * mempolicy. Allows more rapid checking of this (combined perhaps
576 * with other PF_* flag bits) on memory allocation hot code paths.
577 *
578 * If called from outside this file, the task 'p' should -only- be
579 * a newly forked child not yet visible on the task list, because
580 * manipulating the task flags of a visible task is not safe.
581 *
582 * The above limitation is why this routine has the funny name
583 * mpol_fix_fork_child_flag().
584 *
585 * It is also safe to call this with a task pointer of current,
586 * which the static wrapper mpol_set_task_struct_flag() does,
587 * for use within this file.
588 */
589
590void mpol_fix_fork_child_flag(struct task_struct *p)
591{
592 if (p->mempolicy)
593 p->flags |= PF_MEMPOLICY;
594 else
595 p->flags &= ~PF_MEMPOLICY;
596}
597
598static void mpol_set_task_struct_flag(void)
599{
600 mpol_fix_fork_child_flag(current);
601}
602
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603/* Set the process memory policy */
David Rientjes028fec42008-04-28 02:12:25 -0700604static long do_set_mempolicy(unsigned short mode, unsigned short flags,
605 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 struct mempolicy *new;
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700608 struct mm_struct *mm = current->mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
David Rientjes028fec42008-04-28 02:12:25 -0700610 new = mpol_new(mode, flags, nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 if (IS_ERR(new))
612 return PTR_ERR(new);
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700613
614 /*
615 * prevent changing our mempolicy while show_numa_maps()
616 * is using it.
617 * Note: do_set_mempolicy() can be called at init time
618 * with no 'mm'.
619 */
620 if (mm)
621 down_write(&mm->mmap_sem);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700622 mpol_put(current->mempolicy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 current->mempolicy = new;
Paul Jacksonc61afb12006-03-24 03:16:08 -0800624 mpol_set_task_struct_flag();
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700625 if (new && new->mode == MPOL_INTERLEAVE &&
David Rientjesf5b087b2008-04-28 02:12:27 -0700626 nodes_weight(new->v.nodes))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700627 current->il_next = first_node(new->v.nodes);
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700628 if (mm)
629 up_write(&mm->mmap_sem);
630
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 return 0;
632}
633
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700634/*
635 * Return nodemask for policy for get_mempolicy() query
636 */
637static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700639 nodes_clear(*nodes);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700640 if (p == &default_policy)
641 return;
642
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700643 switch (p->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -0700644 case MPOL_BIND:
645 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700647 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 break;
649 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700650 if (!(p->flags & MPOL_F_LOCAL))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700651 node_set(p->v.preferred_node, *nodes);
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700652 /* else return empty node mask for local allocation */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 break;
654 default:
655 BUG();
656 }
657}
658
659static int lookup_node(struct mm_struct *mm, unsigned long addr)
660{
661 struct page *p;
662 int err;
663
664 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
665 if (err >= 0) {
666 err = page_to_nid(p);
667 put_page(p);
668 }
669 return err;
670}
671
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672/* Retrieve NUMA policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700673static long do_get_mempolicy(int *policy, nodemask_t *nmask,
674 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700676 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 struct mm_struct *mm = current->mm;
678 struct vm_area_struct *vma = NULL;
679 struct mempolicy *pol = current->mempolicy;
680
Paul Jacksoncf2a4732006-01-08 01:01:54 -0800681 cpuset_update_task_memory_state();
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700682 if (flags &
683 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 return -EINVAL;
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700685
686 if (flags & MPOL_F_MEMS_ALLOWED) {
687 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
688 return -EINVAL;
689 *policy = 0; /* just so it's initialized */
690 *nmask = cpuset_current_mems_allowed;
691 return 0;
692 }
693
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 if (flags & MPOL_F_ADDR) {
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700695 /*
696 * Do NOT fall back to task policy if the
697 * vma/shared policy at addr is NULL. We
698 * want to return MPOL_DEFAULT in this case.
699 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 down_read(&mm->mmap_sem);
701 vma = find_vma_intersection(mm, addr, addr+1);
702 if (!vma) {
703 up_read(&mm->mmap_sem);
704 return -EFAULT;
705 }
706 if (vma->vm_ops && vma->vm_ops->get_policy)
707 pol = vma->vm_ops->get_policy(vma, addr);
708 else
709 pol = vma->vm_policy;
710 } else if (addr)
711 return -EINVAL;
712
713 if (!pol)
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700714 pol = &default_policy; /* indicates default behavior */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715
716 if (flags & MPOL_F_NODE) {
717 if (flags & MPOL_F_ADDR) {
718 err = lookup_node(mm, addr);
719 if (err < 0)
720 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700721 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 } else if (pol == current->mempolicy &&
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700723 pol->mode == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700724 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 } else {
726 err = -EINVAL;
727 goto out;
728 }
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700729 } else {
730 *policy = pol == &default_policy ? MPOL_DEFAULT :
731 pol->mode;
732 *policy |= pol->flags;
733 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734
735 if (vma) {
736 up_read(&current->mm->mmap_sem);
737 vma = NULL;
738 }
739
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 err = 0;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700741 if (nmask)
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700742 get_policy_nodemask(pol, nmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743
744 out:
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700745 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 if (vma)
747 up_read(&current->mm->mmap_sem);
748 return err;
749}
750
Christoph Lameterb20a3502006-03-22 00:09:12 -0800751#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700752/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800753 * page migration
754 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800755static void migrate_page_add(struct page *page, struct list_head *pagelist,
756 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800757{
758 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800759 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800760 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800761 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
762 isolate_lru_page(page, pagelist);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800763}
764
Christoph Lameter742755a2006-06-23 02:03:55 -0700765static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700766{
Mel Gorman769848c2007-07-17 04:03:05 -0700767 return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700768}
769
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800770/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800771 * Migrate pages from one node to a target node.
772 * Returns error or the number of pages not migrated.
773 */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700774static int migrate_to_node(struct mm_struct *mm, int source, int dest,
775 int flags)
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800776{
777 nodemask_t nmask;
778 LIST_HEAD(pagelist);
779 int err = 0;
780
781 nodes_clear(nmask);
782 node_set(source, nmask);
783
784 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
785 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
786
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700787 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700788 err = migrate_pages(&pagelist, new_node_page, dest);
789
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800790 return err;
791}
792
793/*
794 * Move pages between the two nodesets so as to preserve the physical
795 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -0800796 *
797 * Returns the number of page that could not be moved.
798 */
799int do_migrate_pages(struct mm_struct *mm,
800 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
801{
802 LIST_HEAD(pagelist);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800803 int busy = 0;
804 int err = 0;
805 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -0800806
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700807 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -0800808
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700809 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
810 if (err)
811 goto out;
812
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800813/*
814 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
815 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
816 * bit in 'tmp', and return that <source, dest> pair for migration.
817 * The pair of nodemasks 'to' and 'from' define the map.
818 *
819 * If no pair of bits is found that way, fallback to picking some
820 * pair of 'source' and 'dest' bits that are not the same. If the
821 * 'source' and 'dest' bits are the same, this represents a node
822 * that will be migrating to itself, so no pages need move.
823 *
824 * If no bits are left in 'tmp', or if all remaining bits left
825 * in 'tmp' correspond to the same bit in 'to', return false
826 * (nothing left to migrate).
827 *
828 * This lets us pick a pair of nodes to migrate between, such that
829 * if possible the dest node is not already occupied by some other
830 * source node, minimizing the risk of overloading the memory on a
831 * node that would happen if we migrated incoming memory to a node
832 * before migrating outgoing memory source that same node.
833 *
834 * A single scan of tmp is sufficient. As we go, we remember the
835 * most recent <s, d> pair that moved (s != d). If we find a pair
836 * that not only moved, but what's better, moved to an empty slot
837 * (d is not set in tmp), then we break out then, with that pair.
838 * Otherwise when we finish scannng from_tmp, we at least have the
839 * most recent <s, d> pair that moved. If we get all the way through
840 * the scan of tmp without finding any node that moved, much less
841 * moved to an empty node, then there is nothing left worth migrating.
842 */
Christoph Lameterd4984712006-01-08 01:00:55 -0800843
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800844 tmp = *from_nodes;
845 while (!nodes_empty(tmp)) {
846 int s,d;
847 int source = -1;
848 int dest = 0;
849
850 for_each_node_mask(s, tmp) {
851 d = node_remap(s, *from_nodes, *to_nodes);
852 if (s == d)
853 continue;
854
855 source = s; /* Node moved. Memorize */
856 dest = d;
857
858 /* dest not in remaining from nodes? */
859 if (!node_isset(dest, tmp))
860 break;
861 }
862 if (source == -1)
863 break;
864
865 node_clear(source, tmp);
866 err = migrate_to_node(mm, source, dest, flags);
867 if (err > 0)
868 busy += err;
869 if (err < 0)
870 break;
Christoph Lameter39743882006-01-08 01:00:51 -0800871 }
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700872out:
Christoph Lameter39743882006-01-08 01:00:51 -0800873 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800874 if (err < 0)
875 return err;
876 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800877
Christoph Lameter39743882006-01-08 01:00:51 -0800878}
879
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800880/*
881 * Allocate a new page for page migration based on vma policy.
882 * Start assuming that page is mapped by vma pointed to by @private.
883 * Search forward from there, if not. N.B., this assumes that the
884 * list of pages handed to migrate_pages()--which is how we get here--
885 * is in virtual address order.
886 */
Christoph Lameter742755a2006-06-23 02:03:55 -0700887static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700888{
889 struct vm_area_struct *vma = (struct vm_area_struct *)private;
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800890 unsigned long uninitialized_var(address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700891
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800892 while (vma) {
893 address = page_address_in_vma(page, vma);
894 if (address != -EFAULT)
895 break;
896 vma = vma->vm_next;
897 }
898
899 /*
900 * if !vma, alloc_page_vma() will use task or system default policy
901 */
902 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700903}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800904#else
905
906static void migrate_page_add(struct page *page, struct list_head *pagelist,
907 unsigned long flags)
908{
909}
910
911int do_migrate_pages(struct mm_struct *mm,
912 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
913{
914 return -ENOSYS;
915}
Christoph Lameter95a402c2006-06-23 02:03:53 -0700916
Keith Owens69939742006-10-11 01:21:28 -0700917static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700918{
919 return NULL;
920}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800921#endif
922
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700923static long do_mbind(unsigned long start, unsigned long len,
David Rientjes028fec42008-04-28 02:12:25 -0700924 unsigned short mode, unsigned short mode_flags,
925 nodemask_t *nmask, unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800926{
927 struct vm_area_struct *vma;
928 struct mm_struct *mm = current->mm;
929 struct mempolicy *new;
930 unsigned long end;
931 int err;
932 LIST_HEAD(pagelist);
933
David Rientjesa3b51e02008-04-28 02:12:23 -0700934 if (flags & ~(unsigned long)(MPOL_MF_STRICT |
935 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800936 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -0800937 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800938 return -EPERM;
939
940 if (start & ~PAGE_MASK)
941 return -EINVAL;
942
943 if (mode == MPOL_DEFAULT)
944 flags &= ~MPOL_MF_STRICT;
945
946 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
947 end = start + len;
948
949 if (end < start)
950 return -EINVAL;
951 if (end == start)
952 return 0;
953
David Rientjes028fec42008-04-28 02:12:25 -0700954 new = mpol_new(mode, mode_flags, nmask);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800955 if (IS_ERR(new))
956 return PTR_ERR(new);
957
958 /*
959 * If we are using the default policy then operation
960 * on discontinuous address spaces is okay after all
961 */
962 if (!new)
963 flags |= MPOL_MF_DISCONTIG_OK;
964
David Rientjes028fec42008-04-28 02:12:25 -0700965 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
966 start, start + len, mode, mode_flags,
967 nmask ? nodes_addr(*nmask)[0] : -1);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800968
969 down_write(&mm->mmap_sem);
970 vma = check_range(mm, start, end, nmask,
971 flags | MPOL_MF_INVERT, &pagelist);
972
973 err = PTR_ERR(vma);
974 if (!IS_ERR(vma)) {
975 int nr_failed = 0;
976
977 err = mbind_range(vma, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800978
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800979 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700980 nr_failed = migrate_pages(&pagelist, new_vma_page,
981 (unsigned long)vma);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800982
983 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
984 err = -EIO;
985 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800986
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800987 up_write(&mm->mmap_sem);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700988 mpol_put(new);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800989 return err;
990}
991
Christoph Lameter39743882006-01-08 01:00:51 -0800992/*
Christoph Lameter8bccd852005-10-29 18:16:59 -0700993 * User space interface with variable sized bitmaps for nodelists.
994 */
995
996/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -0800997static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -0700998 unsigned long maxnode)
999{
1000 unsigned long k;
1001 unsigned long nlongs;
1002 unsigned long endmask;
1003
1004 --maxnode;
1005 nodes_clear(*nodes);
1006 if (maxnode == 0 || !nmask)
1007 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -08001008 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -08001009 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001010
1011 nlongs = BITS_TO_LONGS(maxnode);
1012 if ((maxnode % BITS_PER_LONG) == 0)
1013 endmask = ~0UL;
1014 else
1015 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1016
1017 /* When the user specified more nodes than supported just check
1018 if the non supported part is all zero. */
1019 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1020 if (nlongs > PAGE_SIZE/sizeof(long))
1021 return -EINVAL;
1022 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1023 unsigned long t;
1024 if (get_user(t, nmask + k))
1025 return -EFAULT;
1026 if (k == nlongs - 1) {
1027 if (t & endmask)
1028 return -EINVAL;
1029 } else if (t)
1030 return -EINVAL;
1031 }
1032 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1033 endmask = ~0UL;
1034 }
1035
1036 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1037 return -EFAULT;
1038 nodes_addr(*nodes)[nlongs-1] &= endmask;
1039 return 0;
1040}
1041
1042/* Copy a kernel node mask to user space */
1043static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1044 nodemask_t *nodes)
1045{
1046 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1047 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1048
1049 if (copy > nbytes) {
1050 if (copy > PAGE_SIZE)
1051 return -EINVAL;
1052 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1053 return -EFAULT;
1054 copy = nbytes;
1055 }
1056 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1057}
1058
1059asmlinkage long sys_mbind(unsigned long start, unsigned long len,
1060 unsigned long mode,
1061 unsigned long __user *nmask, unsigned long maxnode,
1062 unsigned flags)
1063{
1064 nodemask_t nodes;
1065 int err;
David Rientjes028fec42008-04-28 02:12:25 -07001066 unsigned short mode_flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001067
David Rientjes028fec42008-04-28 02:12:25 -07001068 mode_flags = mode & MPOL_MODE_FLAGS;
1069 mode &= ~MPOL_MODE_FLAGS;
David Rientjesa3b51e02008-04-28 02:12:23 -07001070 if (mode >= MPOL_MAX)
1071 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001072 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1073 (mode_flags & MPOL_F_RELATIVE_NODES))
1074 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001075 err = get_nodes(&nodes, nmask, maxnode);
1076 if (err)
1077 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001078 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001079}
1080
1081/* Set the process memory policy */
1082asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
1083 unsigned long maxnode)
1084{
1085 int err;
1086 nodemask_t nodes;
David Rientjes028fec42008-04-28 02:12:25 -07001087 unsigned short flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001088
David Rientjes028fec42008-04-28 02:12:25 -07001089 flags = mode & MPOL_MODE_FLAGS;
1090 mode &= ~MPOL_MODE_FLAGS;
1091 if ((unsigned int)mode >= MPOL_MAX)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001092 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001093 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1094 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001095 err = get_nodes(&nodes, nmask, maxnode);
1096 if (err)
1097 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001098 return do_set_mempolicy(mode, flags, &nodes);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001099}
1100
Christoph Lameter39743882006-01-08 01:00:51 -08001101asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
1102 const unsigned long __user *old_nodes,
1103 const unsigned long __user *new_nodes)
1104{
1105 struct mm_struct *mm;
1106 struct task_struct *task;
1107 nodemask_t old;
1108 nodemask_t new;
1109 nodemask_t task_nodes;
1110 int err;
1111
1112 err = get_nodes(&old, old_nodes, maxnode);
1113 if (err)
1114 return err;
1115
1116 err = get_nodes(&new, new_nodes, maxnode);
1117 if (err)
1118 return err;
1119
1120 /* Find the mm_struct */
1121 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001122 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter39743882006-01-08 01:00:51 -08001123 if (!task) {
1124 read_unlock(&tasklist_lock);
1125 return -ESRCH;
1126 }
1127 mm = get_task_mm(task);
1128 read_unlock(&tasklist_lock);
1129
1130 if (!mm)
1131 return -EINVAL;
1132
1133 /*
1134 * Check if this process has the right to modify the specified
1135 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -08001136 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -08001137 * userid as the target process.
1138 */
1139 if ((current->euid != task->suid) && (current->euid != task->uid) &&
1140 (current->uid != task->suid) && (current->uid != task->uid) &&
Christoph Lameter74c00242006-03-14 19:50:21 -08001141 !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001142 err = -EPERM;
1143 goto out;
1144 }
1145
1146 task_nodes = cpuset_mems_allowed(task);
1147 /* Is the user allowed to access the target nodes? */
Christoph Lameter74c00242006-03-14 19:50:21 -08001148 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001149 err = -EPERM;
1150 goto out;
1151 }
1152
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07001153 if (!nodes_subset(new, node_states[N_HIGH_MEMORY])) {
Christoph Lameter3b42d282007-08-31 00:12:08 -07001154 err = -EINVAL;
1155 goto out;
1156 }
1157
David Quigley86c3a762006-06-23 02:04:02 -07001158 err = security_task_movememory(task);
1159 if (err)
1160 goto out;
1161
Christoph Lameter511030b2006-02-28 16:58:57 -08001162 err = do_migrate_pages(mm, &old, &new,
Christoph Lameter74c00242006-03-14 19:50:21 -08001163 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter39743882006-01-08 01:00:51 -08001164out:
1165 mmput(mm);
1166 return err;
1167}
1168
1169
Christoph Lameter8bccd852005-10-29 18:16:59 -07001170/* Retrieve NUMA policy */
1171asmlinkage long sys_get_mempolicy(int __user *policy,
1172 unsigned long __user *nmask,
1173 unsigned long maxnode,
1174 unsigned long addr, unsigned long flags)
1175{
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001176 int err;
1177 int uninitialized_var(pval);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001178 nodemask_t nodes;
1179
1180 if (nmask != NULL && maxnode < MAX_NUMNODES)
1181 return -EINVAL;
1182
1183 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1184
1185 if (err)
1186 return err;
1187
1188 if (policy && put_user(pval, policy))
1189 return -EFAULT;
1190
1191 if (nmask)
1192 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1193
1194 return err;
1195}
1196
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197#ifdef CONFIG_COMPAT
1198
1199asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1200 compat_ulong_t __user *nmask,
1201 compat_ulong_t maxnode,
1202 compat_ulong_t addr, compat_ulong_t flags)
1203{
1204 long err;
1205 unsigned long __user *nm = NULL;
1206 unsigned long nr_bits, alloc_size;
1207 DECLARE_BITMAP(bm, MAX_NUMNODES);
1208
1209 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1210 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1211
1212 if (nmask)
1213 nm = compat_alloc_user_space(alloc_size);
1214
1215 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1216
1217 if (!err && nmask) {
1218 err = copy_from_user(bm, nm, alloc_size);
1219 /* ensure entire bitmap is zeroed */
1220 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1221 err |= compat_put_bitmap(nmask, bm, nr_bits);
1222 }
1223
1224 return err;
1225}
1226
1227asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1228 compat_ulong_t maxnode)
1229{
1230 long err = 0;
1231 unsigned long __user *nm = NULL;
1232 unsigned long nr_bits, alloc_size;
1233 DECLARE_BITMAP(bm, MAX_NUMNODES);
1234
1235 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1236 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1237
1238 if (nmask) {
1239 err = compat_get_bitmap(bm, nmask, nr_bits);
1240 nm = compat_alloc_user_space(alloc_size);
1241 err |= copy_to_user(nm, bm, alloc_size);
1242 }
1243
1244 if (err)
1245 return -EFAULT;
1246
1247 return sys_set_mempolicy(mode, nm, nr_bits+1);
1248}
1249
1250asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1251 compat_ulong_t mode, compat_ulong_t __user *nmask,
1252 compat_ulong_t maxnode, compat_ulong_t flags)
1253{
1254 long err = 0;
1255 unsigned long __user *nm = NULL;
1256 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001257 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258
1259 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1260 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1261
1262 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001263 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001265 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266 }
1267
1268 if (err)
1269 return -EFAULT;
1270
1271 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1272}
1273
1274#endif
1275
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001276/*
1277 * get_vma_policy(@task, @vma, @addr)
1278 * @task - task for fallback if vma policy == default
1279 * @vma - virtual memory area whose policy is sought
1280 * @addr - address in @vma for shared policy lookup
1281 *
1282 * Returns effective policy for a VMA at specified address.
1283 * Falls back to @task or system default policy, as necessary.
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001284 * Current or other task's task mempolicy and non-shared vma policies
1285 * are protected by the task's mmap_sem, which must be held for read by
1286 * the caller.
1287 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1288 * count--added by the get_policy() vm_op, as appropriate--to protect against
1289 * freeing by another task. It is the caller's responsibility to free the
1290 * extra reference for shared policies.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001291 */
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001292static struct mempolicy *get_vma_policy(struct task_struct *task,
Christoph Lameter48fce342006-01-08 01:01:03 -08001293 struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001295 struct mempolicy *pol = task->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296
1297 if (vma) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001298 if (vma->vm_ops && vma->vm_ops->get_policy) {
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001299 struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
1300 addr);
1301 if (vpol)
1302 pol = vpol;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001303 } else if (vma->vm_policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304 pol = vma->vm_policy;
1305 }
1306 if (!pol)
1307 pol = &default_policy;
1308 return pol;
1309}
1310
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001311/*
1312 * Return a nodemask representing a mempolicy for filtering nodes for
1313 * page allocation
1314 */
1315static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
Mel Gorman19770b32008-04-28 02:12:18 -07001316{
1317 /* Lower zones don't get a nodemask applied for MPOL_BIND */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001318 if (unlikely(policy->mode == MPOL_BIND) &&
Mel Gorman19770b32008-04-28 02:12:18 -07001319 gfp_zone(gfp) >= policy_zone &&
1320 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1321 return &policy->v.nodes;
1322
1323 return NULL;
1324}
1325
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001326/* Return a zonelist indicated by gfp for node representing a mempolicy */
1327static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328{
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001329 int nd = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001331 switch (policy->mode) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001332 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001333 if (!(policy->flags & MPOL_F_LOCAL))
1334 nd = policy->v.preferred_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 break;
1336 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07001337 /*
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001338 * Normally, MPOL_BIND allocations are node-local within the
1339 * allowed nodemask. However, if __GFP_THISNODE is set and the
1340 * current node is part of the mask, we use the zonelist for
1341 * the first node in the mask instead.
Mel Gorman19770b32008-04-28 02:12:18 -07001342 */
Mel Gorman19770b32008-04-28 02:12:18 -07001343 if (unlikely(gfp & __GFP_THISNODE) &&
1344 unlikely(!node_isset(nd, policy->v.nodes)))
1345 nd = first_node(policy->v.nodes);
1346 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 case MPOL_INTERLEAVE: /* should not happen */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 break;
1349 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 BUG();
1351 }
Mel Gorman0e884602008-04-28 02:12:14 -07001352 return node_zonelist(nd, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353}
1354
1355/* Do dynamic interleaving for a process */
1356static unsigned interleave_nodes(struct mempolicy *policy)
1357{
1358 unsigned nid, next;
1359 struct task_struct *me = current;
1360
1361 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001362 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001364 next = first_node(policy->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001365 if (next < MAX_NUMNODES)
1366 me->il_next = next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 return nid;
1368}
1369
Christoph Lameterdc85da12006-01-18 17:42:36 -08001370/*
1371 * Depending on the memory policy provide a node from which to allocate the
1372 * next slab entry.
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001373 * @policy must be protected by freeing by the caller. If @policy is
1374 * the current task's mempolicy, this protection is implicit, as only the
1375 * task can change it's policy. The system default policy requires no
1376 * such protection.
Christoph Lameterdc85da12006-01-18 17:42:36 -08001377 */
1378unsigned slab_node(struct mempolicy *policy)
1379{
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001380 if (!policy || policy->flags & MPOL_F_LOCAL)
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001381 return numa_node_id();
Christoph Lameter765c4502006-09-27 01:50:08 -07001382
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001383 switch (policy->mode) {
1384 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001385 /*
1386 * handled MPOL_F_LOCAL above
1387 */
1388 return policy->v.preferred_node;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001389
Christoph Lameterdc85da12006-01-18 17:42:36 -08001390 case MPOL_INTERLEAVE:
1391 return interleave_nodes(policy);
1392
Mel Gormandd1a2392008-04-28 02:12:17 -07001393 case MPOL_BIND: {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001394 /*
1395 * Follow bind policy behavior and start allocation at the
1396 * first node.
1397 */
Mel Gorman19770b32008-04-28 02:12:18 -07001398 struct zonelist *zonelist;
1399 struct zone *zone;
1400 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1401 zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0];
1402 (void)first_zones_zonelist(zonelist, highest_zoneidx,
1403 &policy->v.nodes,
1404 &zone);
1405 return zone->node;
Mel Gormandd1a2392008-04-28 02:12:17 -07001406 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08001407
Christoph Lameterdc85da12006-01-18 17:42:36 -08001408 default:
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001409 BUG();
Christoph Lameterdc85da12006-01-18 17:42:36 -08001410 }
1411}
1412
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413/* Do static interleaving for a VMA with known offset. */
1414static unsigned offset_il_node(struct mempolicy *pol,
1415 struct vm_area_struct *vma, unsigned long off)
1416{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001417 unsigned nnodes = nodes_weight(pol->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001418 unsigned target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419 int c;
1420 int nid = -1;
1421
David Rientjesf5b087b2008-04-28 02:12:27 -07001422 if (!nnodes)
1423 return numa_node_id();
1424 target = (unsigned int)off % nnodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 c = 0;
1426 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001427 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428 c++;
1429 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430 return nid;
1431}
1432
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001433/* Determine a node number for interleave */
1434static inline unsigned interleave_nid(struct mempolicy *pol,
1435 struct vm_area_struct *vma, unsigned long addr, int shift)
1436{
1437 if (vma) {
1438 unsigned long off;
1439
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001440 /*
1441 * for small pages, there is no difference between
1442 * shift and PAGE_SHIFT, so the bit-shift is safe.
1443 * for huge pages, since vm_pgoff is in units of small
1444 * pages, we need to shift off the always 0 bits to get
1445 * a useful offset.
1446 */
1447 BUG_ON(shift < PAGE_SHIFT);
1448 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001449 off += (addr - vma->vm_start) >> shift;
1450 return offset_il_node(pol, vma, off);
1451 } else
1452 return interleave_nodes(pol);
1453}
1454
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001455#ifdef CONFIG_HUGETLBFS
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001456/*
1457 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1458 * @vma = virtual memory area whose policy is sought
1459 * @addr = address in @vma for shared policy lookup and interleave policy
1460 * @gfp_flags = for requested zone
Mel Gorman19770b32008-04-28 02:12:18 -07001461 * @mpol = pointer to mempolicy pointer for reference counted mempolicy
1462 * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001463 *
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001464 * Returns a zonelist suitable for a huge page allocation and a pointer
1465 * to the struct mempolicy for conditional unref after allocation.
1466 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1467 * @nodemask for filtering the zonelist.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001468 */
Mel Gorman396faf02007-07-17 04:03:13 -07001469struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
Mel Gorman19770b32008-04-28 02:12:18 -07001470 gfp_t gfp_flags, struct mempolicy **mpol,
1471 nodemask_t **nodemask)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001472{
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001473 struct zonelist *zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001474
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001475 *mpol = get_vma_policy(current, vma, addr);
Mel Gorman19770b32008-04-28 02:12:18 -07001476 *nodemask = NULL; /* assume !MPOL_BIND */
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001477
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001478 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1479 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
1480 HPAGE_SHIFT), gfp_flags);
1481 } else {
1482 zl = policy_zonelist(gfp_flags, *mpol);
1483 if ((*mpol)->mode == MPOL_BIND)
1484 *nodemask = &(*mpol)->v.nodes;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001485 }
1486 return zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001487}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001488#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001489
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490/* Allocate a page in interleaved policy.
1491 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001492static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1493 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494{
1495 struct zonelist *zl;
1496 struct page *page;
1497
Mel Gorman0e884602008-04-28 02:12:14 -07001498 zl = node_zonelist(nid, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 page = __alloc_pages(gfp, order, zl);
Mel Gormandd1a2392008-04-28 02:12:17 -07001500 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
Christoph Lameterca889e62006-06-30 01:55:44 -07001501 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 return page;
1503}
1504
1505/**
1506 * alloc_page_vma - Allocate a page for a VMA.
1507 *
1508 * @gfp:
1509 * %GFP_USER user allocation.
1510 * %GFP_KERNEL kernel allocations,
1511 * %GFP_HIGHMEM highmem/user allocations,
1512 * %GFP_FS allocation should not call back into a file system.
1513 * %GFP_ATOMIC don't sleep.
1514 *
1515 * @vma: Pointer to VMA or NULL if not available.
1516 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1517 *
1518 * This function allocates a page from the kernel page pool and applies
1519 * a NUMA policy associated with the VMA or the current process.
1520 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1521 * mm_struct of the VMA to prevent it from going away. Should be used for
1522 * all allocations for pages that will be mapped into
1523 * user space. Returns NULL when no page can be allocated.
1524 *
1525 * Should be called with the mm_sem of the vma hold.
1526 */
1527struct page *
Al Virodd0fc662005-10-07 07:46:04 +01001528alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001530 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001531 struct zonelist *zl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001533 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001535 if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001536 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001537
1538 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001539 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540 return alloc_page_interleave(gfp, 0, nid);
1541 }
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001542 zl = policy_zonelist(gfp, pol);
1543 if (unlikely(mpol_needs_cond_ref(pol))) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001544 /*
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001545 * slow path: ref counted shared policy
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001546 */
Mel Gorman19770b32008-04-28 02:12:18 -07001547 struct page *page = __alloc_pages_nodemask(gfp, 0,
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001548 zl, policy_nodemask(gfp, pol));
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001549 __mpol_put(pol);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001550 return page;
1551 }
1552 /*
1553 * fast path: default or task policy
1554 */
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001555 return __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001556}
1557
1558/**
1559 * alloc_pages_current - Allocate pages.
1560 *
1561 * @gfp:
1562 * %GFP_USER user allocation,
1563 * %GFP_KERNEL kernel allocation,
1564 * %GFP_HIGHMEM highmem allocation,
1565 * %GFP_FS don't call back into a file system.
1566 * %GFP_ATOMIC don't sleep.
1567 * @order: Power of two of allocation size in pages. 0 is a single page.
1568 *
1569 * Allocate a page from the kernel page pool. When not in
1570 * interrupt context and apply the current process NUMA policy.
1571 * Returns NULL when no page can be allocated.
1572 *
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001573 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574 * 1) it's ok to take cpuset_sem (can WAIT), and
1575 * 2) allocating for current task (not interrupt).
1576 */
Al Virodd0fc662005-10-07 07:46:04 +01001577struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578{
1579 struct mempolicy *pol = current->mempolicy;
1580
1581 if ((gfp & __GFP_WAIT) && !in_interrupt())
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001582 cpuset_update_task_memory_state();
Christoph Lameter9b819d22006-09-25 23:31:40 -07001583 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584 pol = &default_policy;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001585
1586 /*
1587 * No reference counting needed for current->mempolicy
1588 * nor system default_policy
1589 */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001590 if (pol->mode == MPOL_INTERLEAVE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
Mel Gorman19770b32008-04-28 02:12:18 -07001592 return __alloc_pages_nodemask(gfp, order,
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001593 policy_zonelist(gfp, pol), policy_nodemask(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594}
1595EXPORT_SYMBOL(alloc_pages_current);
1596
Paul Jackson42253992006-01-08 01:01:59 -08001597/*
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001598 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
Paul Jackson42253992006-01-08 01:01:59 -08001599 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1600 * with the mems_allowed returned by cpuset_mems_allowed(). This
1601 * keeps mempolicies cpuset relative after its cpuset moves. See
1602 * further kernel/cpuset.c update_nodemask().
1603 */
Paul Jackson42253992006-01-08 01:01:59 -08001604
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001605/* Slow path of a mempolicy duplicate */
1606struct mempolicy *__mpol_dup(struct mempolicy *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607{
1608 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1609
1610 if (!new)
1611 return ERR_PTR(-ENOMEM);
Paul Jackson42253992006-01-08 01:01:59 -08001612 if (current_cpuset_is_being_rebound()) {
1613 nodemask_t mems = cpuset_mems_allowed(current);
1614 mpol_rebind_policy(old, &mems);
1615 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616 *new = *old;
1617 atomic_set(&new->refcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 return new;
1619}
1620
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001621/*
1622 * If *frompol needs [has] an extra ref, copy *frompol to *tompol ,
1623 * eliminate the * MPOL_F_* flags that require conditional ref and
1624 * [NOTE!!!] drop the extra ref. Not safe to reference *frompol directly
1625 * after return. Use the returned value.
1626 *
1627 * Allows use of a mempolicy for, e.g., multiple allocations with a single
1628 * policy lookup, even if the policy needs/has extra ref on lookup.
1629 * shmem_readahead needs this.
1630 */
1631struct mempolicy *__mpol_cond_copy(struct mempolicy *tompol,
1632 struct mempolicy *frompol)
1633{
1634 if (!mpol_needs_cond_ref(frompol))
1635 return frompol;
1636
1637 *tompol = *frompol;
1638 tompol->flags &= ~MPOL_F_SHARED; /* copy doesn't need unref */
1639 __mpol_put(frompol);
1640 return tompol;
1641}
1642
David Rientjesf5b087b2008-04-28 02:12:27 -07001643static int mpol_match_intent(const struct mempolicy *a,
1644 const struct mempolicy *b)
1645{
1646 if (a->flags != b->flags)
1647 return 0;
1648 if (!mpol_store_user_nodemask(a))
1649 return 1;
1650 return nodes_equal(a->w.user_nodemask, b->w.user_nodemask);
1651}
1652
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653/* Slow path of a mempolicy comparison */
1654int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1655{
1656 if (!a || !b)
1657 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001658 if (a->mode != b->mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001660 if (a->mode != MPOL_DEFAULT && !mpol_match_intent(a, b))
David Rientjesf5b087b2008-04-28 02:12:27 -07001661 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001662 switch (a->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -07001663 case MPOL_BIND:
1664 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -07001666 return nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001668 return a->v.preferred_node == b->v.preferred_node &&
1669 a->flags == b->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670 default:
1671 BUG();
1672 return 0;
1673 }
1674}
1675
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677 * Shared memory backing store policy support.
1678 *
1679 * Remember policies even when nobody has shared memory mapped.
1680 * The policies are kept in Red-Black tree linked from the inode.
1681 * They are protected by the sp->lock spinlock, which should be held
1682 * for any accesses to the tree.
1683 */
1684
1685/* lookup first element intersecting start-end */
1686/* Caller holds sp->lock */
1687static struct sp_node *
1688sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1689{
1690 struct rb_node *n = sp->root.rb_node;
1691
1692 while (n) {
1693 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1694
1695 if (start >= p->end)
1696 n = n->rb_right;
1697 else if (end <= p->start)
1698 n = n->rb_left;
1699 else
1700 break;
1701 }
1702 if (!n)
1703 return NULL;
1704 for (;;) {
1705 struct sp_node *w = NULL;
1706 struct rb_node *prev = rb_prev(n);
1707 if (!prev)
1708 break;
1709 w = rb_entry(prev, struct sp_node, nd);
1710 if (w->end <= start)
1711 break;
1712 n = prev;
1713 }
1714 return rb_entry(n, struct sp_node, nd);
1715}
1716
1717/* Insert a new shared policy into the list. */
1718/* Caller holds sp->lock */
1719static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1720{
1721 struct rb_node **p = &sp->root.rb_node;
1722 struct rb_node *parent = NULL;
1723 struct sp_node *nd;
1724
1725 while (*p) {
1726 parent = *p;
1727 nd = rb_entry(parent, struct sp_node, nd);
1728 if (new->start < nd->start)
1729 p = &(*p)->rb_left;
1730 else if (new->end > nd->end)
1731 p = &(*p)->rb_right;
1732 else
1733 BUG();
1734 }
1735 rb_link_node(&new->nd, parent, p);
1736 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07001737 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001738 new->policy ? new->policy->mode : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739}
1740
1741/* Find shared policy intersecting idx */
1742struct mempolicy *
1743mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1744{
1745 struct mempolicy *pol = NULL;
1746 struct sp_node *sn;
1747
1748 if (!sp->root.rb_node)
1749 return NULL;
1750 spin_lock(&sp->lock);
1751 sn = sp_lookup(sp, idx, idx+1);
1752 if (sn) {
1753 mpol_get(sn->policy);
1754 pol = sn->policy;
1755 }
1756 spin_unlock(&sp->lock);
1757 return pol;
1758}
1759
1760static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1761{
Paul Mundt140d5a42007-07-15 23:38:16 -07001762 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001763 rb_erase(&n->nd, &sp->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001764 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765 kmem_cache_free(sn_cache, n);
1766}
1767
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001768static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
1769 struct mempolicy *pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001770{
1771 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1772
1773 if (!n)
1774 return NULL;
1775 n->start = start;
1776 n->end = end;
1777 mpol_get(pol);
Lee Schermerhornaab0b102008-04-28 02:13:13 -07001778 pol->flags |= MPOL_F_SHARED; /* for unref */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779 n->policy = pol;
1780 return n;
1781}
1782
1783/* Replace a policy range. */
1784static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1785 unsigned long end, struct sp_node *new)
1786{
1787 struct sp_node *n, *new2 = NULL;
1788
1789restart:
1790 spin_lock(&sp->lock);
1791 n = sp_lookup(sp, start, end);
1792 /* Take care of old policies in the same range. */
1793 while (n && n->start < end) {
1794 struct rb_node *next = rb_next(&n->nd);
1795 if (n->start >= start) {
1796 if (n->end <= end)
1797 sp_delete(sp, n);
1798 else
1799 n->start = end;
1800 } else {
1801 /* Old policy spanning whole new range. */
1802 if (n->end > end) {
1803 if (!new2) {
1804 spin_unlock(&sp->lock);
1805 new2 = sp_alloc(end, n->end, n->policy);
1806 if (!new2)
1807 return -ENOMEM;
1808 goto restart;
1809 }
1810 n->end = start;
1811 sp_insert(sp, new2);
1812 new2 = NULL;
1813 break;
1814 } else
1815 n->end = start;
1816 }
1817 if (!next)
1818 break;
1819 n = rb_entry(next, struct sp_node, nd);
1820 }
1821 if (new)
1822 sp_insert(sp, new);
1823 spin_unlock(&sp->lock);
1824 if (new2) {
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001825 mpol_put(new2->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001826 kmem_cache_free(sn_cache, new2);
1827 }
1828 return 0;
1829}
1830
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001831/**
1832 * mpol_shared_policy_init - initialize shared policy for inode
1833 * @sp: pointer to inode shared policy
1834 * @mpol: struct mempolicy to install
1835 *
1836 * Install non-NULL @mpol in inode's shared policy rb-tree.
1837 * On entry, the current task has a reference on a non-NULL @mpol.
1838 * This must be released on exit.
1839 */
1840void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
Robin Holt7339ff82006-01-14 13:20:48 -08001841{
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001842 sp->root = RB_ROOT; /* empty tree == default mempolicy */
1843 spin_lock_init(&sp->lock);
Robin Holt7339ff82006-01-14 13:20:48 -08001844
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001845 if (mpol) {
1846 struct vm_area_struct pvma;
1847 struct mempolicy *new;
Robin Holt7339ff82006-01-14 13:20:48 -08001848
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001849 /* contextualize the tmpfs mount point mempolicy */
1850 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
1851 mpol_put(mpol); /* drop our ref on sb mpol */
1852 if (IS_ERR(new))
1853 return; /* no valid nodemask intersection */
Robin Holt7339ff82006-01-14 13:20:48 -08001854
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001855 /* Create pseudo-vma that contains just the policy */
1856 memset(&pvma, 0, sizeof(struct vm_area_struct));
1857 pvma.vm_end = TASK_SIZE; /* policy covers entire file */
1858 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
1859 mpol_put(new); /* drop initial ref */
Robin Holt7339ff82006-01-14 13:20:48 -08001860 }
1861}
1862
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863int mpol_set_shared_policy(struct shared_policy *info,
1864 struct vm_area_struct *vma, struct mempolicy *npol)
1865{
1866 int err;
1867 struct sp_node *new = NULL;
1868 unsigned long sz = vma_pages(vma);
1869
David Rientjes028fec42008-04-28 02:12:25 -07001870 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001871 vma->vm_pgoff,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001872 sz, npol ? npol->mode : -1,
David Rientjes028fec42008-04-28 02:12:25 -07001873 npol ? npol->flags : -1,
Paul Mundt140d5a42007-07-15 23:38:16 -07001874 npol ? nodes_addr(npol->v.nodes)[0] : -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875
1876 if (npol) {
1877 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1878 if (!new)
1879 return -ENOMEM;
1880 }
1881 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1882 if (err && new)
1883 kmem_cache_free(sn_cache, new);
1884 return err;
1885}
1886
1887/* Free a backing policy store on inode delete. */
1888void mpol_free_shared_policy(struct shared_policy *p)
1889{
1890 struct sp_node *n;
1891 struct rb_node *next;
1892
1893 if (!p->root.rb_node)
1894 return;
1895 spin_lock(&p->lock);
1896 next = rb_first(&p->root);
1897 while (next) {
1898 n = rb_entry(next, struct sp_node, nd);
1899 next = rb_next(&n->nd);
Andi Kleen90c50292005-07-27 11:43:50 -07001900 rb_erase(&n->nd, &p->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001901 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902 kmem_cache_free(sn_cache, n);
1903 }
1904 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905}
1906
1907/* assumes fs == KERNEL_DS */
1908void __init numa_policy_init(void)
1909{
Paul Mundtb71636e2007-07-15 23:38:15 -07001910 nodemask_t interleave_nodes;
1911 unsigned long largest = 0;
1912 int nid, prefer = 0;
1913
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914 policy_cache = kmem_cache_create("numa_policy",
1915 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09001916 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917
1918 sn_cache = kmem_cache_create("shared_policy_node",
1919 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09001920 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921
Paul Mundtb71636e2007-07-15 23:38:15 -07001922 /*
1923 * Set interleaving policy for system init. Interleaving is only
1924 * enabled across suitably sized nodes (default is >= 16MB), or
1925 * fall back to the largest node if they're all smaller.
1926 */
1927 nodes_clear(interleave_nodes);
Christoph Lameter56bbd652007-10-16 01:25:35 -07001928 for_each_node_state(nid, N_HIGH_MEMORY) {
Paul Mundtb71636e2007-07-15 23:38:15 -07001929 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001930
Paul Mundtb71636e2007-07-15 23:38:15 -07001931 /* Preserve the largest node */
1932 if (largest < total_pages) {
1933 largest = total_pages;
1934 prefer = nid;
1935 }
1936
1937 /* Interleave this node? */
1938 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
1939 node_set(nid, interleave_nodes);
1940 }
1941
1942 /* All too small, use the largest */
1943 if (unlikely(nodes_empty(interleave_nodes)))
1944 node_set(prefer, interleave_nodes);
1945
David Rientjes028fec42008-04-28 02:12:25 -07001946 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947 printk("numa_policy_init: interleaving failed\n");
1948}
1949
Christoph Lameter8bccd852005-10-29 18:16:59 -07001950/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001951void numa_default_policy(void)
1952{
David Rientjes028fec42008-04-28 02:12:25 -07001953 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954}
Paul Jackson68860ec2005-10-30 15:02:36 -08001955
Paul Jackson42253992006-01-08 01:01:59 -08001956/*
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001957 * Parse and format mempolicy from/to strings
1958 */
1959
1960/*
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001961 * "local" is pseudo-policy: MPOL_PREFERRED with MPOL_F_LOCAL flag
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07001962 * Used only for mpol_parse_str() and mpol_to_str()
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001963 */
Lee Schermerhorn53f25562008-04-28 02:13:20 -07001964#define MPOL_LOCAL (MPOL_INTERLEAVE + 1)
Helge Deller15ad7cd2006-12-06 20:40:36 -08001965static const char * const policy_types[] =
Lee Schermerhorn53f25562008-04-28 02:13:20 -07001966 { "default", "prefer", "bind", "interleave", "local" };
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001967
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001968
1969#ifdef CONFIG_TMPFS
1970/**
1971 * mpol_parse_str - parse string to mempolicy
1972 * @str: string containing mempolicy to parse
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001973 * @mpol: pointer to struct mempolicy pointer, returned on success.
1974 * @no_context: flag whether to "contextualize" the mempolicy
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001975 *
1976 * Format of input:
1977 * <mode>[=<flags>][:<nodelist>]
1978 *
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001979 * if @no_context is true, save the input nodemask in w.user_nodemask in
1980 * the returned mempolicy. This will be used to "clone" the mempolicy in
1981 * a specific context [cpuset] at a later time. Used to parse tmpfs mpol
1982 * mount option. Note that if 'static' or 'relative' mode flags were
1983 * specified, the input nodemask will already have been saved. Saving
1984 * it again is redundant, but safe.
1985 *
1986 * On success, returns 0, else 1
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001987 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001988int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001989{
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001990 struct mempolicy *new = NULL;
1991 unsigned short uninitialized_var(mode);
1992 unsigned short uninitialized_var(mode_flags);
1993 nodemask_t nodes;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001994 char *nodelist = strchr(str, ':');
1995 char *flags = strchr(str, '=');
1996 int i;
1997 int err = 1;
1998
1999 if (nodelist) {
2000 /* NUL-terminate mode or flags string */
2001 *nodelist++ = '\0';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002002 if (nodelist_parse(nodelist, nodes))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002003 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002004 if (!nodes_subset(nodes, node_states[N_HIGH_MEMORY]))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002005 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002006 } else
2007 nodes_clear(nodes);
2008
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002009 if (flags)
2010 *flags++ = '\0'; /* terminate mode string */
2011
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002012 for (i = 0; i <= MPOL_LOCAL; i++) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002013 if (!strcmp(str, policy_types[i])) {
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002014 mode = i;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002015 break;
2016 }
2017 }
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002018 if (i > MPOL_LOCAL)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002019 goto out;
2020
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002021 switch (mode) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002022 case MPOL_PREFERRED:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002023 /*
2024 * Insist on a nodelist of one node only
2025 */
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002026 if (nodelist) {
2027 char *rest = nodelist;
2028 while (isdigit(*rest))
2029 rest++;
2030 if (!*rest)
2031 err = 0;
2032 }
2033 break;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002034 case MPOL_INTERLEAVE:
2035 /*
2036 * Default to online nodes with memory if no nodelist
2037 */
2038 if (!nodelist)
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002039 nodes = node_states[N_HIGH_MEMORY];
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002040 err = 0;
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002041 break;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002042 case MPOL_LOCAL:
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002043 /*
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002044 * Don't allow a nodelist; mpol_new() checks flags
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002045 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002046 if (nodelist)
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002047 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002048 mode = MPOL_PREFERRED;
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002049 break;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002050
2051 /*
2052 * case MPOL_BIND: mpol_new() enforces non-empty nodemask.
2053 * case MPOL_DEFAULT: mpol_new() enforces empty nodemask, ignores flags.
2054 */
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002055 }
2056
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002057 mode_flags = 0;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002058 if (flags) {
2059 /*
2060 * Currently, we only support two mutually exclusive
2061 * mode flags.
2062 */
2063 if (!strcmp(flags, "static"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002064 mode_flags |= MPOL_F_STATIC_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002065 else if (!strcmp(flags, "relative"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002066 mode_flags |= MPOL_F_RELATIVE_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002067 else
2068 err = 1;
2069 }
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002070
2071 new = mpol_new(mode, mode_flags, &nodes);
2072 if (IS_ERR(new))
2073 err = 1;
2074 else if (no_context)
2075 new->w.user_nodemask = nodes; /* save for contextualization */
2076
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002077out:
2078 /* Restore string for error message */
2079 if (nodelist)
2080 *--nodelist = ':';
2081 if (flags)
2082 *--flags = '=';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002083 if (!err)
2084 *mpol = new;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002085 return err;
2086}
2087#endif /* CONFIG_TMPFS */
2088
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002089/**
2090 * mpol_to_str - format a mempolicy structure for printing
2091 * @buffer: to contain formatted mempolicy string
2092 * @maxlen: length of @buffer
2093 * @pol: pointer to mempolicy to be formatted
2094 * @no_context: "context free" mempolicy - use nodemask in w.user_nodemask
2095 *
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002096 * Convert a mempolicy into a string.
2097 * Returns the number of characters in buffer (if positive)
2098 * or an error (negative)
2099 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002100int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol, int no_context)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002101{
2102 char *p = buffer;
2103 int l;
2104 nodemask_t nodes;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002105 unsigned short mode;
David Rientjesf5b087b2008-04-28 02:12:27 -07002106 unsigned short flags = pol ? pol->flags : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002107
Lee Schermerhorn22919902008-04-28 02:13:22 -07002108 /*
2109 * Sanity check: room for longest mode, flag and some nodes
2110 */
2111 VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16);
2112
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002113 if (!pol || pol == &default_policy)
2114 mode = MPOL_DEFAULT;
2115 else
2116 mode = pol->mode;
2117
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002118 switch (mode) {
2119 case MPOL_DEFAULT:
2120 nodes_clear(nodes);
2121 break;
2122
2123 case MPOL_PREFERRED:
2124 nodes_clear(nodes);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002125 if (flags & MPOL_F_LOCAL)
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002126 mode = MPOL_LOCAL; /* pseudo-policy */
2127 else
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002128 node_set(pol->v.preferred_node, nodes);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002129 break;
2130
2131 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07002132 /* Fall through */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002133 case MPOL_INTERLEAVE:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002134 if (no_context)
2135 nodes = pol->w.user_nodemask;
2136 else
2137 nodes = pol->v.nodes;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002138 break;
2139
2140 default:
2141 BUG();
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002142 }
2143
2144 l = strlen(policy_types[mode]);
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002145 if (buffer + maxlen < p + l + 1)
2146 return -ENOSPC;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002147
2148 strcpy(p, policy_types[mode]);
2149 p += l;
2150
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002151 if (flags & MPOL_MODE_FLAGS) {
David Rientjesf5b087b2008-04-28 02:12:27 -07002152 if (buffer + maxlen < p + 2)
2153 return -ENOSPC;
2154 *p++ = '=';
2155
Lee Schermerhorn22919902008-04-28 02:13:22 -07002156 /*
2157 * Currently, the only defined flags are mutually exclusive
2158 */
David Rientjesf5b087b2008-04-28 02:12:27 -07002159 if (flags & MPOL_F_STATIC_NODES)
Lee Schermerhorn22919902008-04-28 02:13:22 -07002160 p += snprintf(p, buffer + maxlen - p, "static");
2161 else if (flags & MPOL_F_RELATIVE_NODES)
2162 p += snprintf(p, buffer + maxlen - p, "relative");
David Rientjesf5b087b2008-04-28 02:12:27 -07002163 }
2164
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002165 if (!nodes_empty(nodes)) {
2166 if (buffer + maxlen < p + 2)
2167 return -ENOSPC;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002168 *p++ = ':';
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002169 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
2170 }
2171 return p - buffer;
2172}
2173
2174struct numa_maps {
2175 unsigned long pages;
2176 unsigned long anon;
Christoph Lameter397874d2006-03-06 15:42:53 -08002177 unsigned long active;
2178 unsigned long writeback;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002179 unsigned long mapcount_max;
Christoph Lameter397874d2006-03-06 15:42:53 -08002180 unsigned long dirty;
2181 unsigned long swapcache;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002182 unsigned long node[MAX_NUMNODES];
2183};
2184
Christoph Lameter397874d2006-03-06 15:42:53 -08002185static void gather_stats(struct page *page, void *private, int pte_dirty)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002186{
2187 struct numa_maps *md = private;
2188 int count = page_mapcount(page);
2189
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002190 md->pages++;
Christoph Lameter397874d2006-03-06 15:42:53 -08002191 if (pte_dirty || PageDirty(page))
2192 md->dirty++;
2193
2194 if (PageSwapCache(page))
2195 md->swapcache++;
2196
2197 if (PageActive(page))
2198 md->active++;
2199
2200 if (PageWriteback(page))
2201 md->writeback++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002202
2203 if (PageAnon(page))
2204 md->anon++;
2205
Christoph Lameter397874d2006-03-06 15:42:53 -08002206 if (count > md->mapcount_max)
2207 md->mapcount_max = count;
2208
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002209 md->node[page_to_nid(page)]++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002210}
2211
Andrew Morton7f709ed2006-03-07 21:55:22 -08002212#ifdef CONFIG_HUGETLB_PAGE
Christoph Lameter397874d2006-03-06 15:42:53 -08002213static void check_huge_range(struct vm_area_struct *vma,
2214 unsigned long start, unsigned long end,
2215 struct numa_maps *md)
2216{
2217 unsigned long addr;
2218 struct page *page;
2219
2220 for (addr = start; addr < end; addr += HPAGE_SIZE) {
2221 pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
2222 pte_t pte;
2223
2224 if (!ptep)
2225 continue;
2226
2227 pte = *ptep;
2228 if (pte_none(pte))
2229 continue;
2230
2231 page = pte_page(pte);
2232 if (!page)
2233 continue;
2234
2235 gather_stats(page, md, pte_dirty(*ptep));
2236 }
2237}
Andrew Morton7f709ed2006-03-07 21:55:22 -08002238#else
2239static inline void check_huge_range(struct vm_area_struct *vma,
2240 unsigned long start, unsigned long end,
2241 struct numa_maps *md)
2242{
2243}
2244#endif
Christoph Lameter397874d2006-03-06 15:42:53 -08002245
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002246/*
2247 * Display pages allocated per node and memory policy via /proc.
2248 */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002249int show_numa_map(struct seq_file *m, void *v)
2250{
Eric W. Biederman99f89552006-06-26 00:25:55 -07002251 struct proc_maps_private *priv = m->private;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002252 struct vm_area_struct *vma = v;
2253 struct numa_maps *md;
Christoph Lameter397874d2006-03-06 15:42:53 -08002254 struct file *file = vma->vm_file;
2255 struct mm_struct *mm = vma->vm_mm;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002256 struct mempolicy *pol;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002257 int n;
2258 char buffer[50];
2259
Christoph Lameter397874d2006-03-06 15:42:53 -08002260 if (!mm)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002261 return 0;
2262
2263 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
2264 if (!md)
2265 return 0;
2266
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002267 pol = get_vma_policy(priv->task, vma, vma->vm_start);
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002268 mpol_to_str(buffer, sizeof(buffer), pol, 0);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002269 mpol_cond_put(pol);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002270
Christoph Lameter397874d2006-03-06 15:42:53 -08002271 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002272
Christoph Lameter397874d2006-03-06 15:42:53 -08002273 if (file) {
2274 seq_printf(m, " file=");
Jan Blunckc32c2f62008-02-14 19:38:43 -08002275 seq_path(m, &file->f_path, "\n\t= ");
Christoph Lameter397874d2006-03-06 15:42:53 -08002276 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
2277 seq_printf(m, " heap");
2278 } else if (vma->vm_start <= mm->start_stack &&
2279 vma->vm_end >= mm->start_stack) {
2280 seq_printf(m, " stack");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002281 }
Christoph Lameter397874d2006-03-06 15:42:53 -08002282
2283 if (is_vm_hugetlb_page(vma)) {
2284 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
2285 seq_printf(m, " huge");
2286 } else {
2287 check_pgd_range(vma, vma->vm_start, vma->vm_end,
Christoph Lameter56bbd652007-10-16 01:25:35 -07002288 &node_states[N_HIGH_MEMORY], MPOL_MF_STATS, md);
Christoph Lameter397874d2006-03-06 15:42:53 -08002289 }
2290
2291 if (!md->pages)
2292 goto out;
2293
2294 if (md->anon)
2295 seq_printf(m," anon=%lu",md->anon);
2296
2297 if (md->dirty)
2298 seq_printf(m," dirty=%lu",md->dirty);
2299
2300 if (md->pages != md->anon && md->pages != md->dirty)
2301 seq_printf(m, " mapped=%lu", md->pages);
2302
2303 if (md->mapcount_max > 1)
2304 seq_printf(m, " mapmax=%lu", md->mapcount_max);
2305
2306 if (md->swapcache)
2307 seq_printf(m," swapcache=%lu", md->swapcache);
2308
2309 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
2310 seq_printf(m," active=%lu", md->active);
2311
2312 if (md->writeback)
2313 seq_printf(m," writeback=%lu", md->writeback);
2314
Christoph Lameter56bbd652007-10-16 01:25:35 -07002315 for_each_node_state(n, N_HIGH_MEMORY)
Christoph Lameter397874d2006-03-06 15:42:53 -08002316 if (md->node[n])
2317 seq_printf(m, " N%d=%lu", n, md->node[n]);
2318out:
2319 seq_putc(m, '\n');
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002320 kfree(md);
2321
2322 if (m->count < m->size)
Eric W. Biederman99f89552006-06-26 00:25:55 -07002323 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002324 return 0;
2325}