blob: e412ffa8e52ea945fb2ffa94a8de388e2de0875e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
Linus Torvalds1da177e2005-04-16 15:20:36 -070066*/
67
68#include <linux/mempolicy.h>
69#include <linux/mm.h>
70#include <linux/highmem.h>
71#include <linux/hugetlb.h>
72#include <linux/kernel.h>
73#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <linux/nodemask.h>
75#include <linux/cpuset.h>
76#include <linux/gfp.h>
77#include <linux/slab.h>
78#include <linux/string.h>
79#include <linux/module.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070080#include <linux/nsproxy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#include <linux/interrupt.h>
82#include <linux/init.h>
83#include <linux/compat.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080084#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080085#include <linux/seq_file.h>
86#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080087#include <linux/migrate.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070088#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070089#include <linux/security.h>
Adrian Bunkdbcb0f12007-10-16 01:26:26 -070090#include <linux/syscalls.h>
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -070091#include <linux/ctype.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080092
Linus Torvalds1da177e2005-04-16 15:20:36 -070093#include <asm/tlbflush.h>
94#include <asm/uaccess.h>
95
Nick Piggin62695a82008-10-18 20:26:09 -070096#include "internal.h"
97
Christoph Lameter38e35862006-01-08 01:01:01 -080098/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080099#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -0800100#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800101#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800102
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800103static struct kmem_cache *policy_cache;
104static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106/* Highest zone. An specific allocation for a zone below that is not
107 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800108enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700110/*
111 * run-time system-wide default policy => local allocation
112 */
Andi Kleend42c6992005-07-06 19:56:03 +0200113struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 .refcnt = ATOMIC_INIT(1), /* never free it */
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700115 .mode = MPOL_PREFERRED,
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700116 .flags = MPOL_F_LOCAL,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117};
118
David Rientjes37012942008-04-28 02:12:33 -0700119static const struct mempolicy_operations {
120 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
121 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
122} mpol_ops[MPOL_MAX];
123
Mel Gorman19770b32008-04-28 02:12:18 -0700124/* Check that the nodemask contains at least one populated zone */
David Rientjes37012942008-04-28 02:12:33 -0700125static int is_valid_nodemask(const nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126{
Mel Gorman19770b32008-04-28 02:12:18 -0700127 int nd, k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
Mel Gorman19770b32008-04-28 02:12:18 -0700129 /* Check that there is something useful in this mask */
130 k = policy_zone;
131
132 for_each_node_mask(nd, *nodemask) {
133 struct zone *z;
134
135 for (k = 0; k <= policy_zone; k++) {
136 z = &NODE_DATA(nd)->node_zones[k];
137 if (z->present_pages > 0)
138 return 1;
Andi Kleendd942ae2006-02-17 01:39:16 +0100139 }
140 }
Mel Gorman19770b32008-04-28 02:12:18 -0700141
142 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143}
144
David Rientjesf5b087b2008-04-28 02:12:27 -0700145static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
146{
David Rientjes4c50bc02008-04-28 02:12:30 -0700147 return pol->flags & (MPOL_F_STATIC_NODES | MPOL_F_RELATIVE_NODES);
148}
149
150static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
151 const nodemask_t *rel)
152{
153 nodemask_t tmp;
154 nodes_fold(tmp, *orig, nodes_weight(*rel));
155 nodes_onto(*ret, tmp, *rel);
David Rientjesf5b087b2008-04-28 02:12:27 -0700156}
157
David Rientjes37012942008-04-28 02:12:33 -0700158static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
159{
160 if (nodes_empty(*nodes))
161 return -EINVAL;
162 pol->v.nodes = *nodes;
163 return 0;
164}
165
166static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
167{
168 if (!nodes)
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700169 pol->flags |= MPOL_F_LOCAL; /* local allocation */
David Rientjes37012942008-04-28 02:12:33 -0700170 else if (nodes_empty(*nodes))
171 return -EINVAL; /* no allowed nodes */
172 else
173 pol->v.preferred_node = first_node(*nodes);
174 return 0;
175}
176
177static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
178{
179 if (!is_valid_nodemask(nodes))
180 return -EINVAL;
181 pol->v.nodes = *nodes;
182 return 0;
183}
184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185/* Create a new policy */
David Rientjes028fec42008-04-28 02:12:25 -0700186static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
187 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188{
189 struct mempolicy *policy;
David Rientjesf5b087b2008-04-28 02:12:27 -0700190 nodemask_t cpuset_context_nmask;
David Rientjes37012942008-04-28 02:12:33 -0700191 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
David Rientjes028fec42008-04-28 02:12:25 -0700193 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
194 mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
Paul Mundt140d5a42007-07-15 23:38:16 -0700195
David Rientjes3e1f0642008-04-28 02:12:34 -0700196 if (mode == MPOL_DEFAULT) {
197 if (nodes && !nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700198 return ERR_PTR(-EINVAL);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700199 return NULL; /* simply delete any existing policy */
David Rientjes37012942008-04-28 02:12:33 -0700200 }
David Rientjes3e1f0642008-04-28 02:12:34 -0700201 VM_BUG_ON(!nodes);
202
203 /*
204 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
205 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
206 * All other modes require a valid pointer to a non-empty nodemask.
207 */
208 if (mode == MPOL_PREFERRED) {
209 if (nodes_empty(*nodes)) {
210 if (((flags & MPOL_F_STATIC_NODES) ||
211 (flags & MPOL_F_RELATIVE_NODES)))
212 return ERR_PTR(-EINVAL);
213 nodes = NULL; /* flag local alloc */
214 }
215 } else if (nodes_empty(*nodes))
216 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
218 if (!policy)
219 return ERR_PTR(-ENOMEM);
220 atomic_set(&policy->refcnt, 1);
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700221 policy->mode = mode;
David Rientjes3e1f0642008-04-28 02:12:34 -0700222 policy->flags = flags;
David Rientjesf5b087b2008-04-28 02:12:27 -0700223
David Rientjes3e1f0642008-04-28 02:12:34 -0700224 if (nodes) {
225 /*
226 * cpuset related setup doesn't apply to local allocation
227 */
David Rientjes37012942008-04-28 02:12:33 -0700228 cpuset_update_task_memory_state();
229 if (flags & MPOL_F_RELATIVE_NODES)
230 mpol_relative_nodemask(&cpuset_context_nmask, nodes,
231 &cpuset_current_mems_allowed);
232 else
233 nodes_and(cpuset_context_nmask, *nodes,
234 cpuset_current_mems_allowed);
235 if (mpol_store_user_nodemask(policy))
236 policy->w.user_nodemask = *nodes;
237 else
238 policy->w.cpuset_mems_allowed =
239 cpuset_mems_allowed(current);
240 }
241
242 ret = mpol_ops[mode].create(policy,
David Rientjes3e1f0642008-04-28 02:12:34 -0700243 nodes ? &cpuset_context_nmask : NULL);
David Rientjes37012942008-04-28 02:12:33 -0700244 if (ret < 0) {
245 kmem_cache_free(policy_cache, policy);
246 return ERR_PTR(ret);
247 }
248 return policy;
249}
250
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700251/* Slow path of a mpol destructor. */
252void __mpol_put(struct mempolicy *p)
253{
254 if (!atomic_dec_and_test(&p->refcnt))
255 return;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700256 kmem_cache_free(policy_cache, p);
257}
258
David Rientjes37012942008-04-28 02:12:33 -0700259static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
260{
261}
262
263static void mpol_rebind_nodemask(struct mempolicy *pol,
264 const nodemask_t *nodes)
265{
266 nodemask_t tmp;
267
268 if (pol->flags & MPOL_F_STATIC_NODES)
269 nodes_and(tmp, pol->w.user_nodemask, *nodes);
270 else if (pol->flags & MPOL_F_RELATIVE_NODES)
271 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
272 else {
273 nodes_remap(tmp, pol->v.nodes, pol->w.cpuset_mems_allowed,
274 *nodes);
275 pol->w.cpuset_mems_allowed = *nodes;
276 }
277
278 pol->v.nodes = tmp;
279 if (!node_isset(current->il_next, tmp)) {
280 current->il_next = next_node(current->il_next, tmp);
281 if (current->il_next >= MAX_NUMNODES)
282 current->il_next = first_node(tmp);
283 if (current->il_next >= MAX_NUMNODES)
284 current->il_next = numa_node_id();
285 }
286}
287
288static void mpol_rebind_preferred(struct mempolicy *pol,
289 const nodemask_t *nodes)
290{
291 nodemask_t tmp;
292
David Rientjes37012942008-04-28 02:12:33 -0700293 if (pol->flags & MPOL_F_STATIC_NODES) {
294 int node = first_node(pol->w.user_nodemask);
295
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700296 if (node_isset(node, *nodes)) {
David Rientjes37012942008-04-28 02:12:33 -0700297 pol->v.preferred_node = node;
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700298 pol->flags &= ~MPOL_F_LOCAL;
299 } else
300 pol->flags |= MPOL_F_LOCAL;
David Rientjes37012942008-04-28 02:12:33 -0700301 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
302 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
303 pol->v.preferred_node = first_node(tmp);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700304 } else if (!(pol->flags & MPOL_F_LOCAL)) {
David Rientjes37012942008-04-28 02:12:33 -0700305 pol->v.preferred_node = node_remap(pol->v.preferred_node,
306 pol->w.cpuset_mems_allowed,
307 *nodes);
308 pol->w.cpuset_mems_allowed = *nodes;
309 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310}
311
David Rientjes1d0d2682008-04-28 02:12:32 -0700312/* Migrate a policy to a different set of nodes */
313static void mpol_rebind_policy(struct mempolicy *pol,
314 const nodemask_t *newmask)
315{
David Rientjes1d0d2682008-04-28 02:12:32 -0700316 if (!pol)
317 return;
David Rientjes1d0d2682008-04-28 02:12:32 -0700318 if (!mpol_store_user_nodemask(pol) &&
319 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
320 return;
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700321 mpol_ops[pol->mode].rebind(pol, newmask);
David Rientjes1d0d2682008-04-28 02:12:32 -0700322}
323
324/*
325 * Wrapper for mpol_rebind_policy() that just requires task
326 * pointer, and updates task mempolicy.
327 */
328
329void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
330{
331 mpol_rebind_policy(tsk->mempolicy, new);
332}
333
334/*
335 * Rebind each vma in mm to new nodemask.
336 *
337 * Call holding a reference to mm. Takes mm->mmap_sem during call.
338 */
339
340void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
341{
342 struct vm_area_struct *vma;
343
344 down_write(&mm->mmap_sem);
345 for (vma = mm->mmap; vma; vma = vma->vm_next)
346 mpol_rebind_policy(vma->vm_policy, new);
347 up_write(&mm->mmap_sem);
348}
349
David Rientjes37012942008-04-28 02:12:33 -0700350static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
351 [MPOL_DEFAULT] = {
352 .rebind = mpol_rebind_default,
353 },
354 [MPOL_INTERLEAVE] = {
355 .create = mpol_new_interleave,
356 .rebind = mpol_rebind_nodemask,
357 },
358 [MPOL_PREFERRED] = {
359 .create = mpol_new_preferred,
360 .rebind = mpol_rebind_preferred,
361 },
362 [MPOL_BIND] = {
363 .create = mpol_new_bind,
364 .rebind = mpol_rebind_nodemask,
365 },
366};
367
Christoph Lameter397874d2006-03-06 15:42:53 -0800368static void gather_stats(struct page *, void *, int pte_dirty);
Christoph Lameterfc301282006-01-18 17:42:29 -0800369static void migrate_page_add(struct page *page, struct list_head *pagelist,
370 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800371
Christoph Lameter38e35862006-01-08 01:01:01 -0800372/* Scan through pages checking if pages follow certain conditions. */
Nick Pigginb5810032005-10-29 18:16:12 -0700373static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800374 unsigned long addr, unsigned long end,
375 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800376 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377{
Hugh Dickins91612e02005-06-21 17:15:07 -0700378 pte_t *orig_pte;
379 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700380 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700381
Hugh Dickins705e87c2005-10-29 18:16:27 -0700382 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700383 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800384 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800385 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700386
387 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800389 page = vm_normal_page(vma, addr, *pte);
390 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800392 /*
393 * The check for PageReserved here is important to avoid
394 * handling zero pages and other pages that may have been
395 * marked special by the system.
396 *
397 * If the PageReserved would not be checked here then f.e.
398 * the location of the zero page could have an influence
399 * on MPOL_MF_STRICT, zero pages would be counted for
400 * the per node stats, and there would be useless attempts
401 * to put zero pages on the migration list.
402 */
Christoph Lameterf4598c82006-01-12 01:05:20 -0800403 if (PageReserved(page))
404 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800405 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800406 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
407 continue;
408
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800409 if (flags & MPOL_MF_STATS)
Christoph Lameter397874d2006-03-06 15:42:53 -0800410 gather_stats(page, private, pte_dirty(*pte));
Nick Piggin053837f2006-01-18 17:42:27 -0800411 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800412 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800413 else
414 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700415 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700416 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700417 return addr != end;
418}
419
Nick Pigginb5810032005-10-29 18:16:12 -0700420static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800421 unsigned long addr, unsigned long end,
422 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800423 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700424{
425 pmd_t *pmd;
426 unsigned long next;
427
428 pmd = pmd_offset(pud, addr);
429 do {
430 next = pmd_addr_end(addr, end);
431 if (pmd_none_or_clear_bad(pmd))
432 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800433 if (check_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800434 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700435 return -EIO;
436 } while (pmd++, addr = next, addr != end);
437 return 0;
438}
439
Nick Pigginb5810032005-10-29 18:16:12 -0700440static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800441 unsigned long addr, unsigned long end,
442 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800443 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700444{
445 pud_t *pud;
446 unsigned long next;
447
448 pud = pud_offset(pgd, addr);
449 do {
450 next = pud_addr_end(addr, end);
451 if (pud_none_or_clear_bad(pud))
452 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800453 if (check_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800454 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700455 return -EIO;
456 } while (pud++, addr = next, addr != end);
457 return 0;
458}
459
Nick Pigginb5810032005-10-29 18:16:12 -0700460static inline int check_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800461 unsigned long addr, unsigned long end,
462 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800463 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700464{
465 pgd_t *pgd;
466 unsigned long next;
467
Nick Pigginb5810032005-10-29 18:16:12 -0700468 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700469 do {
470 next = pgd_addr_end(addr, end);
471 if (pgd_none_or_clear_bad(pgd))
472 continue;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800473 if (check_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800474 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700475 return -EIO;
476 } while (pgd++, addr = next, addr != end);
477 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478}
479
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800480/*
481 * Check if all pages in a range are on a set of nodes.
482 * If pagelist != NULL then isolate pages from the LRU and
483 * put them on the pagelist.
484 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485static struct vm_area_struct *
486check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800487 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488{
489 int err;
490 struct vm_area_struct *first, *vma, *prev;
491
Nick Piggin053837f2006-01-18 17:42:27 -0800492
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 first = find_vma(mm, start);
494 if (!first)
495 return ERR_PTR(-EFAULT);
496 prev = NULL;
497 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800498 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
499 if (!vma->vm_next && vma->vm_end < end)
500 return ERR_PTR(-EFAULT);
501 if (prev && prev->vm_end < vma->vm_start)
502 return ERR_PTR(-EFAULT);
503 }
504 if (!is_vm_hugetlb_page(vma) &&
505 ((flags & MPOL_MF_STRICT) ||
506 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
507 vma_migratable(vma)))) {
Andi Kleen5b952b32005-09-13 01:25:08 -0700508 unsigned long endvma = vma->vm_end;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800509
Andi Kleen5b952b32005-09-13 01:25:08 -0700510 if (endvma > end)
511 endvma = end;
512 if (vma->vm_start > start)
513 start = vma->vm_start;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800514 err = check_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800515 flags, private);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 if (err) {
517 first = ERR_PTR(err);
518 break;
519 }
520 }
521 prev = vma;
522 }
523 return first;
524}
525
526/* Apply policy to a single VMA */
527static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
528{
529 int err = 0;
530 struct mempolicy *old = vma->vm_policy;
531
Paul Mundt140d5a42007-07-15 23:38:16 -0700532 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 vma->vm_start, vma->vm_end, vma->vm_pgoff,
534 vma->vm_ops, vma->vm_file,
535 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
536
537 if (vma->vm_ops && vma->vm_ops->set_policy)
538 err = vma->vm_ops->set_policy(vma, new);
539 if (!err) {
540 mpol_get(new);
541 vma->vm_policy = new;
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700542 mpol_put(old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 }
544 return err;
545}
546
547/* Step 2: apply policy to a range and do splits. */
548static int mbind_range(struct vm_area_struct *vma, unsigned long start,
549 unsigned long end, struct mempolicy *new)
550{
551 struct vm_area_struct *next;
552 int err;
553
554 err = 0;
555 for (; vma && vma->vm_start < end; vma = next) {
556 next = vma->vm_next;
557 if (vma->vm_start < start)
558 err = split_vma(vma->vm_mm, vma, start, 1);
559 if (!err && vma->vm_end > end)
560 err = split_vma(vma->vm_mm, vma, end, 0);
561 if (!err)
562 err = policy_vma(vma, new);
563 if (err)
564 break;
565 }
566 return err;
567}
568
Paul Jacksonc61afb12006-03-24 03:16:08 -0800569/*
570 * Update task->flags PF_MEMPOLICY bit: set iff non-default
571 * mempolicy. Allows more rapid checking of this (combined perhaps
572 * with other PF_* flag bits) on memory allocation hot code paths.
573 *
574 * If called from outside this file, the task 'p' should -only- be
575 * a newly forked child not yet visible on the task list, because
576 * manipulating the task flags of a visible task is not safe.
577 *
578 * The above limitation is why this routine has the funny name
579 * mpol_fix_fork_child_flag().
580 *
581 * It is also safe to call this with a task pointer of current,
582 * which the static wrapper mpol_set_task_struct_flag() does,
583 * for use within this file.
584 */
585
586void mpol_fix_fork_child_flag(struct task_struct *p)
587{
588 if (p->mempolicy)
589 p->flags |= PF_MEMPOLICY;
590 else
591 p->flags &= ~PF_MEMPOLICY;
592}
593
594static void mpol_set_task_struct_flag(void)
595{
596 mpol_fix_fork_child_flag(current);
597}
598
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599/* Set the process memory policy */
David Rientjes028fec42008-04-28 02:12:25 -0700600static long do_set_mempolicy(unsigned short mode, unsigned short flags,
601 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 struct mempolicy *new;
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700604 struct mm_struct *mm = current->mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
David Rientjes028fec42008-04-28 02:12:25 -0700606 new = mpol_new(mode, flags, nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 if (IS_ERR(new))
608 return PTR_ERR(new);
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700609
610 /*
611 * prevent changing our mempolicy while show_numa_maps()
612 * is using it.
613 * Note: do_set_mempolicy() can be called at init time
614 * with no 'mm'.
615 */
616 if (mm)
617 down_write(&mm->mmap_sem);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -0700618 mpol_put(current->mempolicy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 current->mempolicy = new;
Paul Jacksonc61afb12006-03-24 03:16:08 -0800620 mpol_set_task_struct_flag();
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700621 if (new && new->mode == MPOL_INTERLEAVE &&
David Rientjesf5b087b2008-04-28 02:12:27 -0700622 nodes_weight(new->v.nodes))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700623 current->il_next = first_node(new->v.nodes);
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700624 if (mm)
625 up_write(&mm->mmap_sem);
626
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 return 0;
628}
629
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700630/*
631 * Return nodemask for policy for get_mempolicy() query
632 */
633static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700635 nodes_clear(*nodes);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700636 if (p == &default_policy)
637 return;
638
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700639 switch (p->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -0700640 case MPOL_BIND:
641 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700643 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 break;
645 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700646 if (!(p->flags & MPOL_F_LOCAL))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700647 node_set(p->v.preferred_node, *nodes);
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700648 /* else return empty node mask for local allocation */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649 break;
650 default:
651 BUG();
652 }
653}
654
655static int lookup_node(struct mm_struct *mm, unsigned long addr)
656{
657 struct page *p;
658 int err;
659
660 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
661 if (err >= 0) {
662 err = page_to_nid(p);
663 put_page(p);
664 }
665 return err;
666}
667
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668/* Retrieve NUMA policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700669static long do_get_mempolicy(int *policy, nodemask_t *nmask,
670 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700672 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 struct mm_struct *mm = current->mm;
674 struct vm_area_struct *vma = NULL;
675 struct mempolicy *pol = current->mempolicy;
676
Paul Jacksoncf2a4732006-01-08 01:01:54 -0800677 cpuset_update_task_memory_state();
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700678 if (flags &
679 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 return -EINVAL;
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700681
682 if (flags & MPOL_F_MEMS_ALLOWED) {
683 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
684 return -EINVAL;
685 *policy = 0; /* just so it's initialized */
686 *nmask = cpuset_current_mems_allowed;
687 return 0;
688 }
689
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690 if (flags & MPOL_F_ADDR) {
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700691 /*
692 * Do NOT fall back to task policy if the
693 * vma/shared policy at addr is NULL. We
694 * want to return MPOL_DEFAULT in this case.
695 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696 down_read(&mm->mmap_sem);
697 vma = find_vma_intersection(mm, addr, addr+1);
698 if (!vma) {
699 up_read(&mm->mmap_sem);
700 return -EFAULT;
701 }
702 if (vma->vm_ops && vma->vm_ops->get_policy)
703 pol = vma->vm_ops->get_policy(vma, addr);
704 else
705 pol = vma->vm_policy;
706 } else if (addr)
707 return -EINVAL;
708
709 if (!pol)
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700710 pol = &default_policy; /* indicates default behavior */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711
712 if (flags & MPOL_F_NODE) {
713 if (flags & MPOL_F_ADDR) {
714 err = lookup_node(mm, addr);
715 if (err < 0)
716 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700717 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 } else if (pol == current->mempolicy &&
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700719 pol->mode == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700720 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 } else {
722 err = -EINVAL;
723 goto out;
724 }
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700725 } else {
726 *policy = pol == &default_policy ? MPOL_DEFAULT :
727 pol->mode;
David Rientjesd79df632008-07-04 12:24:13 -0700728 /*
729 * Internal mempolicy flags must be masked off before exposing
730 * the policy to userspace.
731 */
732 *policy |= (pol->flags & MPOL_MODE_FLAGS);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700733 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734
735 if (vma) {
736 up_read(&current->mm->mmap_sem);
737 vma = NULL;
738 }
739
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 err = 0;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700741 if (nmask)
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700742 get_policy_nodemask(pol, nmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743
744 out:
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700745 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 if (vma)
747 up_read(&current->mm->mmap_sem);
748 return err;
749}
750
Christoph Lameterb20a3502006-03-22 00:09:12 -0800751#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700752/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800753 * page migration
754 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800755static void migrate_page_add(struct page *page, struct list_head *pagelist,
756 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800757{
758 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800759 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800760 */
Nick Piggin62695a82008-10-18 20:26:09 -0700761 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
762 if (!isolate_lru_page(page)) {
763 list_add_tail(&page->lru, pagelist);
764 }
765 }
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800766}
767
Christoph Lameter742755a2006-06-23 02:03:55 -0700768static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700769{
Mel Gorman769848c2007-07-17 04:03:05 -0700770 return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700771}
772
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800773/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800774 * Migrate pages from one node to a target node.
775 * Returns error or the number of pages not migrated.
776 */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700777static int migrate_to_node(struct mm_struct *mm, int source, int dest,
778 int flags)
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800779{
780 nodemask_t nmask;
781 LIST_HEAD(pagelist);
782 int err = 0;
783
784 nodes_clear(nmask);
785 node_set(source, nmask);
786
787 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
788 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
789
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700790 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700791 err = migrate_pages(&pagelist, new_node_page, dest);
792
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800793 return err;
794}
795
796/*
797 * Move pages between the two nodesets so as to preserve the physical
798 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -0800799 *
800 * Returns the number of page that could not be moved.
801 */
802int do_migrate_pages(struct mm_struct *mm,
803 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
804{
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800805 int busy = 0;
Christoph Lameter0aedadf2008-11-06 12:53:30 -0800806 int err;
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800807 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -0800808
Christoph Lameter0aedadf2008-11-06 12:53:30 -0800809 err = migrate_prep();
810 if (err)
811 return err;
812
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700813 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -0800814
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700815 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
816 if (err)
817 goto out;
818
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800819/*
820 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
821 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
822 * bit in 'tmp', and return that <source, dest> pair for migration.
823 * The pair of nodemasks 'to' and 'from' define the map.
824 *
825 * If no pair of bits is found that way, fallback to picking some
826 * pair of 'source' and 'dest' bits that are not the same. If the
827 * 'source' and 'dest' bits are the same, this represents a node
828 * that will be migrating to itself, so no pages need move.
829 *
830 * If no bits are left in 'tmp', or if all remaining bits left
831 * in 'tmp' correspond to the same bit in 'to', return false
832 * (nothing left to migrate).
833 *
834 * This lets us pick a pair of nodes to migrate between, such that
835 * if possible the dest node is not already occupied by some other
836 * source node, minimizing the risk of overloading the memory on a
837 * node that would happen if we migrated incoming memory to a node
838 * before migrating outgoing memory source that same node.
839 *
840 * A single scan of tmp is sufficient. As we go, we remember the
841 * most recent <s, d> pair that moved (s != d). If we find a pair
842 * that not only moved, but what's better, moved to an empty slot
843 * (d is not set in tmp), then we break out then, with that pair.
844 * Otherwise when we finish scannng from_tmp, we at least have the
845 * most recent <s, d> pair that moved. If we get all the way through
846 * the scan of tmp without finding any node that moved, much less
847 * moved to an empty node, then there is nothing left worth migrating.
848 */
Christoph Lameterd4984712006-01-08 01:00:55 -0800849
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800850 tmp = *from_nodes;
851 while (!nodes_empty(tmp)) {
852 int s,d;
853 int source = -1;
854 int dest = 0;
855
856 for_each_node_mask(s, tmp) {
857 d = node_remap(s, *from_nodes, *to_nodes);
858 if (s == d)
859 continue;
860
861 source = s; /* Node moved. Memorize */
862 dest = d;
863
864 /* dest not in remaining from nodes? */
865 if (!node_isset(dest, tmp))
866 break;
867 }
868 if (source == -1)
869 break;
870
871 node_clear(source, tmp);
872 err = migrate_to_node(mm, source, dest, flags);
873 if (err > 0)
874 busy += err;
875 if (err < 0)
876 break;
Christoph Lameter39743882006-01-08 01:00:51 -0800877 }
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700878out:
Christoph Lameter39743882006-01-08 01:00:51 -0800879 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800880 if (err < 0)
881 return err;
882 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800883
Christoph Lameter39743882006-01-08 01:00:51 -0800884}
885
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800886/*
887 * Allocate a new page for page migration based on vma policy.
888 * Start assuming that page is mapped by vma pointed to by @private.
889 * Search forward from there, if not. N.B., this assumes that the
890 * list of pages handed to migrate_pages()--which is how we get here--
891 * is in virtual address order.
892 */
Christoph Lameter742755a2006-06-23 02:03:55 -0700893static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700894{
895 struct vm_area_struct *vma = (struct vm_area_struct *)private;
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800896 unsigned long uninitialized_var(address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700897
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -0800898 while (vma) {
899 address = page_address_in_vma(page, vma);
900 if (address != -EFAULT)
901 break;
902 vma = vma->vm_next;
903 }
904
905 /*
906 * if !vma, alloc_page_vma() will use task or system default policy
907 */
908 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700909}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800910#else
911
912static void migrate_page_add(struct page *page, struct list_head *pagelist,
913 unsigned long flags)
914{
915}
916
917int do_migrate_pages(struct mm_struct *mm,
918 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
919{
920 return -ENOSYS;
921}
Christoph Lameter95a402c2006-06-23 02:03:53 -0700922
Keith Owens69939742006-10-11 01:21:28 -0700923static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700924{
925 return NULL;
926}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800927#endif
928
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700929static long do_mbind(unsigned long start, unsigned long len,
David Rientjes028fec42008-04-28 02:12:25 -0700930 unsigned short mode, unsigned short mode_flags,
931 nodemask_t *nmask, unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800932{
933 struct vm_area_struct *vma;
934 struct mm_struct *mm = current->mm;
935 struct mempolicy *new;
936 unsigned long end;
937 int err;
938 LIST_HEAD(pagelist);
939
David Rientjesa3b51e02008-04-28 02:12:23 -0700940 if (flags & ~(unsigned long)(MPOL_MF_STRICT |
941 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800942 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -0800943 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800944 return -EPERM;
945
946 if (start & ~PAGE_MASK)
947 return -EINVAL;
948
949 if (mode == MPOL_DEFAULT)
950 flags &= ~MPOL_MF_STRICT;
951
952 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
953 end = start + len;
954
955 if (end < start)
956 return -EINVAL;
957 if (end == start)
958 return 0;
959
David Rientjes028fec42008-04-28 02:12:25 -0700960 new = mpol_new(mode, mode_flags, nmask);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800961 if (IS_ERR(new))
962 return PTR_ERR(new);
963
964 /*
965 * If we are using the default policy then operation
966 * on discontinuous address spaces is okay after all
967 */
968 if (!new)
969 flags |= MPOL_MF_DISCONTIG_OK;
970
David Rientjes028fec42008-04-28 02:12:25 -0700971 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
972 start, start + len, mode, mode_flags,
973 nmask ? nodes_addr(*nmask)[0] : -1);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800974
Christoph Lameter0aedadf2008-11-06 12:53:30 -0800975 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
976
977 err = migrate_prep();
978 if (err)
979 return err;
980 }
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800981 down_write(&mm->mmap_sem);
982 vma = check_range(mm, start, end, nmask,
983 flags | MPOL_MF_INVERT, &pagelist);
984
985 err = PTR_ERR(vma);
986 if (!IS_ERR(vma)) {
987 int nr_failed = 0;
988
989 err = mbind_range(vma, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800990
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800991 if (!list_empty(&pagelist))
Christoph Lameter95a402c2006-06-23 02:03:53 -0700992 nr_failed = migrate_pages(&pagelist, new_vma_page,
993 (unsigned long)vma);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800994
995 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
996 err = -EIO;
997 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800998
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800999 up_write(&mm->mmap_sem);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001000 mpol_put(new);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001001 return err;
1002}
1003
Christoph Lameter39743882006-01-08 01:00:51 -08001004/*
Christoph Lameter8bccd852005-10-29 18:16:59 -07001005 * User space interface with variable sized bitmaps for nodelists.
1006 */
1007
1008/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -08001009static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -07001010 unsigned long maxnode)
1011{
1012 unsigned long k;
1013 unsigned long nlongs;
1014 unsigned long endmask;
1015
1016 --maxnode;
1017 nodes_clear(*nodes);
1018 if (maxnode == 0 || !nmask)
1019 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -08001020 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -08001021 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001022
1023 nlongs = BITS_TO_LONGS(maxnode);
1024 if ((maxnode % BITS_PER_LONG) == 0)
1025 endmask = ~0UL;
1026 else
1027 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1028
1029 /* When the user specified more nodes than supported just check
1030 if the non supported part is all zero. */
1031 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1032 if (nlongs > PAGE_SIZE/sizeof(long))
1033 return -EINVAL;
1034 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1035 unsigned long t;
1036 if (get_user(t, nmask + k))
1037 return -EFAULT;
1038 if (k == nlongs - 1) {
1039 if (t & endmask)
1040 return -EINVAL;
1041 } else if (t)
1042 return -EINVAL;
1043 }
1044 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1045 endmask = ~0UL;
1046 }
1047
1048 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1049 return -EFAULT;
1050 nodes_addr(*nodes)[nlongs-1] &= endmask;
1051 return 0;
1052}
1053
1054/* Copy a kernel node mask to user space */
1055static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1056 nodemask_t *nodes)
1057{
1058 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1059 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1060
1061 if (copy > nbytes) {
1062 if (copy > PAGE_SIZE)
1063 return -EINVAL;
1064 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1065 return -EFAULT;
1066 copy = nbytes;
1067 }
1068 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1069}
1070
1071asmlinkage long sys_mbind(unsigned long start, unsigned long len,
1072 unsigned long mode,
1073 unsigned long __user *nmask, unsigned long maxnode,
1074 unsigned flags)
1075{
1076 nodemask_t nodes;
1077 int err;
David Rientjes028fec42008-04-28 02:12:25 -07001078 unsigned short mode_flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001079
David Rientjes028fec42008-04-28 02:12:25 -07001080 mode_flags = mode & MPOL_MODE_FLAGS;
1081 mode &= ~MPOL_MODE_FLAGS;
David Rientjesa3b51e02008-04-28 02:12:23 -07001082 if (mode >= MPOL_MAX)
1083 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001084 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1085 (mode_flags & MPOL_F_RELATIVE_NODES))
1086 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001087 err = get_nodes(&nodes, nmask, maxnode);
1088 if (err)
1089 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001090 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001091}
1092
1093/* Set the process memory policy */
1094asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
1095 unsigned long maxnode)
1096{
1097 int err;
1098 nodemask_t nodes;
David Rientjes028fec42008-04-28 02:12:25 -07001099 unsigned short flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001100
David Rientjes028fec42008-04-28 02:12:25 -07001101 flags = mode & MPOL_MODE_FLAGS;
1102 mode &= ~MPOL_MODE_FLAGS;
1103 if ((unsigned int)mode >= MPOL_MAX)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001104 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001105 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1106 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001107 err = get_nodes(&nodes, nmask, maxnode);
1108 if (err)
1109 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001110 return do_set_mempolicy(mode, flags, &nodes);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001111}
1112
Christoph Lameter39743882006-01-08 01:00:51 -08001113asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
1114 const unsigned long __user *old_nodes,
1115 const unsigned long __user *new_nodes)
1116{
David Howellsc69e8d92008-11-14 10:39:19 +11001117 const struct cred *cred = current_cred(), *tcred;
Christoph Lameter39743882006-01-08 01:00:51 -08001118 struct mm_struct *mm;
1119 struct task_struct *task;
1120 nodemask_t old;
1121 nodemask_t new;
1122 nodemask_t task_nodes;
1123 int err;
1124
1125 err = get_nodes(&old, old_nodes, maxnode);
1126 if (err)
1127 return err;
1128
1129 err = get_nodes(&new, new_nodes, maxnode);
1130 if (err)
1131 return err;
1132
1133 /* Find the mm_struct */
1134 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001135 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter39743882006-01-08 01:00:51 -08001136 if (!task) {
1137 read_unlock(&tasklist_lock);
1138 return -ESRCH;
1139 }
1140 mm = get_task_mm(task);
1141 read_unlock(&tasklist_lock);
1142
1143 if (!mm)
1144 return -EINVAL;
1145
1146 /*
1147 * Check if this process has the right to modify the specified
1148 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -08001149 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -08001150 * userid as the target process.
1151 */
David Howellsc69e8d92008-11-14 10:39:19 +11001152 rcu_read_lock();
1153 tcred = __task_cred(task);
David Howellsb6dff3e2008-11-14 10:39:16 +11001154 if (cred->euid != tcred->suid && cred->euid != tcred->uid &&
1155 cred->uid != tcred->suid && cred->uid != tcred->uid &&
Christoph Lameter74c00242006-03-14 19:50:21 -08001156 !capable(CAP_SYS_NICE)) {
David Howellsc69e8d92008-11-14 10:39:19 +11001157 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001158 err = -EPERM;
1159 goto out;
1160 }
David Howellsc69e8d92008-11-14 10:39:19 +11001161 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001162
1163 task_nodes = cpuset_mems_allowed(task);
1164 /* Is the user allowed to access the target nodes? */
Christoph Lameter74c00242006-03-14 19:50:21 -08001165 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001166 err = -EPERM;
1167 goto out;
1168 }
1169
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07001170 if (!nodes_subset(new, node_states[N_HIGH_MEMORY])) {
Christoph Lameter3b42d282007-08-31 00:12:08 -07001171 err = -EINVAL;
1172 goto out;
1173 }
1174
David Quigley86c3a762006-06-23 02:04:02 -07001175 err = security_task_movememory(task);
1176 if (err)
1177 goto out;
1178
Christoph Lameter511030b2006-02-28 16:58:57 -08001179 err = do_migrate_pages(mm, &old, &new,
Christoph Lameter74c00242006-03-14 19:50:21 -08001180 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter39743882006-01-08 01:00:51 -08001181out:
1182 mmput(mm);
1183 return err;
1184}
1185
1186
Christoph Lameter8bccd852005-10-29 18:16:59 -07001187/* Retrieve NUMA policy */
1188asmlinkage long sys_get_mempolicy(int __user *policy,
1189 unsigned long __user *nmask,
1190 unsigned long maxnode,
1191 unsigned long addr, unsigned long flags)
1192{
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001193 int err;
1194 int uninitialized_var(pval);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001195 nodemask_t nodes;
1196
1197 if (nmask != NULL && maxnode < MAX_NUMNODES)
1198 return -EINVAL;
1199
1200 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1201
1202 if (err)
1203 return err;
1204
1205 if (policy && put_user(pval, policy))
1206 return -EFAULT;
1207
1208 if (nmask)
1209 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1210
1211 return err;
1212}
1213
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214#ifdef CONFIG_COMPAT
1215
1216asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1217 compat_ulong_t __user *nmask,
1218 compat_ulong_t maxnode,
1219 compat_ulong_t addr, compat_ulong_t flags)
1220{
1221 long err;
1222 unsigned long __user *nm = NULL;
1223 unsigned long nr_bits, alloc_size;
1224 DECLARE_BITMAP(bm, MAX_NUMNODES);
1225
1226 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1227 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1228
1229 if (nmask)
1230 nm = compat_alloc_user_space(alloc_size);
1231
1232 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1233
1234 if (!err && nmask) {
1235 err = copy_from_user(bm, nm, alloc_size);
1236 /* ensure entire bitmap is zeroed */
1237 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1238 err |= compat_put_bitmap(nmask, bm, nr_bits);
1239 }
1240
1241 return err;
1242}
1243
1244asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1245 compat_ulong_t maxnode)
1246{
1247 long err = 0;
1248 unsigned long __user *nm = NULL;
1249 unsigned long nr_bits, alloc_size;
1250 DECLARE_BITMAP(bm, MAX_NUMNODES);
1251
1252 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1253 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1254
1255 if (nmask) {
1256 err = compat_get_bitmap(bm, nmask, nr_bits);
1257 nm = compat_alloc_user_space(alloc_size);
1258 err |= copy_to_user(nm, bm, alloc_size);
1259 }
1260
1261 if (err)
1262 return -EFAULT;
1263
1264 return sys_set_mempolicy(mode, nm, nr_bits+1);
1265}
1266
1267asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1268 compat_ulong_t mode, compat_ulong_t __user *nmask,
1269 compat_ulong_t maxnode, compat_ulong_t flags)
1270{
1271 long err = 0;
1272 unsigned long __user *nm = NULL;
1273 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001274 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275
1276 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1277 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1278
1279 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001280 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001282 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283 }
1284
1285 if (err)
1286 return -EFAULT;
1287
1288 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1289}
1290
1291#endif
1292
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001293/*
1294 * get_vma_policy(@task, @vma, @addr)
1295 * @task - task for fallback if vma policy == default
1296 * @vma - virtual memory area whose policy is sought
1297 * @addr - address in @vma for shared policy lookup
1298 *
1299 * Returns effective policy for a VMA at specified address.
1300 * Falls back to @task or system default policy, as necessary.
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001301 * Current or other task's task mempolicy and non-shared vma policies
1302 * are protected by the task's mmap_sem, which must be held for read by
1303 * the caller.
1304 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1305 * count--added by the get_policy() vm_op, as appropriate--to protect against
1306 * freeing by another task. It is the caller's responsibility to free the
1307 * extra reference for shared policies.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001308 */
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001309static struct mempolicy *get_vma_policy(struct task_struct *task,
Christoph Lameter48fce342006-01-08 01:01:03 -08001310 struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001312 struct mempolicy *pol = task->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313
1314 if (vma) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001315 if (vma->vm_ops && vma->vm_ops->get_policy) {
Lee Schermerhornae4d8c12008-04-28 02:13:11 -07001316 struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
1317 addr);
1318 if (vpol)
1319 pol = vpol;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001320 } else if (vma->vm_policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321 pol = vma->vm_policy;
1322 }
1323 if (!pol)
1324 pol = &default_policy;
1325 return pol;
1326}
1327
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001328/*
1329 * Return a nodemask representing a mempolicy for filtering nodes for
1330 * page allocation
1331 */
1332static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
Mel Gorman19770b32008-04-28 02:12:18 -07001333{
1334 /* Lower zones don't get a nodemask applied for MPOL_BIND */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001335 if (unlikely(policy->mode == MPOL_BIND) &&
Mel Gorman19770b32008-04-28 02:12:18 -07001336 gfp_zone(gfp) >= policy_zone &&
1337 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1338 return &policy->v.nodes;
1339
1340 return NULL;
1341}
1342
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001343/* Return a zonelist indicated by gfp for node representing a mempolicy */
1344static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345{
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001346 int nd = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001348 switch (policy->mode) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001350 if (!(policy->flags & MPOL_F_LOCAL))
1351 nd = policy->v.preferred_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 break;
1353 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07001354 /*
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001355 * Normally, MPOL_BIND allocations are node-local within the
1356 * allowed nodemask. However, if __GFP_THISNODE is set and the
1357 * current node is part of the mask, we use the zonelist for
1358 * the first node in the mask instead.
Mel Gorman19770b32008-04-28 02:12:18 -07001359 */
Mel Gorman19770b32008-04-28 02:12:18 -07001360 if (unlikely(gfp & __GFP_THISNODE) &&
1361 unlikely(!node_isset(nd, policy->v.nodes)))
1362 nd = first_node(policy->v.nodes);
1363 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 case MPOL_INTERLEAVE: /* should not happen */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 break;
1366 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 BUG();
1368 }
Mel Gorman0e884602008-04-28 02:12:14 -07001369 return node_zonelist(nd, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370}
1371
1372/* Do dynamic interleaving for a process */
1373static unsigned interleave_nodes(struct mempolicy *policy)
1374{
1375 unsigned nid, next;
1376 struct task_struct *me = current;
1377
1378 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001379 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001381 next = first_node(policy->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001382 if (next < MAX_NUMNODES)
1383 me->il_next = next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 return nid;
1385}
1386
Christoph Lameterdc85da12006-01-18 17:42:36 -08001387/*
1388 * Depending on the memory policy provide a node from which to allocate the
1389 * next slab entry.
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001390 * @policy must be protected by freeing by the caller. If @policy is
1391 * the current task's mempolicy, this protection is implicit, as only the
1392 * task can change it's policy. The system default policy requires no
1393 * such protection.
Christoph Lameterdc85da12006-01-18 17:42:36 -08001394 */
1395unsigned slab_node(struct mempolicy *policy)
1396{
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001397 if (!policy || policy->flags & MPOL_F_LOCAL)
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001398 return numa_node_id();
Christoph Lameter765c4502006-09-27 01:50:08 -07001399
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001400 switch (policy->mode) {
1401 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001402 /*
1403 * handled MPOL_F_LOCAL above
1404 */
1405 return policy->v.preferred_node;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001406
Christoph Lameterdc85da12006-01-18 17:42:36 -08001407 case MPOL_INTERLEAVE:
1408 return interleave_nodes(policy);
1409
Mel Gormandd1a2392008-04-28 02:12:17 -07001410 case MPOL_BIND: {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001411 /*
1412 * Follow bind policy behavior and start allocation at the
1413 * first node.
1414 */
Mel Gorman19770b32008-04-28 02:12:18 -07001415 struct zonelist *zonelist;
1416 struct zone *zone;
1417 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1418 zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0];
1419 (void)first_zones_zonelist(zonelist, highest_zoneidx,
1420 &policy->v.nodes,
1421 &zone);
1422 return zone->node;
Mel Gormandd1a2392008-04-28 02:12:17 -07001423 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08001424
Christoph Lameterdc85da12006-01-18 17:42:36 -08001425 default:
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001426 BUG();
Christoph Lameterdc85da12006-01-18 17:42:36 -08001427 }
1428}
1429
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430/* Do static interleaving for a VMA with known offset. */
1431static unsigned offset_il_node(struct mempolicy *pol,
1432 struct vm_area_struct *vma, unsigned long off)
1433{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001434 unsigned nnodes = nodes_weight(pol->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001435 unsigned target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 int c;
1437 int nid = -1;
1438
David Rientjesf5b087b2008-04-28 02:12:27 -07001439 if (!nnodes)
1440 return numa_node_id();
1441 target = (unsigned int)off % nnodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442 c = 0;
1443 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001444 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 c++;
1446 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447 return nid;
1448}
1449
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001450/* Determine a node number for interleave */
1451static inline unsigned interleave_nid(struct mempolicy *pol,
1452 struct vm_area_struct *vma, unsigned long addr, int shift)
1453{
1454 if (vma) {
1455 unsigned long off;
1456
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001457 /*
1458 * for small pages, there is no difference between
1459 * shift and PAGE_SHIFT, so the bit-shift is safe.
1460 * for huge pages, since vm_pgoff is in units of small
1461 * pages, we need to shift off the always 0 bits to get
1462 * a useful offset.
1463 */
1464 BUG_ON(shift < PAGE_SHIFT);
1465 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001466 off += (addr - vma->vm_start) >> shift;
1467 return offset_il_node(pol, vma, off);
1468 } else
1469 return interleave_nodes(pol);
1470}
1471
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001472#ifdef CONFIG_HUGETLBFS
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001473/*
1474 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1475 * @vma = virtual memory area whose policy is sought
1476 * @addr = address in @vma for shared policy lookup and interleave policy
1477 * @gfp_flags = for requested zone
Mel Gorman19770b32008-04-28 02:12:18 -07001478 * @mpol = pointer to mempolicy pointer for reference counted mempolicy
1479 * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001480 *
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001481 * Returns a zonelist suitable for a huge page allocation and a pointer
1482 * to the struct mempolicy for conditional unref after allocation.
1483 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1484 * @nodemask for filtering the zonelist.
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001485 */
Mel Gorman396faf02007-07-17 04:03:13 -07001486struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
Mel Gorman19770b32008-04-28 02:12:18 -07001487 gfp_t gfp_flags, struct mempolicy **mpol,
1488 nodemask_t **nodemask)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001489{
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001490 struct zonelist *zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001491
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001492 *mpol = get_vma_policy(current, vma, addr);
Mel Gorman19770b32008-04-28 02:12:18 -07001493 *nodemask = NULL; /* assume !MPOL_BIND */
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001494
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001495 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1496 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
Andi Kleena5516432008-07-23 21:27:41 -07001497 huge_page_shift(hstate_vma(vma))), gfp_flags);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001498 } else {
1499 zl = policy_zonelist(gfp_flags, *mpol);
1500 if ((*mpol)->mode == MPOL_BIND)
1501 *nodemask = &(*mpol)->v.nodes;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001502 }
1503 return zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001504}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001505#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001506
Linus Torvalds1da177e2005-04-16 15:20:36 -07001507/* Allocate a page in interleaved policy.
1508 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001509static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1510 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511{
1512 struct zonelist *zl;
1513 struct page *page;
1514
Mel Gorman0e884602008-04-28 02:12:14 -07001515 zl = node_zonelist(nid, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516 page = __alloc_pages(gfp, order, zl);
Mel Gormandd1a2392008-04-28 02:12:17 -07001517 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
Christoph Lameterca889e62006-06-30 01:55:44 -07001518 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519 return page;
1520}
1521
1522/**
1523 * alloc_page_vma - Allocate a page for a VMA.
1524 *
1525 * @gfp:
1526 * %GFP_USER user allocation.
1527 * %GFP_KERNEL kernel allocations,
1528 * %GFP_HIGHMEM highmem/user allocations,
1529 * %GFP_FS allocation should not call back into a file system.
1530 * %GFP_ATOMIC don't sleep.
1531 *
1532 * @vma: Pointer to VMA or NULL if not available.
1533 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1534 *
1535 * This function allocates a page from the kernel page pool and applies
1536 * a NUMA policy associated with the VMA or the current process.
1537 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1538 * mm_struct of the VMA to prevent it from going away. Should be used for
1539 * all allocations for pages that will be mapped into
1540 * user space. Returns NULL when no page can be allocated.
1541 *
1542 * Should be called with the mm_sem of the vma hold.
1543 */
1544struct page *
Al Virodd0fc662005-10-07 07:46:04 +01001545alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546{
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001547 struct mempolicy *pol = get_vma_policy(current, vma, addr);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001548 struct zonelist *zl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001550 cpuset_update_task_memory_state();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001552 if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001554
1555 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001556 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 return alloc_page_interleave(gfp, 0, nid);
1558 }
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001559 zl = policy_zonelist(gfp, pol);
1560 if (unlikely(mpol_needs_cond_ref(pol))) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001561 /*
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001562 * slow path: ref counted shared policy
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001563 */
Mel Gorman19770b32008-04-28 02:12:18 -07001564 struct page *page = __alloc_pages_nodemask(gfp, 0,
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001565 zl, policy_nodemask(gfp, pol));
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001566 __mpol_put(pol);
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001567 return page;
1568 }
1569 /*
1570 * fast path: default or task policy
1571 */
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001572 return __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573}
1574
1575/**
1576 * alloc_pages_current - Allocate pages.
1577 *
1578 * @gfp:
1579 * %GFP_USER user allocation,
1580 * %GFP_KERNEL kernel allocation,
1581 * %GFP_HIGHMEM highmem allocation,
1582 * %GFP_FS don't call back into a file system.
1583 * %GFP_ATOMIC don't sleep.
1584 * @order: Power of two of allocation size in pages. 0 is a single page.
1585 *
1586 * Allocate a page from the kernel page pool. When not in
1587 * interrupt context and apply the current process NUMA policy.
1588 * Returns NULL when no page can be allocated.
1589 *
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001590 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591 * 1) it's ok to take cpuset_sem (can WAIT), and
1592 * 2) allocating for current task (not interrupt).
1593 */
Al Virodd0fc662005-10-07 07:46:04 +01001594struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595{
1596 struct mempolicy *pol = current->mempolicy;
1597
1598 if ((gfp & __GFP_WAIT) && !in_interrupt())
Paul Jacksoncf2a4732006-01-08 01:01:54 -08001599 cpuset_update_task_memory_state();
Christoph Lameter9b819d22006-09-25 23:31:40 -07001600 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 pol = &default_policy;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001602
1603 /*
1604 * No reference counting needed for current->mempolicy
1605 * nor system default_policy
1606 */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001607 if (pol->mode == MPOL_INTERLEAVE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
Mel Gorman19770b32008-04-28 02:12:18 -07001609 return __alloc_pages_nodemask(gfp, order,
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001610 policy_zonelist(gfp, pol), policy_nodemask(gfp, pol));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611}
1612EXPORT_SYMBOL(alloc_pages_current);
1613
Paul Jackson42253992006-01-08 01:01:59 -08001614/*
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001615 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
Paul Jackson42253992006-01-08 01:01:59 -08001616 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1617 * with the mems_allowed returned by cpuset_mems_allowed(). This
1618 * keeps mempolicies cpuset relative after its cpuset moves. See
1619 * further kernel/cpuset.c update_nodemask().
1620 */
Paul Jackson42253992006-01-08 01:01:59 -08001621
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001622/* Slow path of a mempolicy duplicate */
1623struct mempolicy *__mpol_dup(struct mempolicy *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624{
1625 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1626
1627 if (!new)
1628 return ERR_PTR(-ENOMEM);
Paul Jackson42253992006-01-08 01:01:59 -08001629 if (current_cpuset_is_being_rebound()) {
1630 nodemask_t mems = cpuset_mems_allowed(current);
1631 mpol_rebind_policy(old, &mems);
1632 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 *new = *old;
1634 atomic_set(&new->refcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635 return new;
1636}
1637
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001638/*
1639 * If *frompol needs [has] an extra ref, copy *frompol to *tompol ,
1640 * eliminate the * MPOL_F_* flags that require conditional ref and
1641 * [NOTE!!!] drop the extra ref. Not safe to reference *frompol directly
1642 * after return. Use the returned value.
1643 *
1644 * Allows use of a mempolicy for, e.g., multiple allocations with a single
1645 * policy lookup, even if the policy needs/has extra ref on lookup.
1646 * shmem_readahead needs this.
1647 */
1648struct mempolicy *__mpol_cond_copy(struct mempolicy *tompol,
1649 struct mempolicy *frompol)
1650{
1651 if (!mpol_needs_cond_ref(frompol))
1652 return frompol;
1653
1654 *tompol = *frompol;
1655 tompol->flags &= ~MPOL_F_SHARED; /* copy doesn't need unref */
1656 __mpol_put(frompol);
1657 return tompol;
1658}
1659
David Rientjesf5b087b2008-04-28 02:12:27 -07001660static int mpol_match_intent(const struct mempolicy *a,
1661 const struct mempolicy *b)
1662{
1663 if (a->flags != b->flags)
1664 return 0;
1665 if (!mpol_store_user_nodemask(a))
1666 return 1;
1667 return nodes_equal(a->w.user_nodemask, b->w.user_nodemask);
1668}
1669
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670/* Slow path of a mempolicy comparison */
1671int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1672{
1673 if (!a || !b)
1674 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001675 if (a->mode != b->mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001677 if (a->mode != MPOL_DEFAULT && !mpol_match_intent(a, b))
David Rientjesf5b087b2008-04-28 02:12:27 -07001678 return 0;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001679 switch (a->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -07001680 case MPOL_BIND:
1681 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -07001683 return nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001685 return a->v.preferred_node == b->v.preferred_node &&
1686 a->flags == b->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001687 default:
1688 BUG();
1689 return 0;
1690 }
1691}
1692
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694 * Shared memory backing store policy support.
1695 *
1696 * Remember policies even when nobody has shared memory mapped.
1697 * The policies are kept in Red-Black tree linked from the inode.
1698 * They are protected by the sp->lock spinlock, which should be held
1699 * for any accesses to the tree.
1700 */
1701
1702/* lookup first element intersecting start-end */
1703/* Caller holds sp->lock */
1704static struct sp_node *
1705sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1706{
1707 struct rb_node *n = sp->root.rb_node;
1708
1709 while (n) {
1710 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1711
1712 if (start >= p->end)
1713 n = n->rb_right;
1714 else if (end <= p->start)
1715 n = n->rb_left;
1716 else
1717 break;
1718 }
1719 if (!n)
1720 return NULL;
1721 for (;;) {
1722 struct sp_node *w = NULL;
1723 struct rb_node *prev = rb_prev(n);
1724 if (!prev)
1725 break;
1726 w = rb_entry(prev, struct sp_node, nd);
1727 if (w->end <= start)
1728 break;
1729 n = prev;
1730 }
1731 return rb_entry(n, struct sp_node, nd);
1732}
1733
1734/* Insert a new shared policy into the list. */
1735/* Caller holds sp->lock */
1736static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1737{
1738 struct rb_node **p = &sp->root.rb_node;
1739 struct rb_node *parent = NULL;
1740 struct sp_node *nd;
1741
1742 while (*p) {
1743 parent = *p;
1744 nd = rb_entry(parent, struct sp_node, nd);
1745 if (new->start < nd->start)
1746 p = &(*p)->rb_left;
1747 else if (new->end > nd->end)
1748 p = &(*p)->rb_right;
1749 else
1750 BUG();
1751 }
1752 rb_link_node(&new->nd, parent, p);
1753 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07001754 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001755 new->policy ? new->policy->mode : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756}
1757
1758/* Find shared policy intersecting idx */
1759struct mempolicy *
1760mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1761{
1762 struct mempolicy *pol = NULL;
1763 struct sp_node *sn;
1764
1765 if (!sp->root.rb_node)
1766 return NULL;
1767 spin_lock(&sp->lock);
1768 sn = sp_lookup(sp, idx, idx+1);
1769 if (sn) {
1770 mpol_get(sn->policy);
1771 pol = sn->policy;
1772 }
1773 spin_unlock(&sp->lock);
1774 return pol;
1775}
1776
1777static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1778{
Paul Mundt140d5a42007-07-15 23:38:16 -07001779 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780 rb_erase(&n->nd, &sp->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001781 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782 kmem_cache_free(sn_cache, n);
1783}
1784
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001785static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
1786 struct mempolicy *pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001787{
1788 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1789
1790 if (!n)
1791 return NULL;
1792 n->start = start;
1793 n->end = end;
1794 mpol_get(pol);
Lee Schermerhornaab0b102008-04-28 02:13:13 -07001795 pol->flags |= MPOL_F_SHARED; /* for unref */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 n->policy = pol;
1797 return n;
1798}
1799
1800/* Replace a policy range. */
1801static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1802 unsigned long end, struct sp_node *new)
1803{
1804 struct sp_node *n, *new2 = NULL;
1805
1806restart:
1807 spin_lock(&sp->lock);
1808 n = sp_lookup(sp, start, end);
1809 /* Take care of old policies in the same range. */
1810 while (n && n->start < end) {
1811 struct rb_node *next = rb_next(&n->nd);
1812 if (n->start >= start) {
1813 if (n->end <= end)
1814 sp_delete(sp, n);
1815 else
1816 n->start = end;
1817 } else {
1818 /* Old policy spanning whole new range. */
1819 if (n->end > end) {
1820 if (!new2) {
1821 spin_unlock(&sp->lock);
1822 new2 = sp_alloc(end, n->end, n->policy);
1823 if (!new2)
1824 return -ENOMEM;
1825 goto restart;
1826 }
1827 n->end = start;
1828 sp_insert(sp, new2);
1829 new2 = NULL;
1830 break;
1831 } else
1832 n->end = start;
1833 }
1834 if (!next)
1835 break;
1836 n = rb_entry(next, struct sp_node, nd);
1837 }
1838 if (new)
1839 sp_insert(sp, new);
1840 spin_unlock(&sp->lock);
1841 if (new2) {
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001842 mpol_put(new2->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843 kmem_cache_free(sn_cache, new2);
1844 }
1845 return 0;
1846}
1847
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001848/**
1849 * mpol_shared_policy_init - initialize shared policy for inode
1850 * @sp: pointer to inode shared policy
1851 * @mpol: struct mempolicy to install
1852 *
1853 * Install non-NULL @mpol in inode's shared policy rb-tree.
1854 * On entry, the current task has a reference on a non-NULL @mpol.
1855 * This must be released on exit.
1856 */
1857void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
Robin Holt7339ff82006-01-14 13:20:48 -08001858{
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001859 sp->root = RB_ROOT; /* empty tree == default mempolicy */
1860 spin_lock_init(&sp->lock);
Robin Holt7339ff82006-01-14 13:20:48 -08001861
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001862 if (mpol) {
1863 struct vm_area_struct pvma;
1864 struct mempolicy *new;
Robin Holt7339ff82006-01-14 13:20:48 -08001865
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001866 /* contextualize the tmpfs mount point mempolicy */
1867 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
1868 mpol_put(mpol); /* drop our ref on sb mpol */
1869 if (IS_ERR(new))
1870 return; /* no valid nodemask intersection */
Robin Holt7339ff82006-01-14 13:20:48 -08001871
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001872 /* Create pseudo-vma that contains just the policy */
1873 memset(&pvma, 0, sizeof(struct vm_area_struct));
1874 pvma.vm_end = TASK_SIZE; /* policy covers entire file */
1875 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
1876 mpol_put(new); /* drop initial ref */
Robin Holt7339ff82006-01-14 13:20:48 -08001877 }
1878}
1879
Linus Torvalds1da177e2005-04-16 15:20:36 -07001880int mpol_set_shared_policy(struct shared_policy *info,
1881 struct vm_area_struct *vma, struct mempolicy *npol)
1882{
1883 int err;
1884 struct sp_node *new = NULL;
1885 unsigned long sz = vma_pages(vma);
1886
David Rientjes028fec42008-04-28 02:12:25 -07001887 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001888 vma->vm_pgoff,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001889 sz, npol ? npol->mode : -1,
David Rientjes028fec42008-04-28 02:12:25 -07001890 npol ? npol->flags : -1,
Paul Mundt140d5a42007-07-15 23:38:16 -07001891 npol ? nodes_addr(npol->v.nodes)[0] : -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892
1893 if (npol) {
1894 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1895 if (!new)
1896 return -ENOMEM;
1897 }
1898 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1899 if (err && new)
1900 kmem_cache_free(sn_cache, new);
1901 return err;
1902}
1903
1904/* Free a backing policy store on inode delete. */
1905void mpol_free_shared_policy(struct shared_policy *p)
1906{
1907 struct sp_node *n;
1908 struct rb_node *next;
1909
1910 if (!p->root.rb_node)
1911 return;
1912 spin_lock(&p->lock);
1913 next = rb_first(&p->root);
1914 while (next) {
1915 n = rb_entry(next, struct sp_node, nd);
1916 next = rb_next(&n->nd);
Andi Kleen90c50292005-07-27 11:43:50 -07001917 rb_erase(&n->nd, &p->root);
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001918 mpol_put(n->policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001919 kmem_cache_free(sn_cache, n);
1920 }
1921 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001922}
1923
1924/* assumes fs == KERNEL_DS */
1925void __init numa_policy_init(void)
1926{
Paul Mundtb71636e2007-07-15 23:38:15 -07001927 nodemask_t interleave_nodes;
1928 unsigned long largest = 0;
1929 int nid, prefer = 0;
1930
Linus Torvalds1da177e2005-04-16 15:20:36 -07001931 policy_cache = kmem_cache_create("numa_policy",
1932 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09001933 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934
1935 sn_cache = kmem_cache_create("shared_policy_node",
1936 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09001937 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938
Paul Mundtb71636e2007-07-15 23:38:15 -07001939 /*
1940 * Set interleaving policy for system init. Interleaving is only
1941 * enabled across suitably sized nodes (default is >= 16MB), or
1942 * fall back to the largest node if they're all smaller.
1943 */
1944 nodes_clear(interleave_nodes);
Christoph Lameter56bbd652007-10-16 01:25:35 -07001945 for_each_node_state(nid, N_HIGH_MEMORY) {
Paul Mundtb71636e2007-07-15 23:38:15 -07001946 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947
Paul Mundtb71636e2007-07-15 23:38:15 -07001948 /* Preserve the largest node */
1949 if (largest < total_pages) {
1950 largest = total_pages;
1951 prefer = nid;
1952 }
1953
1954 /* Interleave this node? */
1955 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
1956 node_set(nid, interleave_nodes);
1957 }
1958
1959 /* All too small, use the largest */
1960 if (unlikely(nodes_empty(interleave_nodes)))
1961 node_set(prefer, interleave_nodes);
1962
David Rientjes028fec42008-04-28 02:12:25 -07001963 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964 printk("numa_policy_init: interleaving failed\n");
1965}
1966
Christoph Lameter8bccd852005-10-29 18:16:59 -07001967/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968void numa_default_policy(void)
1969{
David Rientjes028fec42008-04-28 02:12:25 -07001970 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971}
Paul Jackson68860ec2005-10-30 15:02:36 -08001972
Paul Jackson42253992006-01-08 01:01:59 -08001973/*
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001974 * Parse and format mempolicy from/to strings
1975 */
1976
1977/*
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001978 * "local" is pseudo-policy: MPOL_PREFERRED with MPOL_F_LOCAL flag
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07001979 * Used only for mpol_parse_str() and mpol_to_str()
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001980 */
Lee Schermerhorn53f25562008-04-28 02:13:20 -07001981#define MPOL_LOCAL (MPOL_INTERLEAVE + 1)
Helge Deller15ad7cd2006-12-06 20:40:36 -08001982static const char * const policy_types[] =
Lee Schermerhorn53f25562008-04-28 02:13:20 -07001983 { "default", "prefer", "bind", "interleave", "local" };
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08001984
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001985
1986#ifdef CONFIG_TMPFS
1987/**
1988 * mpol_parse_str - parse string to mempolicy
1989 * @str: string containing mempolicy to parse
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001990 * @mpol: pointer to struct mempolicy pointer, returned on success.
1991 * @no_context: flag whether to "contextualize" the mempolicy
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07001992 *
1993 * Format of input:
1994 * <mode>[=<flags>][:<nodelist>]
1995 *
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001996 * if @no_context is true, save the input nodemask in w.user_nodemask in
1997 * the returned mempolicy. This will be used to "clone" the mempolicy in
1998 * a specific context [cpuset] at a later time. Used to parse tmpfs mpol
1999 * mount option. Note that if 'static' or 'relative' mode flags were
2000 * specified, the input nodemask will already have been saved. Saving
2001 * it again is redundant, but safe.
2002 *
2003 * On success, returns 0, else 1
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002004 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002005int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002006{
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002007 struct mempolicy *new = NULL;
2008 unsigned short uninitialized_var(mode);
2009 unsigned short uninitialized_var(mode_flags);
2010 nodemask_t nodes;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002011 char *nodelist = strchr(str, ':');
2012 char *flags = strchr(str, '=');
2013 int i;
2014 int err = 1;
2015
2016 if (nodelist) {
2017 /* NUL-terminate mode or flags string */
2018 *nodelist++ = '\0';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002019 if (nodelist_parse(nodelist, nodes))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002020 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002021 if (!nodes_subset(nodes, node_states[N_HIGH_MEMORY]))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002022 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002023 } else
2024 nodes_clear(nodes);
2025
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002026 if (flags)
2027 *flags++ = '\0'; /* terminate mode string */
2028
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002029 for (i = 0; i <= MPOL_LOCAL; i++) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002030 if (!strcmp(str, policy_types[i])) {
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002031 mode = i;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002032 break;
2033 }
2034 }
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002035 if (i > MPOL_LOCAL)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002036 goto out;
2037
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002038 switch (mode) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002039 case MPOL_PREFERRED:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002040 /*
2041 * Insist on a nodelist of one node only
2042 */
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002043 if (nodelist) {
2044 char *rest = nodelist;
2045 while (isdigit(*rest))
2046 rest++;
2047 if (!*rest)
2048 err = 0;
2049 }
2050 break;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002051 case MPOL_INTERLEAVE:
2052 /*
2053 * Default to online nodes with memory if no nodelist
2054 */
2055 if (!nodelist)
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002056 nodes = node_states[N_HIGH_MEMORY];
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002057 err = 0;
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002058 break;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002059 case MPOL_LOCAL:
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002060 /*
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002061 * Don't allow a nodelist; mpol_new() checks flags
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002062 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002063 if (nodelist)
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002064 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002065 mode = MPOL_PREFERRED;
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002066 break;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002067
2068 /*
2069 * case MPOL_BIND: mpol_new() enforces non-empty nodemask.
2070 * case MPOL_DEFAULT: mpol_new() enforces empty nodemask, ignores flags.
2071 */
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002072 }
2073
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002074 mode_flags = 0;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002075 if (flags) {
2076 /*
2077 * Currently, we only support two mutually exclusive
2078 * mode flags.
2079 */
2080 if (!strcmp(flags, "static"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002081 mode_flags |= MPOL_F_STATIC_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002082 else if (!strcmp(flags, "relative"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002083 mode_flags |= MPOL_F_RELATIVE_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002084 else
2085 err = 1;
2086 }
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002087
2088 new = mpol_new(mode, mode_flags, &nodes);
2089 if (IS_ERR(new))
2090 err = 1;
2091 else if (no_context)
2092 new->w.user_nodemask = nodes; /* save for contextualization */
2093
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002094out:
2095 /* Restore string for error message */
2096 if (nodelist)
2097 *--nodelist = ':';
2098 if (flags)
2099 *--flags = '=';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002100 if (!err)
2101 *mpol = new;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002102 return err;
2103}
2104#endif /* CONFIG_TMPFS */
2105
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002106/**
2107 * mpol_to_str - format a mempolicy structure for printing
2108 * @buffer: to contain formatted mempolicy string
2109 * @maxlen: length of @buffer
2110 * @pol: pointer to mempolicy to be formatted
2111 * @no_context: "context free" mempolicy - use nodemask in w.user_nodemask
2112 *
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002113 * Convert a mempolicy into a string.
2114 * Returns the number of characters in buffer (if positive)
2115 * or an error (negative)
2116 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002117int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol, int no_context)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002118{
2119 char *p = buffer;
2120 int l;
2121 nodemask_t nodes;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002122 unsigned short mode;
David Rientjesf5b087b2008-04-28 02:12:27 -07002123 unsigned short flags = pol ? pol->flags : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002124
Lee Schermerhorn22919902008-04-28 02:13:22 -07002125 /*
2126 * Sanity check: room for longest mode, flag and some nodes
2127 */
2128 VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16);
2129
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002130 if (!pol || pol == &default_policy)
2131 mode = MPOL_DEFAULT;
2132 else
2133 mode = pol->mode;
2134
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002135 switch (mode) {
2136 case MPOL_DEFAULT:
2137 nodes_clear(nodes);
2138 break;
2139
2140 case MPOL_PREFERRED:
2141 nodes_clear(nodes);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002142 if (flags & MPOL_F_LOCAL)
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002143 mode = MPOL_LOCAL; /* pseudo-policy */
2144 else
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002145 node_set(pol->v.preferred_node, nodes);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002146 break;
2147
2148 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07002149 /* Fall through */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002150 case MPOL_INTERLEAVE:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002151 if (no_context)
2152 nodes = pol->w.user_nodemask;
2153 else
2154 nodes = pol->v.nodes;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002155 break;
2156
2157 default:
2158 BUG();
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002159 }
2160
2161 l = strlen(policy_types[mode]);
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002162 if (buffer + maxlen < p + l + 1)
2163 return -ENOSPC;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002164
2165 strcpy(p, policy_types[mode]);
2166 p += l;
2167
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002168 if (flags & MPOL_MODE_FLAGS) {
David Rientjesf5b087b2008-04-28 02:12:27 -07002169 if (buffer + maxlen < p + 2)
2170 return -ENOSPC;
2171 *p++ = '=';
2172
Lee Schermerhorn22919902008-04-28 02:13:22 -07002173 /*
2174 * Currently, the only defined flags are mutually exclusive
2175 */
David Rientjesf5b087b2008-04-28 02:12:27 -07002176 if (flags & MPOL_F_STATIC_NODES)
Lee Schermerhorn22919902008-04-28 02:13:22 -07002177 p += snprintf(p, buffer + maxlen - p, "static");
2178 else if (flags & MPOL_F_RELATIVE_NODES)
2179 p += snprintf(p, buffer + maxlen - p, "relative");
David Rientjesf5b087b2008-04-28 02:12:27 -07002180 }
2181
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002182 if (!nodes_empty(nodes)) {
2183 if (buffer + maxlen < p + 2)
2184 return -ENOSPC;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002185 *p++ = ':';
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002186 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
2187 }
2188 return p - buffer;
2189}
2190
2191struct numa_maps {
2192 unsigned long pages;
2193 unsigned long anon;
Christoph Lameter397874d2006-03-06 15:42:53 -08002194 unsigned long active;
2195 unsigned long writeback;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002196 unsigned long mapcount_max;
Christoph Lameter397874d2006-03-06 15:42:53 -08002197 unsigned long dirty;
2198 unsigned long swapcache;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002199 unsigned long node[MAX_NUMNODES];
2200};
2201
Christoph Lameter397874d2006-03-06 15:42:53 -08002202static void gather_stats(struct page *page, void *private, int pte_dirty)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002203{
2204 struct numa_maps *md = private;
2205 int count = page_mapcount(page);
2206
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002207 md->pages++;
Christoph Lameter397874d2006-03-06 15:42:53 -08002208 if (pte_dirty || PageDirty(page))
2209 md->dirty++;
2210
2211 if (PageSwapCache(page))
2212 md->swapcache++;
2213
Lee Schermerhorn894bc312008-10-18 20:26:39 -07002214 if (PageActive(page) || PageUnevictable(page))
Christoph Lameter397874d2006-03-06 15:42:53 -08002215 md->active++;
2216
2217 if (PageWriteback(page))
2218 md->writeback++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002219
2220 if (PageAnon(page))
2221 md->anon++;
2222
Christoph Lameter397874d2006-03-06 15:42:53 -08002223 if (count > md->mapcount_max)
2224 md->mapcount_max = count;
2225
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002226 md->node[page_to_nid(page)]++;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002227}
2228
Andrew Morton7f709ed2006-03-07 21:55:22 -08002229#ifdef CONFIG_HUGETLB_PAGE
Christoph Lameter397874d2006-03-06 15:42:53 -08002230static void check_huge_range(struct vm_area_struct *vma,
2231 unsigned long start, unsigned long end,
2232 struct numa_maps *md)
2233{
2234 unsigned long addr;
2235 struct page *page;
Andi Kleena5516432008-07-23 21:27:41 -07002236 struct hstate *h = hstate_vma(vma);
2237 unsigned long sz = huge_page_size(h);
Christoph Lameter397874d2006-03-06 15:42:53 -08002238
Andi Kleena5516432008-07-23 21:27:41 -07002239 for (addr = start; addr < end; addr += sz) {
2240 pte_t *ptep = huge_pte_offset(vma->vm_mm,
2241 addr & huge_page_mask(h));
Christoph Lameter397874d2006-03-06 15:42:53 -08002242 pte_t pte;
2243
2244 if (!ptep)
2245 continue;
2246
2247 pte = *ptep;
2248 if (pte_none(pte))
2249 continue;
2250
2251 page = pte_page(pte);
2252 if (!page)
2253 continue;
2254
2255 gather_stats(page, md, pte_dirty(*ptep));
2256 }
2257}
Andrew Morton7f709ed2006-03-07 21:55:22 -08002258#else
2259static inline void check_huge_range(struct vm_area_struct *vma,
2260 unsigned long start, unsigned long end,
2261 struct numa_maps *md)
2262{
2263}
2264#endif
Christoph Lameter397874d2006-03-06 15:42:53 -08002265
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002266/*
2267 * Display pages allocated per node and memory policy via /proc.
2268 */
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002269int show_numa_map(struct seq_file *m, void *v)
2270{
Eric W. Biederman99f89552006-06-26 00:25:55 -07002271 struct proc_maps_private *priv = m->private;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002272 struct vm_area_struct *vma = v;
2273 struct numa_maps *md;
Christoph Lameter397874d2006-03-06 15:42:53 -08002274 struct file *file = vma->vm_file;
2275 struct mm_struct *mm = vma->vm_mm;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002276 struct mempolicy *pol;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002277 int n;
2278 char buffer[50];
2279
Christoph Lameter397874d2006-03-06 15:42:53 -08002280 if (!mm)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002281 return 0;
2282
2283 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
2284 if (!md)
2285 return 0;
2286
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07002287 pol = get_vma_policy(priv->task, vma, vma->vm_start);
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002288 mpol_to_str(buffer, sizeof(buffer), pol, 0);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002289 mpol_cond_put(pol);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002290
Christoph Lameter397874d2006-03-06 15:42:53 -08002291 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002292
Christoph Lameter397874d2006-03-06 15:42:53 -08002293 if (file) {
2294 seq_printf(m, " file=");
Jan Blunckc32c2f62008-02-14 19:38:43 -08002295 seq_path(m, &file->f_path, "\n\t= ");
Christoph Lameter397874d2006-03-06 15:42:53 -08002296 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
2297 seq_printf(m, " heap");
2298 } else if (vma->vm_start <= mm->start_stack &&
2299 vma->vm_end >= mm->start_stack) {
2300 seq_printf(m, " stack");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002301 }
Christoph Lameter397874d2006-03-06 15:42:53 -08002302
2303 if (is_vm_hugetlb_page(vma)) {
2304 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
2305 seq_printf(m, " huge");
2306 } else {
2307 check_pgd_range(vma, vma->vm_start, vma->vm_end,
Christoph Lameter56bbd652007-10-16 01:25:35 -07002308 &node_states[N_HIGH_MEMORY], MPOL_MF_STATS, md);
Christoph Lameter397874d2006-03-06 15:42:53 -08002309 }
2310
2311 if (!md->pages)
2312 goto out;
2313
2314 if (md->anon)
2315 seq_printf(m," anon=%lu",md->anon);
2316
2317 if (md->dirty)
2318 seq_printf(m," dirty=%lu",md->dirty);
2319
2320 if (md->pages != md->anon && md->pages != md->dirty)
2321 seq_printf(m, " mapped=%lu", md->pages);
2322
2323 if (md->mapcount_max > 1)
2324 seq_printf(m, " mapmax=%lu", md->mapcount_max);
2325
2326 if (md->swapcache)
2327 seq_printf(m," swapcache=%lu", md->swapcache);
2328
2329 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
2330 seq_printf(m," active=%lu", md->active);
2331
2332 if (md->writeback)
2333 seq_printf(m," writeback=%lu", md->writeback);
2334
Christoph Lameter56bbd652007-10-16 01:25:35 -07002335 for_each_node_state(n, N_HIGH_MEMORY)
Christoph Lameter397874d2006-03-06 15:42:53 -08002336 if (md->node[n])
2337 seq_printf(m, " N%d=%lu", n, md->node[n]);
2338out:
2339 seq_putc(m, '\n');
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002340 kfree(md);
2341
2342 if (m->count < m->size)
Eric W. Biederman99f89552006-06-26 00:25:55 -07002343 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002344 return 0;
2345}