blob: 0e0961b8c39ceb18a7eca485753d1a74c905d353 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07005 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
David Rientjes00ef2d22013-02-22 16:35:36 -080029 * As a special case NUMA_NO_NODE here means do the allocation
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
Linus Torvalds1da177e2005-04-16 15:20:36 -070066*/
67
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -070068#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
69
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <linux/mempolicy.h>
71#include <linux/mm.h>
72#include <linux/highmem.h>
73#include <linux/hugetlb.h>
74#include <linux/kernel.h>
75#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/nodemask.h>
77#include <linux/cpuset.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/slab.h>
79#include <linux/string.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040080#include <linux/export.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070081#include <linux/nsproxy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070082#include <linux/interrupt.h>
83#include <linux/init.h>
84#include <linux/compat.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080085#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080086#include <linux/seq_file.h>
87#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080088#include <linux/migrate.h>
Hugh Dickins62b61f62009-12-14 17:59:33 -080089#include <linux/ksm.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070090#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070091#include <linux/security.h>
Adrian Bunkdbcb0f12007-10-16 01:26:26 -070092#include <linux/syscalls.h>
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -070093#include <linux/ctype.h>
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -080094#include <linux/mm_inline.h>
Lee Schermerhornb24f53a2012-10-25 14:16:32 +020095#include <linux/mmu_notifier.h>
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -070096#include <linux/printk.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080097
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <asm/tlbflush.h>
99#include <asm/uaccess.h>
Michal Hocko778d3b02011-07-26 16:08:30 -0700100#include <linux/random.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101
Nick Piggin62695a82008-10-18 20:26:09 -0700102#include "internal.h"
103
Christoph Lameter38e35862006-01-08 01:01:01 -0800104/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800105#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -0800106#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800107
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800108static struct kmem_cache *policy_cache;
109static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111/* Highest zone. An specific allocation for a zone below that is not
112 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800113enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700115/*
116 * run-time system-wide default policy => local allocation
117 */
H Hartley Sweetene754d792011-10-31 17:09:23 -0700118static struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 .refcnt = ATOMIC_INIT(1), /* never free it */
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700120 .mode = MPOL_PREFERRED,
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700121 .flags = MPOL_F_LOCAL,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122};
123
Mel Gorman5606e382012-11-02 18:19:13 +0000124static struct mempolicy preferred_node_policy[MAX_NUMNODES];
125
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -0700126struct mempolicy *get_task_policy(struct task_struct *p)
Mel Gorman5606e382012-11-02 18:19:13 +0000127{
128 struct mempolicy *pol = p->mempolicy;
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700129 int node;
Mel Gorman5606e382012-11-02 18:19:13 +0000130
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700131 if (pol)
132 return pol;
Mel Gorman5606e382012-11-02 18:19:13 +0000133
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700134 node = numa_node_id();
135 if (node != NUMA_NO_NODE) {
136 pol = &preferred_node_policy[node];
137 /* preferred_node_policy is not initialised early in boot */
138 if (pol->mode)
139 return pol;
Mel Gorman5606e382012-11-02 18:19:13 +0000140 }
141
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700142 return &default_policy;
Mel Gorman5606e382012-11-02 18:19:13 +0000143}
144
David Rientjes37012942008-04-28 02:12:33 -0700145static const struct mempolicy_operations {
146 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
Miao Xie708c1bb2010-05-24 14:32:07 -0700147 /*
148 * If read-side task has no lock to protect task->mempolicy, write-side
149 * task will rebind the task->mempolicy by two step. The first step is
150 * setting all the newly nodes, and the second step is cleaning all the
151 * disallowed nodes. In this way, we can avoid finding no node to alloc
152 * page.
153 * If we have a lock to protect task->mempolicy in read-side, we do
154 * rebind directly.
155 *
156 * step:
157 * MPOL_REBIND_ONCE - do rebind work at once
158 * MPOL_REBIND_STEP1 - set all the newly nodes
159 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
160 */
161 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
162 enum mpol_rebind_step step);
David Rientjes37012942008-04-28 02:12:33 -0700163} mpol_ops[MPOL_MAX];
164
David Rientjesf5b087b2008-04-28 02:12:27 -0700165static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
166{
Bob Liu6d556292010-05-24 14:31:59 -0700167 return pol->flags & MPOL_MODE_FLAGS;
David Rientjes4c50bc02008-04-28 02:12:30 -0700168}
169
170static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
171 const nodemask_t *rel)
172{
173 nodemask_t tmp;
174 nodes_fold(tmp, *orig, nodes_weight(*rel));
175 nodes_onto(*ret, tmp, *rel);
David Rientjesf5b087b2008-04-28 02:12:27 -0700176}
177
David Rientjes37012942008-04-28 02:12:33 -0700178static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
179{
180 if (nodes_empty(*nodes))
181 return -EINVAL;
182 pol->v.nodes = *nodes;
183 return 0;
184}
185
186static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
187{
188 if (!nodes)
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700189 pol->flags |= MPOL_F_LOCAL; /* local allocation */
David Rientjes37012942008-04-28 02:12:33 -0700190 else if (nodes_empty(*nodes))
191 return -EINVAL; /* no allowed nodes */
192 else
193 pol->v.preferred_node = first_node(*nodes);
194 return 0;
195}
196
197static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
198{
Zhihui Zhang859f7ef2014-12-18 16:17:09 -0800199 if (nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700200 return -EINVAL;
201 pol->v.nodes = *nodes;
202 return 0;
203}
204
Miao Xie58568d22009-06-16 15:31:49 -0700205/*
206 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
207 * any, for the new policy. mpol_new() has already validated the nodes
208 * parameter with respect to the policy mode and flags. But, we need to
209 * handle an empty nodemask with MPOL_PREFERRED here.
210 *
211 * Must be called holding task's alloc_lock to protect task's mems_allowed
212 * and mempolicy. May also be called holding the mmap_semaphore for write.
213 */
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700214static int mpol_set_nodemask(struct mempolicy *pol,
215 const nodemask_t *nodes, struct nodemask_scratch *nsc)
Miao Xie58568d22009-06-16 15:31:49 -0700216{
Miao Xie58568d22009-06-16 15:31:49 -0700217 int ret;
218
219 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
220 if (pol == NULL)
221 return 0;
Lai Jiangshan01f13bd2012-12-12 13:51:33 -0800222 /* Check N_MEMORY */
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700223 nodes_and(nsc->mask1,
Lai Jiangshan01f13bd2012-12-12 13:51:33 -0800224 cpuset_current_mems_allowed, node_states[N_MEMORY]);
Miao Xie58568d22009-06-16 15:31:49 -0700225
226 VM_BUG_ON(!nodes);
227 if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
228 nodes = NULL; /* explicit local allocation */
229 else {
230 if (pol->flags & MPOL_F_RELATIVE_NODES)
Zhihui Zhang859f7ef2014-12-18 16:17:09 -0800231 mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
Miao Xie58568d22009-06-16 15:31:49 -0700232 else
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700233 nodes_and(nsc->mask2, *nodes, nsc->mask1);
234
Miao Xie58568d22009-06-16 15:31:49 -0700235 if (mpol_store_user_nodemask(pol))
236 pol->w.user_nodemask = *nodes;
237 else
238 pol->w.cpuset_mems_allowed =
239 cpuset_current_mems_allowed;
240 }
241
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700242 if (nodes)
243 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
244 else
245 ret = mpol_ops[pol->mode].create(pol, NULL);
Miao Xie58568d22009-06-16 15:31:49 -0700246 return ret;
247}
248
249/*
250 * This function just creates a new policy, does some check and simple
251 * initialization. You must invoke mpol_set_nodemask() to set nodes.
252 */
David Rientjes028fec42008-04-28 02:12:25 -0700253static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
254 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255{
256 struct mempolicy *policy;
257
David Rientjes028fec42008-04-28 02:12:25 -0700258 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
David Rientjes00ef2d22013-02-22 16:35:36 -0800259 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
Paul Mundt140d5a42007-07-15 23:38:16 -0700260
David Rientjes3e1f0642008-04-28 02:12:34 -0700261 if (mode == MPOL_DEFAULT) {
262 if (nodes && !nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700263 return ERR_PTR(-EINVAL);
Lee Schermerhornd3a71032012-10-25 14:16:29 +0200264 return NULL;
David Rientjes37012942008-04-28 02:12:33 -0700265 }
David Rientjes3e1f0642008-04-28 02:12:34 -0700266 VM_BUG_ON(!nodes);
267
268 /*
269 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
270 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
271 * All other modes require a valid pointer to a non-empty nodemask.
272 */
273 if (mode == MPOL_PREFERRED) {
274 if (nodes_empty(*nodes)) {
275 if (((flags & MPOL_F_STATIC_NODES) ||
276 (flags & MPOL_F_RELATIVE_NODES)))
277 return ERR_PTR(-EINVAL);
David Rientjes3e1f0642008-04-28 02:12:34 -0700278 }
Peter Zijlstra479e2802012-10-25 14:16:28 +0200279 } else if (mode == MPOL_LOCAL) {
280 if (!nodes_empty(*nodes))
281 return ERR_PTR(-EINVAL);
282 mode = MPOL_PREFERRED;
David Rientjes3e1f0642008-04-28 02:12:34 -0700283 } else if (nodes_empty(*nodes))
284 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
286 if (!policy)
287 return ERR_PTR(-ENOMEM);
288 atomic_set(&policy->refcnt, 1);
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700289 policy->mode = mode;
David Rientjes3e1f0642008-04-28 02:12:34 -0700290 policy->flags = flags;
David Rientjesf5b087b2008-04-28 02:12:27 -0700291
David Rientjes37012942008-04-28 02:12:33 -0700292 return policy;
293}
294
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700295/* Slow path of a mpol destructor. */
296void __mpol_put(struct mempolicy *p)
297{
298 if (!atomic_dec_and_test(&p->refcnt))
299 return;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700300 kmem_cache_free(policy_cache, p);
301}
302
Miao Xie708c1bb2010-05-24 14:32:07 -0700303static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
304 enum mpol_rebind_step step)
David Rientjes37012942008-04-28 02:12:33 -0700305{
306}
307
Miao Xie708c1bb2010-05-24 14:32:07 -0700308/*
309 * step:
310 * MPOL_REBIND_ONCE - do rebind work at once
311 * MPOL_REBIND_STEP1 - set all the newly nodes
312 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
313 */
314static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
315 enum mpol_rebind_step step)
David Rientjes37012942008-04-28 02:12:33 -0700316{
317 nodemask_t tmp;
318
319 if (pol->flags & MPOL_F_STATIC_NODES)
320 nodes_and(tmp, pol->w.user_nodemask, *nodes);
321 else if (pol->flags & MPOL_F_RELATIVE_NODES)
322 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
323 else {
Miao Xie708c1bb2010-05-24 14:32:07 -0700324 /*
325 * if step == 1, we use ->w.cpuset_mems_allowed to cache the
326 * result
327 */
328 if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
329 nodes_remap(tmp, pol->v.nodes,
330 pol->w.cpuset_mems_allowed, *nodes);
331 pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
332 } else if (step == MPOL_REBIND_STEP2) {
333 tmp = pol->w.cpuset_mems_allowed;
334 pol->w.cpuset_mems_allowed = *nodes;
335 } else
336 BUG();
David Rientjes37012942008-04-28 02:12:33 -0700337 }
338
Miao Xie708c1bb2010-05-24 14:32:07 -0700339 if (nodes_empty(tmp))
340 tmp = *nodes;
341
342 if (step == MPOL_REBIND_STEP1)
343 nodes_or(pol->v.nodes, pol->v.nodes, tmp);
344 else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
345 pol->v.nodes = tmp;
346 else
347 BUG();
348
David Rientjes37012942008-04-28 02:12:33 -0700349 if (!node_isset(current->il_next, tmp)) {
350 current->il_next = next_node(current->il_next, tmp);
351 if (current->il_next >= MAX_NUMNODES)
352 current->il_next = first_node(tmp);
353 if (current->il_next >= MAX_NUMNODES)
354 current->il_next = numa_node_id();
355 }
356}
357
358static void mpol_rebind_preferred(struct mempolicy *pol,
Miao Xie708c1bb2010-05-24 14:32:07 -0700359 const nodemask_t *nodes,
360 enum mpol_rebind_step step)
David Rientjes37012942008-04-28 02:12:33 -0700361{
362 nodemask_t tmp;
363
David Rientjes37012942008-04-28 02:12:33 -0700364 if (pol->flags & MPOL_F_STATIC_NODES) {
365 int node = first_node(pol->w.user_nodemask);
366
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700367 if (node_isset(node, *nodes)) {
David Rientjes37012942008-04-28 02:12:33 -0700368 pol->v.preferred_node = node;
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700369 pol->flags &= ~MPOL_F_LOCAL;
370 } else
371 pol->flags |= MPOL_F_LOCAL;
David Rientjes37012942008-04-28 02:12:33 -0700372 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
373 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
374 pol->v.preferred_node = first_node(tmp);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700375 } else if (!(pol->flags & MPOL_F_LOCAL)) {
David Rientjes37012942008-04-28 02:12:33 -0700376 pol->v.preferred_node = node_remap(pol->v.preferred_node,
377 pol->w.cpuset_mems_allowed,
378 *nodes);
379 pol->w.cpuset_mems_allowed = *nodes;
380 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381}
382
Miao Xie708c1bb2010-05-24 14:32:07 -0700383/*
384 * mpol_rebind_policy - Migrate a policy to a different set of nodes
385 *
386 * If read-side task has no lock to protect task->mempolicy, write-side
387 * task will rebind the task->mempolicy by two step. The first step is
388 * setting all the newly nodes, and the second step is cleaning all the
389 * disallowed nodes. In this way, we can avoid finding no node to alloc
390 * page.
391 * If we have a lock to protect task->mempolicy in read-side, we do
392 * rebind directly.
393 *
394 * step:
395 * MPOL_REBIND_ONCE - do rebind work at once
396 * MPOL_REBIND_STEP1 - set all the newly nodes
397 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
398 */
399static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
400 enum mpol_rebind_step step)
David Rientjes1d0d2682008-04-28 02:12:32 -0700401{
David Rientjes1d0d2682008-04-28 02:12:32 -0700402 if (!pol)
403 return;
Wang Sheng-Hui89c522c2012-05-29 15:06:16 -0700404 if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
David Rientjes1d0d2682008-04-28 02:12:32 -0700405 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
406 return;
Miao Xie708c1bb2010-05-24 14:32:07 -0700407
408 if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
409 return;
410
411 if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
412 BUG();
413
414 if (step == MPOL_REBIND_STEP1)
415 pol->flags |= MPOL_F_REBINDING;
416 else if (step == MPOL_REBIND_STEP2)
417 pol->flags &= ~MPOL_F_REBINDING;
418 else if (step >= MPOL_REBIND_NSTEP)
419 BUG();
420
421 mpol_ops[pol->mode].rebind(pol, newmask, step);
David Rientjes1d0d2682008-04-28 02:12:32 -0700422}
423
424/*
425 * Wrapper for mpol_rebind_policy() that just requires task
426 * pointer, and updates task mempolicy.
Miao Xie58568d22009-06-16 15:31:49 -0700427 *
428 * Called with task's alloc_lock held.
David Rientjes1d0d2682008-04-28 02:12:32 -0700429 */
430
Miao Xie708c1bb2010-05-24 14:32:07 -0700431void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
432 enum mpol_rebind_step step)
David Rientjes1d0d2682008-04-28 02:12:32 -0700433{
Miao Xie708c1bb2010-05-24 14:32:07 -0700434 mpol_rebind_policy(tsk->mempolicy, new, step);
David Rientjes1d0d2682008-04-28 02:12:32 -0700435}
436
437/*
438 * Rebind each vma in mm to new nodemask.
439 *
440 * Call holding a reference to mm. Takes mm->mmap_sem during call.
441 */
442
443void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
444{
445 struct vm_area_struct *vma;
446
447 down_write(&mm->mmap_sem);
448 for (vma = mm->mmap; vma; vma = vma->vm_next)
Miao Xie708c1bb2010-05-24 14:32:07 -0700449 mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
David Rientjes1d0d2682008-04-28 02:12:32 -0700450 up_write(&mm->mmap_sem);
451}
452
David Rientjes37012942008-04-28 02:12:33 -0700453static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
454 [MPOL_DEFAULT] = {
455 .rebind = mpol_rebind_default,
456 },
457 [MPOL_INTERLEAVE] = {
458 .create = mpol_new_interleave,
459 .rebind = mpol_rebind_nodemask,
460 },
461 [MPOL_PREFERRED] = {
462 .create = mpol_new_preferred,
463 .rebind = mpol_rebind_preferred,
464 },
465 [MPOL_BIND] = {
466 .create = mpol_new_bind,
467 .rebind = mpol_rebind_nodemask,
468 },
469};
470
Christoph Lameterfc301282006-01-18 17:42:29 -0800471static void migrate_page_add(struct page *page, struct list_head *pagelist,
472 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800473
Naoya Horiguchi98094942013-09-11 14:22:14 -0700474/*
475 * Scan through pages checking if pages follow certain conditions,
476 * and move them to the pagelist if they do.
477 */
478static int queue_pages_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800479 unsigned long addr, unsigned long end,
480 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800481 void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482{
Hugh Dickins91612e02005-06-21 17:15:07 -0700483 pte_t *orig_pte;
484 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700485 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700486
Hugh Dickins705e87c2005-10-29 18:16:27 -0700487 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700488 do {
Linus Torvalds6aab3412005-11-28 14:34:23 -0800489 struct page *page;
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800490 int nid;
Hugh Dickins91612e02005-06-21 17:15:07 -0700491
492 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800494 page = vm_normal_page(vma, addr, *pte);
495 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800497 /*
Hugh Dickins62b61f62009-12-14 17:59:33 -0800498 * vm_normal_page() filters out zero pages, but there might
499 * still be PageReserved pages to skip, perhaps in a VDSO.
Nick Piggin053837f2006-01-18 17:42:27 -0800500 */
Hugh Dickinsb79bc0a2013-02-22 16:35:13 -0800501 if (PageReserved(page))
Christoph Lameterf4598c82006-01-12 01:05:20 -0800502 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800503 nid = page_to_nid(page);
Christoph Lameter38e35862006-01-08 01:01:01 -0800504 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
505 continue;
506
Stephen Wilsonb1f72d12011-05-24 17:12:43 -0700507 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
Christoph Lameterfc301282006-01-18 17:42:29 -0800508 migrate_page_add(page, private, flags);
Christoph Lameter38e35862006-01-08 01:01:01 -0800509 else
510 break;
Hugh Dickins91612e02005-06-21 17:15:07 -0700511 } while (pte++, addr += PAGE_SIZE, addr != end);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700512 pte_unmap_unlock(orig_pte, ptl);
Hugh Dickins91612e02005-06-21 17:15:07 -0700513 return addr != end;
514}
515
Naoya Horiguchi98094942013-09-11 14:22:14 -0700516static void queue_pages_hugetlb_pmd_range(struct vm_area_struct *vma,
517 pmd_t *pmd, const nodemask_t *nodes, unsigned long flags,
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700518 void *private)
519{
520#ifdef CONFIG_HUGETLB_PAGE
521 int nid;
522 struct page *page;
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800523 spinlock_t *ptl;
Naoya Horiguchid4c54912014-06-06 10:00:01 -0400524 pte_t entry;
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700525
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800526 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, (pte_t *)pmd);
Naoya Horiguchid4c54912014-06-06 10:00:01 -0400527 entry = huge_ptep_get((pte_t *)pmd);
528 if (!pte_present(entry))
529 goto unlock;
530 page = pte_page(entry);
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700531 nid = page_to_nid(page);
532 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
533 goto unlock;
534 /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
535 if (flags & (MPOL_MF_MOVE_ALL) ||
536 (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
537 isolate_huge_page(page, private);
538unlock:
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800539 spin_unlock(ptl);
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700540#else
541 BUG();
542#endif
543}
544
Naoya Horiguchi98094942013-09-11 14:22:14 -0700545static inline int queue_pages_pmd_range(struct vm_area_struct *vma, pud_t *pud,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800546 unsigned long addr, unsigned long end,
547 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800548 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700549{
550 pmd_t *pmd;
551 unsigned long next;
552
553 pmd = pmd_offset(pud, addr);
554 do {
555 next = pmd_addr_end(addr, end);
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700556 if (!pmd_present(*pmd))
557 continue;
558 if (pmd_huge(*pmd) && is_vm_hugetlb_page(vma)) {
Naoya Horiguchi98094942013-09-11 14:22:14 -0700559 queue_pages_hugetlb_pmd_range(vma, pmd, nodes,
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700560 flags, private);
561 continue;
562 }
Kirill A. Shutemove1803772012-12-12 13:50:59 -0800563 split_huge_page_pmd(vma, addr, pmd);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -0700564 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
Hugh Dickins91612e02005-06-21 17:15:07 -0700565 continue;
Naoya Horiguchi98094942013-09-11 14:22:14 -0700566 if (queue_pages_pte_range(vma, pmd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800567 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700568 return -EIO;
569 } while (pmd++, addr = next, addr != end);
570 return 0;
571}
572
Naoya Horiguchi98094942013-09-11 14:22:14 -0700573static inline int queue_pages_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800574 unsigned long addr, unsigned long end,
575 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800576 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700577{
578 pud_t *pud;
579 unsigned long next;
580
581 pud = pud_offset(pgd, addr);
582 do {
583 next = pud_addr_end(addr, end);
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700584 if (pud_huge(*pud) && is_vm_hugetlb_page(vma))
585 continue;
Hugh Dickins91612e02005-06-21 17:15:07 -0700586 if (pud_none_or_clear_bad(pud))
587 continue;
Naoya Horiguchi98094942013-09-11 14:22:14 -0700588 if (queue_pages_pmd_range(vma, pud, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800589 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700590 return -EIO;
591 } while (pud++, addr = next, addr != end);
592 return 0;
593}
594
Naoya Horiguchi98094942013-09-11 14:22:14 -0700595static inline int queue_pages_pgd_range(struct vm_area_struct *vma,
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800596 unsigned long addr, unsigned long end,
597 const nodemask_t *nodes, unsigned long flags,
Christoph Lameter38e35862006-01-08 01:01:01 -0800598 void *private)
Hugh Dickins91612e02005-06-21 17:15:07 -0700599{
600 pgd_t *pgd;
601 unsigned long next;
602
Nick Pigginb5810032005-10-29 18:16:12 -0700603 pgd = pgd_offset(vma->vm_mm, addr);
Hugh Dickins91612e02005-06-21 17:15:07 -0700604 do {
605 next = pgd_addr_end(addr, end);
606 if (pgd_none_or_clear_bad(pgd))
607 continue;
Naoya Horiguchi98094942013-09-11 14:22:14 -0700608 if (queue_pages_pud_range(vma, pgd, addr, next, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800609 flags, private))
Hugh Dickins91612e02005-06-21 17:15:07 -0700610 return -EIO;
611 } while (pgd++, addr = next, addr != end);
612 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613}
614
Aneesh Kumar K.V58772312013-12-06 00:08:22 +0530615#ifdef CONFIG_NUMA_BALANCING
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200616/*
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200617 * This is used to mark a range of virtual addresses to be inaccessible.
618 * These are later cleared by a NUMA hinting fault. Depending on these
619 * faults, pages may be migrated for better NUMA placement.
620 *
621 * This is assuming that NUMA faults are handled using PROT_NONE. If
622 * an architecture makes a different choice, it will need further
623 * changes to the core.
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200624 */
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200625unsigned long change_prot_numa(struct vm_area_struct *vma,
626 unsigned long addr, unsigned long end)
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200627{
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200628 int nr_updated;
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200629
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200630 nr_updated = change_protection(vma, addr, end, vma->vm_page_prot, 0, 1);
Mel Gorman03c5a6e2012-11-02 14:52:48 +0000631 if (nr_updated)
632 count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200633
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200634 return nr_updated;
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200635}
636#else
637static unsigned long change_prot_numa(struct vm_area_struct *vma,
638 unsigned long addr, unsigned long end)
639{
640 return 0;
641}
Aneesh Kumar K.V58772312013-12-06 00:08:22 +0530642#endif /* CONFIG_NUMA_BALANCING */
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200643
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800644/*
Naoya Horiguchi98094942013-09-11 14:22:14 -0700645 * Walk through page tables and collect pages to be migrated.
646 *
647 * If pages found in a given range are on a set of nodes (determined by
648 * @nodes and @flags,) it's isolated and queued to the pagelist which is
649 * passed via @private.)
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800650 */
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700651static int
Naoya Horiguchi98094942013-09-11 14:22:14 -0700652queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Christoph Lameter38e35862006-01-08 01:01:01 -0800653 const nodemask_t *nodes, unsigned long flags, void *private)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654{
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700655 int err = 0;
656 struct vm_area_struct *vma, *prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700658 vma = find_vma(mm, start);
659 if (!vma)
660 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 prev = NULL;
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700662 for (; vma && vma->vm_start < end; vma = vma->vm_next) {
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200663 unsigned long endvma = vma->vm_end;
664
665 if (endvma > end)
666 endvma = end;
667 if (vma->vm_start > start)
668 start = vma->vm_start;
669
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800670 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
671 if (!vma->vm_next && vma->vm_end < end)
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700672 return -EFAULT;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800673 if (prev && prev->vm_end < vma->vm_start)
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700674 return -EFAULT;
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800675 }
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800676
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200677 if (flags & MPOL_MF_LAZY) {
Mel Gorman2c0346a2014-10-09 15:29:36 -0700678 /* Similar to task_numa_work, skip inaccessible VMAs */
679 if (vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))
680 change_prot_numa(vma, start, endvma);
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200681 goto next;
682 }
683
684 if ((flags & MPOL_MF_STRICT) ||
685 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
686 vma_migratable(vma))) {
687
Naoya Horiguchi98094942013-09-11 14:22:14 -0700688 err = queue_pages_pgd_range(vma, start, endvma, nodes,
Christoph Lameter38e35862006-01-08 01:01:01 -0800689 flags, private);
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700690 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 }
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200693next:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 prev = vma;
695 }
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700696 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697}
698
KOSAKI Motohiro869833f22012-10-08 16:29:16 -0700699/*
700 * Apply policy to a single VMA
701 * This must be called with the mmap_sem held for writing.
702 */
703static int vma_replace_policy(struct vm_area_struct *vma,
704 struct mempolicy *pol)
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700705{
KOSAKI Motohiro869833f22012-10-08 16:29:16 -0700706 int err;
707 struct mempolicy *old;
708 struct mempolicy *new;
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700709
710 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
711 vma->vm_start, vma->vm_end, vma->vm_pgoff,
712 vma->vm_ops, vma->vm_file,
713 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
714
KOSAKI Motohiro869833f22012-10-08 16:29:16 -0700715 new = mpol_dup(pol);
716 if (IS_ERR(new))
717 return PTR_ERR(new);
718
719 if (vma->vm_ops && vma->vm_ops->set_policy) {
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700720 err = vma->vm_ops->set_policy(vma, new);
KOSAKI Motohiro869833f22012-10-08 16:29:16 -0700721 if (err)
722 goto err_out;
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700723 }
KOSAKI Motohiro869833f22012-10-08 16:29:16 -0700724
725 old = vma->vm_policy;
726 vma->vm_policy = new; /* protected by mmap_sem */
727 mpol_put(old);
728
729 return 0;
730 err_out:
731 mpol_put(new);
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700732 return err;
733}
734
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735/* Step 2: apply policy to a range and do splits. */
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800736static int mbind_range(struct mm_struct *mm, unsigned long start,
737 unsigned long end, struct mempolicy *new_pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738{
739 struct vm_area_struct *next;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800740 struct vm_area_struct *prev;
741 struct vm_area_struct *vma;
742 int err = 0;
KOSAKI Motohiroe26a5112011-12-28 15:57:11 -0800743 pgoff_t pgoff;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800744 unsigned long vmstart;
745 unsigned long vmend;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
Linus Torvalds097d5912012-03-06 18:23:36 -0800747 vma = find_vma(mm, start);
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800748 if (!vma || vma->vm_start > start)
749 return -EFAULT;
750
Linus Torvalds097d5912012-03-06 18:23:36 -0800751 prev = vma->vm_prev;
KOSAKI Motohiroe26a5112011-12-28 15:57:11 -0800752 if (start > vma->vm_start)
753 prev = vma;
754
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800755 for (; vma && vma->vm_start < end; prev = vma, vma = next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 next = vma->vm_next;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800757 vmstart = max(start, vma->vm_start);
758 vmend = min(end, vma->vm_end);
759
KOSAKI Motohiroe26a5112011-12-28 15:57:11 -0800760 if (mpol_equal(vma_policy(vma), new_pol))
761 continue;
762
763 pgoff = vma->vm_pgoff +
764 ((vmstart - vma->vm_start) >> PAGE_SHIFT);
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800765 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
KOSAKI Motohiroe26a5112011-12-28 15:57:11 -0800766 vma->anon_vma, vma->vm_file, pgoff,
Caspar Zhang8aacc9f2011-09-14 16:20:58 -0700767 new_pol);
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800768 if (prev) {
769 vma = prev;
770 next = vma->vm_next;
Oleg Nesterov3964acd2013-07-31 13:53:28 -0700771 if (mpol_equal(vma_policy(vma), new_pol))
772 continue;
773 /* vma_merge() joined vma && vma->next, case 8 */
774 goto replace;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800775 }
776 if (vma->vm_start != vmstart) {
777 err = split_vma(vma->vm_mm, vma, vmstart, 1);
778 if (err)
779 goto out;
780 }
781 if (vma->vm_end != vmend) {
782 err = split_vma(vma->vm_mm, vma, vmend, 0);
783 if (err)
784 goto out;
785 }
Oleg Nesterov3964acd2013-07-31 13:53:28 -0700786 replace:
KOSAKI Motohiro869833f22012-10-08 16:29:16 -0700787 err = vma_replace_policy(vma, new_pol);
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700788 if (err)
789 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 }
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800791
792 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793 return err;
794}
795
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796/* Set the process memory policy */
David Rientjes028fec42008-04-28 02:12:25 -0700797static long do_set_mempolicy(unsigned short mode, unsigned short flags,
798 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799{
Miao Xie58568d22009-06-16 15:31:49 -0700800 struct mempolicy *new, *old;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700801 NODEMASK_SCRATCH(scratch);
Miao Xie58568d22009-06-16 15:31:49 -0700802 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700804 if (!scratch)
805 return -ENOMEM;
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700806
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700807 new = mpol_new(mode, flags, nodes);
808 if (IS_ERR(new)) {
809 ret = PTR_ERR(new);
810 goto out;
811 }
Oleg Nesterov2c7c3a72014-10-09 15:27:55 -0700812
Miao Xie58568d22009-06-16 15:31:49 -0700813 task_lock(current);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700814 ret = mpol_set_nodemask(new, nodes, scratch);
Miao Xie58568d22009-06-16 15:31:49 -0700815 if (ret) {
816 task_unlock(current);
Miao Xie58568d22009-06-16 15:31:49 -0700817 mpol_put(new);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700818 goto out;
Miao Xie58568d22009-06-16 15:31:49 -0700819 }
820 old = current->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 current->mempolicy = new;
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700822 if (new && new->mode == MPOL_INTERLEAVE &&
David Rientjesf5b087b2008-04-28 02:12:27 -0700823 nodes_weight(new->v.nodes))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700824 current->il_next = first_node(new->v.nodes);
Miao Xie58568d22009-06-16 15:31:49 -0700825 task_unlock(current);
Miao Xie58568d22009-06-16 15:31:49 -0700826 mpol_put(old);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700827 ret = 0;
828out:
829 NODEMASK_SCRATCH_FREE(scratch);
830 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831}
832
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700833/*
834 * Return nodemask for policy for get_mempolicy() query
Miao Xie58568d22009-06-16 15:31:49 -0700835 *
836 * Called with task's alloc_lock held
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700837 */
838static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700840 nodes_clear(*nodes);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700841 if (p == &default_policy)
842 return;
843
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700844 switch (p->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -0700845 case MPOL_BIND:
846 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700848 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849 break;
850 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700851 if (!(p->flags & MPOL_F_LOCAL))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700852 node_set(p->v.preferred_node, *nodes);
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700853 /* else return empty node mask for local allocation */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854 break;
855 default:
856 BUG();
857 }
858}
859
860static int lookup_node(struct mm_struct *mm, unsigned long addr)
861{
862 struct page *p;
863 int err;
864
865 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
866 if (err >= 0) {
867 err = page_to_nid(p);
868 put_page(p);
869 }
870 return err;
871}
872
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873/* Retrieve NUMA policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700874static long do_get_mempolicy(int *policy, nodemask_t *nmask,
875 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700877 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 struct mm_struct *mm = current->mm;
879 struct vm_area_struct *vma = NULL;
880 struct mempolicy *pol = current->mempolicy;
881
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700882 if (flags &
883 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 return -EINVAL;
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700885
886 if (flags & MPOL_F_MEMS_ALLOWED) {
887 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
888 return -EINVAL;
889 *policy = 0; /* just so it's initialized */
Miao Xie58568d22009-06-16 15:31:49 -0700890 task_lock(current);
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700891 *nmask = cpuset_current_mems_allowed;
Miao Xie58568d22009-06-16 15:31:49 -0700892 task_unlock(current);
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700893 return 0;
894 }
895
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 if (flags & MPOL_F_ADDR) {
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700897 /*
898 * Do NOT fall back to task policy if the
899 * vma/shared policy at addr is NULL. We
900 * want to return MPOL_DEFAULT in this case.
901 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 down_read(&mm->mmap_sem);
903 vma = find_vma_intersection(mm, addr, addr+1);
904 if (!vma) {
905 up_read(&mm->mmap_sem);
906 return -EFAULT;
907 }
908 if (vma->vm_ops && vma->vm_ops->get_policy)
909 pol = vma->vm_ops->get_policy(vma, addr);
910 else
911 pol = vma->vm_policy;
912 } else if (addr)
913 return -EINVAL;
914
915 if (!pol)
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700916 pol = &default_policy; /* indicates default behavior */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917
918 if (flags & MPOL_F_NODE) {
919 if (flags & MPOL_F_ADDR) {
920 err = lookup_node(mm, addr);
921 if (err < 0)
922 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700923 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 } else if (pol == current->mempolicy &&
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700925 pol->mode == MPOL_INTERLEAVE) {
Christoph Lameter8bccd852005-10-29 18:16:59 -0700926 *policy = current->il_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927 } else {
928 err = -EINVAL;
929 goto out;
930 }
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700931 } else {
932 *policy = pol == &default_policy ? MPOL_DEFAULT :
933 pol->mode;
David Rientjesd79df632008-07-04 12:24:13 -0700934 /*
935 * Internal mempolicy flags must be masked off before exposing
936 * the policy to userspace.
937 */
938 *policy |= (pol->flags & MPOL_MODE_FLAGS);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700939 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940
941 if (vma) {
942 up_read(&current->mm->mmap_sem);
943 vma = NULL;
944 }
945
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 err = 0;
Miao Xie58568d22009-06-16 15:31:49 -0700947 if (nmask) {
Lee Schermerhornc6b6ef82010-03-23 13:35:41 -0700948 if (mpol_store_user_nodemask(pol)) {
949 *nmask = pol->w.user_nodemask;
950 } else {
951 task_lock(current);
952 get_policy_nodemask(pol, nmask);
953 task_unlock(current);
954 }
Miao Xie58568d22009-06-16 15:31:49 -0700955 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956
957 out:
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700958 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 if (vma)
960 up_read(&current->mm->mmap_sem);
961 return err;
962}
963
Christoph Lameterb20a3502006-03-22 00:09:12 -0800964#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700965/*
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800966 * page migration
967 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800968static void migrate_page_add(struct page *page, struct list_head *pagelist,
969 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800970{
971 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800972 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800973 */
Nick Piggin62695a82008-10-18 20:26:09 -0700974 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
975 if (!isolate_lru_page(page)) {
976 list_add_tail(&page->lru, pagelist);
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -0800977 inc_zone_page_state(page, NR_ISOLATED_ANON +
978 page_is_file_cache(page));
Nick Piggin62695a82008-10-18 20:26:09 -0700979 }
980 }
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800981}
982
Christoph Lameter742755a2006-06-23 02:03:55 -0700983static struct page *new_node_page(struct page *page, unsigned long node, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700984{
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700985 if (PageHuge(page))
986 return alloc_huge_page_node(page_hstate(compound_head(page)),
987 node);
988 else
989 return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700990}
991
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800992/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800993 * Migrate pages from one node to a target node.
994 * Returns error or the number of pages not migrated.
995 */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700996static int migrate_to_node(struct mm_struct *mm, int source, int dest,
997 int flags)
Christoph Lameter7e2ab152006-02-01 03:05:40 -0800998{
999 nodemask_t nmask;
1000 LIST_HEAD(pagelist);
1001 int err = 0;
1002
1003 nodes_clear(nmask);
1004 node_set(source, nmask);
1005
Minchan Kim08270802012-10-08 16:33:38 -07001006 /*
1007 * This does not "check" the range but isolates all pages that
1008 * need migration. Between passing in the full user address
1009 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
1010 */
1011 VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
Naoya Horiguchi98094942013-09-11 14:22:14 -07001012 queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001013 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1014
Minchan Kimcf608ac2010-10-26 14:21:29 -07001015 if (!list_empty(&pagelist)) {
David Rientjes68711a72014-06-04 16:08:25 -07001016 err = migrate_pages(&pagelist, new_node_page, NULL, dest,
Hugh Dickins9c620e22013-02-22 16:35:14 -08001017 MIGRATE_SYNC, MR_SYSCALL);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001018 if (err)
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -07001019 putback_movable_pages(&pagelist);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001020 }
Christoph Lameter95a402c2006-06-23 02:03:53 -07001021
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001022 return err;
1023}
1024
1025/*
1026 * Move pages between the two nodesets so as to preserve the physical
1027 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -08001028 *
1029 * Returns the number of page that could not be moved.
1030 */
Andrew Morton0ce72d42012-05-29 15:06:24 -07001031int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1032 const nodemask_t *to, int flags)
Christoph Lameter39743882006-01-08 01:00:51 -08001033{
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001034 int busy = 0;
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001035 int err;
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001036 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -08001037
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001038 err = migrate_prep();
1039 if (err)
1040 return err;
1041
Lee Schermerhorn53f25562008-04-28 02:13:20 -07001042 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -08001043
KOSAKI Motohiroda0aa132010-03-05 13:41:59 -08001044 /*
1045 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1046 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
1047 * bit in 'tmp', and return that <source, dest> pair for migration.
1048 * The pair of nodemasks 'to' and 'from' define the map.
1049 *
1050 * If no pair of bits is found that way, fallback to picking some
1051 * pair of 'source' and 'dest' bits that are not the same. If the
1052 * 'source' and 'dest' bits are the same, this represents a node
1053 * that will be migrating to itself, so no pages need move.
1054 *
1055 * If no bits are left in 'tmp', or if all remaining bits left
1056 * in 'tmp' correspond to the same bit in 'to', return false
1057 * (nothing left to migrate).
1058 *
1059 * This lets us pick a pair of nodes to migrate between, such that
1060 * if possible the dest node is not already occupied by some other
1061 * source node, minimizing the risk of overloading the memory on a
1062 * node that would happen if we migrated incoming memory to a node
1063 * before migrating outgoing memory source that same node.
1064 *
1065 * A single scan of tmp is sufficient. As we go, we remember the
1066 * most recent <s, d> pair that moved (s != d). If we find a pair
1067 * that not only moved, but what's better, moved to an empty slot
1068 * (d is not set in tmp), then we break out then, with that pair.
Justin P. Mattockae0e47f2011-03-01 15:06:02 +01001069 * Otherwise when we finish scanning from_tmp, we at least have the
KOSAKI Motohiroda0aa132010-03-05 13:41:59 -08001070 * most recent <s, d> pair that moved. If we get all the way through
1071 * the scan of tmp without finding any node that moved, much less
1072 * moved to an empty node, then there is nothing left worth migrating.
1073 */
Christoph Lameterd4984712006-01-08 01:00:55 -08001074
Andrew Morton0ce72d42012-05-29 15:06:24 -07001075 tmp = *from;
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001076 while (!nodes_empty(tmp)) {
1077 int s,d;
Jianguo Wub76ac7e2013-11-12 15:07:39 -08001078 int source = NUMA_NO_NODE;
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001079 int dest = 0;
1080
1081 for_each_node_mask(s, tmp) {
Larry Woodman4a5b18c2012-05-29 15:06:24 -07001082
1083 /*
1084 * do_migrate_pages() tries to maintain the relative
1085 * node relationship of the pages established between
1086 * threads and memory areas.
1087 *
1088 * However if the number of source nodes is not equal to
1089 * the number of destination nodes we can not preserve
1090 * this node relative relationship. In that case, skip
1091 * copying memory from a node that is in the destination
1092 * mask.
1093 *
1094 * Example: [2,3,4] -> [3,4,5] moves everything.
1095 * [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1096 */
1097
Andrew Morton0ce72d42012-05-29 15:06:24 -07001098 if ((nodes_weight(*from) != nodes_weight(*to)) &&
1099 (node_isset(s, *to)))
Larry Woodman4a5b18c2012-05-29 15:06:24 -07001100 continue;
1101
Andrew Morton0ce72d42012-05-29 15:06:24 -07001102 d = node_remap(s, *from, *to);
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001103 if (s == d)
1104 continue;
1105
1106 source = s; /* Node moved. Memorize */
1107 dest = d;
1108
1109 /* dest not in remaining from nodes? */
1110 if (!node_isset(dest, tmp))
1111 break;
1112 }
Jianguo Wub76ac7e2013-11-12 15:07:39 -08001113 if (source == NUMA_NO_NODE)
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001114 break;
1115
1116 node_clear(source, tmp);
1117 err = migrate_to_node(mm, source, dest, flags);
1118 if (err > 0)
1119 busy += err;
1120 if (err < 0)
1121 break;
Christoph Lameter39743882006-01-08 01:00:51 -08001122 }
1123 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001124 if (err < 0)
1125 return err;
1126 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -08001127
Christoph Lameter39743882006-01-08 01:00:51 -08001128}
1129
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -08001130/*
1131 * Allocate a new page for page migration based on vma policy.
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001132 * Start by assuming the page is mapped by the same vma as contains @start.
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -08001133 * Search forward from there, if not. N.B., this assumes that the
1134 * list of pages handed to migrate_pages()--which is how we get here--
1135 * is in virtual address order.
1136 */
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001137static struct page *new_page(struct page *page, unsigned long start, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -07001138{
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001139 struct vm_area_struct *vma;
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -08001140 unsigned long uninitialized_var(address);
Christoph Lameter95a402c2006-06-23 02:03:53 -07001141
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001142 vma = find_vma(current->mm, start);
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -08001143 while (vma) {
1144 address = page_address_in_vma(page, vma);
1145 if (address != -EFAULT)
1146 break;
1147 vma = vma->vm_next;
1148 }
1149
Wanpeng Li11c731e2013-12-18 17:08:56 -08001150 if (PageHuge(page)) {
Michal Hockocc817172014-01-23 15:53:15 -08001151 BUG_ON(!vma);
1152 return alloc_huge_page_noerr(vma, address, 1);
Wanpeng Li11c731e2013-12-18 17:08:56 -08001153 }
1154 /*
1155 * if !vma, alloc_page_vma() will use task or system default policy
1156 */
Lee Schermerhorn3ad33b22007-11-14 16:59:10 -08001157 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
Christoph Lameter95a402c2006-06-23 02:03:53 -07001158}
Christoph Lameterb20a3502006-03-22 00:09:12 -08001159#else
1160
1161static void migrate_page_add(struct page *page, struct list_head *pagelist,
1162 unsigned long flags)
1163{
1164}
1165
Andrew Morton0ce72d42012-05-29 15:06:24 -07001166int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1167 const nodemask_t *to, int flags)
Christoph Lameterb20a3502006-03-22 00:09:12 -08001168{
1169 return -ENOSYS;
1170}
Christoph Lameter95a402c2006-06-23 02:03:53 -07001171
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001172static struct page *new_page(struct page *page, unsigned long start, int **x)
Christoph Lameter95a402c2006-06-23 02:03:53 -07001173{
1174 return NULL;
1175}
Christoph Lameterb20a3502006-03-22 00:09:12 -08001176#endif
1177
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001178static long do_mbind(unsigned long start, unsigned long len,
David Rientjes028fec42008-04-28 02:12:25 -07001179 unsigned short mode, unsigned short mode_flags,
1180 nodemask_t *nmask, unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001181{
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001182 struct mm_struct *mm = current->mm;
1183 struct mempolicy *new;
1184 unsigned long end;
1185 int err;
1186 LIST_HEAD(pagelist);
1187
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001188 if (flags & ~(unsigned long)MPOL_MF_VALID)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001189 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -08001190 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001191 return -EPERM;
1192
1193 if (start & ~PAGE_MASK)
1194 return -EINVAL;
1195
1196 if (mode == MPOL_DEFAULT)
1197 flags &= ~MPOL_MF_STRICT;
1198
1199 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1200 end = start + len;
1201
1202 if (end < start)
1203 return -EINVAL;
1204 if (end == start)
1205 return 0;
1206
David Rientjes028fec42008-04-28 02:12:25 -07001207 new = mpol_new(mode, mode_flags, nmask);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001208 if (IS_ERR(new))
1209 return PTR_ERR(new);
1210
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001211 if (flags & MPOL_MF_LAZY)
1212 new->flags |= MPOL_F_MOF;
1213
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001214 /*
1215 * If we are using the default policy then operation
1216 * on discontinuous address spaces is okay after all
1217 */
1218 if (!new)
1219 flags |= MPOL_MF_DISCONTIG_OK;
1220
David Rientjes028fec42008-04-28 02:12:25 -07001221 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1222 start, start + len, mode, mode_flags,
David Rientjes00ef2d22013-02-22 16:35:36 -08001223 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001224
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001225 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1226
1227 err = migrate_prep();
1228 if (err)
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001229 goto mpol_out;
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001230 }
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07001231 {
1232 NODEMASK_SCRATCH(scratch);
1233 if (scratch) {
1234 down_write(&mm->mmap_sem);
1235 task_lock(current);
1236 err = mpol_set_nodemask(new, nmask, scratch);
1237 task_unlock(current);
1238 if (err)
1239 up_write(&mm->mmap_sem);
1240 } else
1241 err = -ENOMEM;
1242 NODEMASK_SCRATCH_FREE(scratch);
1243 }
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001244 if (err)
1245 goto mpol_out;
1246
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001247 err = queue_pages_range(mm, start, end, nmask,
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001248 flags | MPOL_MF_INVERT, &pagelist);
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001249 if (!err)
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -08001250 err = mbind_range(mm, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001251
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001252 if (!err) {
1253 int nr_failed = 0;
1254
Minchan Kimcf608ac2010-10-26 14:21:29 -07001255 if (!list_empty(&pagelist)) {
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001256 WARN_ON_ONCE(flags & MPOL_MF_LAZY);
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001257 nr_failed = migrate_pages(&pagelist, new_page, NULL,
1258 start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001259 if (nr_failed)
Naoya Horiguchi74060e42013-09-11 14:22:06 -07001260 putback_movable_pages(&pagelist);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001261 }
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001262
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001263 if (nr_failed && (flags & MPOL_MF_STRICT))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001264 err = -EIO;
KOSAKI Motohiroab8a3e12009-10-26 16:49:58 -07001265 } else
Joonsoo Kimb0e5fd72013-12-18 17:08:51 -08001266 putback_movable_pages(&pagelist);
Christoph Lameterb20a3502006-03-22 00:09:12 -08001267
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001268 up_write(&mm->mmap_sem);
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001269 mpol_out:
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001270 mpol_put(new);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001271 return err;
1272}
1273
Christoph Lameter39743882006-01-08 01:00:51 -08001274/*
Christoph Lameter8bccd852005-10-29 18:16:59 -07001275 * User space interface with variable sized bitmaps for nodelists.
1276 */
1277
1278/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -08001279static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -07001280 unsigned long maxnode)
1281{
1282 unsigned long k;
1283 unsigned long nlongs;
1284 unsigned long endmask;
1285
1286 --maxnode;
1287 nodes_clear(*nodes);
1288 if (maxnode == 0 || !nmask)
1289 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -08001290 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -08001291 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001292
1293 nlongs = BITS_TO_LONGS(maxnode);
1294 if ((maxnode % BITS_PER_LONG) == 0)
1295 endmask = ~0UL;
1296 else
1297 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1298
1299 /* When the user specified more nodes than supported just check
1300 if the non supported part is all zero. */
1301 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1302 if (nlongs > PAGE_SIZE/sizeof(long))
1303 return -EINVAL;
1304 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1305 unsigned long t;
1306 if (get_user(t, nmask + k))
1307 return -EFAULT;
1308 if (k == nlongs - 1) {
1309 if (t & endmask)
1310 return -EINVAL;
1311 } else if (t)
1312 return -EINVAL;
1313 }
1314 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1315 endmask = ~0UL;
1316 }
1317
1318 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1319 return -EFAULT;
1320 nodes_addr(*nodes)[nlongs-1] &= endmask;
1321 return 0;
1322}
1323
1324/* Copy a kernel node mask to user space */
1325static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1326 nodemask_t *nodes)
1327{
1328 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1329 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1330
1331 if (copy > nbytes) {
1332 if (copy > PAGE_SIZE)
1333 return -EINVAL;
1334 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1335 return -EFAULT;
1336 copy = nbytes;
1337 }
1338 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1339}
1340
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001341SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
Rasmus Villemoesf7f28ca2014-06-04 16:07:57 -07001342 unsigned long, mode, const unsigned long __user *, nmask,
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001343 unsigned long, maxnode, unsigned, flags)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001344{
1345 nodemask_t nodes;
1346 int err;
David Rientjes028fec42008-04-28 02:12:25 -07001347 unsigned short mode_flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001348
David Rientjes028fec42008-04-28 02:12:25 -07001349 mode_flags = mode & MPOL_MODE_FLAGS;
1350 mode &= ~MPOL_MODE_FLAGS;
David Rientjesa3b51e02008-04-28 02:12:23 -07001351 if (mode >= MPOL_MAX)
1352 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001353 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1354 (mode_flags & MPOL_F_RELATIVE_NODES))
1355 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001356 err = get_nodes(&nodes, nmask, maxnode);
1357 if (err)
1358 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001359 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001360}
1361
1362/* Set the process memory policy */
Rasmus Villemoes23c89022014-06-04 16:07:58 -07001363SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001364 unsigned long, maxnode)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001365{
1366 int err;
1367 nodemask_t nodes;
David Rientjes028fec42008-04-28 02:12:25 -07001368 unsigned short flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001369
David Rientjes028fec42008-04-28 02:12:25 -07001370 flags = mode & MPOL_MODE_FLAGS;
1371 mode &= ~MPOL_MODE_FLAGS;
1372 if ((unsigned int)mode >= MPOL_MAX)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001373 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001374 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1375 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001376 err = get_nodes(&nodes, nmask, maxnode);
1377 if (err)
1378 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001379 return do_set_mempolicy(mode, flags, &nodes);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001380}
1381
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001382SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1383 const unsigned long __user *, old_nodes,
1384 const unsigned long __user *, new_nodes)
Christoph Lameter39743882006-01-08 01:00:51 -08001385{
David Howellsc69e8d92008-11-14 10:39:19 +11001386 const struct cred *cred = current_cred(), *tcred;
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001387 struct mm_struct *mm = NULL;
Christoph Lameter39743882006-01-08 01:00:51 -08001388 struct task_struct *task;
Christoph Lameter39743882006-01-08 01:00:51 -08001389 nodemask_t task_nodes;
1390 int err;
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001391 nodemask_t *old;
1392 nodemask_t *new;
1393 NODEMASK_SCRATCH(scratch);
Christoph Lameter39743882006-01-08 01:00:51 -08001394
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001395 if (!scratch)
1396 return -ENOMEM;
Christoph Lameter39743882006-01-08 01:00:51 -08001397
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001398 old = &scratch->mask1;
1399 new = &scratch->mask2;
1400
1401 err = get_nodes(old, old_nodes, maxnode);
Christoph Lameter39743882006-01-08 01:00:51 -08001402 if (err)
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001403 goto out;
1404
1405 err = get_nodes(new, new_nodes, maxnode);
1406 if (err)
1407 goto out;
Christoph Lameter39743882006-01-08 01:00:51 -08001408
1409 /* Find the mm_struct */
Zeng Zhaoming55cfaa32010-12-02 14:31:13 -08001410 rcu_read_lock();
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001411 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter39743882006-01-08 01:00:51 -08001412 if (!task) {
Zeng Zhaoming55cfaa32010-12-02 14:31:13 -08001413 rcu_read_unlock();
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001414 err = -ESRCH;
1415 goto out;
Christoph Lameter39743882006-01-08 01:00:51 -08001416 }
Christoph Lameter3268c632012-03-21 16:34:06 -07001417 get_task_struct(task);
Christoph Lameter39743882006-01-08 01:00:51 -08001418
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001419 err = -EINVAL;
Christoph Lameter39743882006-01-08 01:00:51 -08001420
1421 /*
1422 * Check if this process has the right to modify the specified
1423 * process. The right exists if the process has administrative
Alexey Dobriyan7f927fc2006-03-28 01:56:53 -08001424 * capabilities, superuser privileges or the same
Christoph Lameter39743882006-01-08 01:00:51 -08001425 * userid as the target process.
1426 */
David Howellsc69e8d92008-11-14 10:39:19 +11001427 tcred = __task_cred(task);
Eric W. Biedermanb38a86e2012-03-12 15:48:24 -07001428 if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
1429 !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
Christoph Lameter74c00242006-03-14 19:50:21 -08001430 !capable(CAP_SYS_NICE)) {
David Howellsc69e8d92008-11-14 10:39:19 +11001431 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001432 err = -EPERM;
Christoph Lameter3268c632012-03-21 16:34:06 -07001433 goto out_put;
Christoph Lameter39743882006-01-08 01:00:51 -08001434 }
David Howellsc69e8d92008-11-14 10:39:19 +11001435 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001436
1437 task_nodes = cpuset_mems_allowed(task);
1438 /* Is the user allowed to access the target nodes? */
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001439 if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001440 err = -EPERM;
Christoph Lameter3268c632012-03-21 16:34:06 -07001441 goto out_put;
Christoph Lameter39743882006-01-08 01:00:51 -08001442 }
1443
Lai Jiangshan01f13bd2012-12-12 13:51:33 -08001444 if (!nodes_subset(*new, node_states[N_MEMORY])) {
Christoph Lameter3b42d282007-08-31 00:12:08 -07001445 err = -EINVAL;
Christoph Lameter3268c632012-03-21 16:34:06 -07001446 goto out_put;
Christoph Lameter3b42d282007-08-31 00:12:08 -07001447 }
1448
David Quigley86c3a762006-06-23 02:04:02 -07001449 err = security_task_movememory(task);
1450 if (err)
Christoph Lameter3268c632012-03-21 16:34:06 -07001451 goto out_put;
David Quigley86c3a762006-06-23 02:04:02 -07001452
Christoph Lameter3268c632012-03-21 16:34:06 -07001453 mm = get_task_mm(task);
1454 put_task_struct(task);
Sasha Levinf2a9ef82012-04-25 16:01:52 -07001455
1456 if (!mm) {
Christoph Lameter3268c632012-03-21 16:34:06 -07001457 err = -EINVAL;
Sasha Levinf2a9ef82012-04-25 16:01:52 -07001458 goto out;
1459 }
1460
1461 err = do_migrate_pages(mm, old, new,
1462 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter3268c632012-03-21 16:34:06 -07001463
1464 mmput(mm);
1465out:
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001466 NODEMASK_SCRATCH_FREE(scratch);
1467
Christoph Lameter39743882006-01-08 01:00:51 -08001468 return err;
Christoph Lameter3268c632012-03-21 16:34:06 -07001469
1470out_put:
1471 put_task_struct(task);
1472 goto out;
1473
Christoph Lameter39743882006-01-08 01:00:51 -08001474}
1475
1476
Christoph Lameter8bccd852005-10-29 18:16:59 -07001477/* Retrieve NUMA policy */
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001478SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1479 unsigned long __user *, nmask, unsigned long, maxnode,
1480 unsigned long, addr, unsigned long, flags)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001481{
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001482 int err;
1483 int uninitialized_var(pval);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001484 nodemask_t nodes;
1485
1486 if (nmask != NULL && maxnode < MAX_NUMNODES)
1487 return -EINVAL;
1488
1489 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1490
1491 if (err)
1492 return err;
1493
1494 if (policy && put_user(pval, policy))
1495 return -EFAULT;
1496
1497 if (nmask)
1498 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1499
1500 return err;
1501}
1502
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503#ifdef CONFIG_COMPAT
1504
Heiko Carstensc93e0f62014-03-03 16:32:26 +01001505COMPAT_SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1506 compat_ulong_t __user *, nmask,
1507 compat_ulong_t, maxnode,
1508 compat_ulong_t, addr, compat_ulong_t, flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509{
1510 long err;
1511 unsigned long __user *nm = NULL;
1512 unsigned long nr_bits, alloc_size;
1513 DECLARE_BITMAP(bm, MAX_NUMNODES);
1514
1515 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1516 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1517
1518 if (nmask)
1519 nm = compat_alloc_user_space(alloc_size);
1520
1521 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1522
1523 if (!err && nmask) {
KAMEZAWA Hiroyuki2bbff6c2011-09-14 16:21:02 -07001524 unsigned long copy_size;
1525 copy_size = min_t(unsigned long, sizeof(bm), alloc_size);
1526 err = copy_from_user(bm, nm, copy_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527 /* ensure entire bitmap is zeroed */
1528 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1529 err |= compat_put_bitmap(nmask, bm, nr_bits);
1530 }
1531
1532 return err;
1533}
1534
Heiko Carstensc93e0f62014-03-03 16:32:26 +01001535COMPAT_SYSCALL_DEFINE3(set_mempolicy, int, mode, compat_ulong_t __user *, nmask,
1536 compat_ulong_t, maxnode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537{
1538 long err = 0;
1539 unsigned long __user *nm = NULL;
1540 unsigned long nr_bits, alloc_size;
1541 DECLARE_BITMAP(bm, MAX_NUMNODES);
1542
1543 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1544 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1545
1546 if (nmask) {
1547 err = compat_get_bitmap(bm, nmask, nr_bits);
1548 nm = compat_alloc_user_space(alloc_size);
1549 err |= copy_to_user(nm, bm, alloc_size);
1550 }
1551
1552 if (err)
1553 return -EFAULT;
1554
1555 return sys_set_mempolicy(mode, nm, nr_bits+1);
1556}
1557
Heiko Carstensc93e0f62014-03-03 16:32:26 +01001558COMPAT_SYSCALL_DEFINE6(mbind, compat_ulong_t, start, compat_ulong_t, len,
1559 compat_ulong_t, mode, compat_ulong_t __user *, nmask,
1560 compat_ulong_t, maxnode, compat_ulong_t, flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561{
1562 long err = 0;
1563 unsigned long __user *nm = NULL;
1564 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001565 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566
1567 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1568 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1569
1570 if (nmask) {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001571 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572 nm = compat_alloc_user_space(alloc_size);
Andi Kleendfcd3c02005-10-29 18:15:48 -07001573 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574 }
1575
1576 if (err)
1577 return -EFAULT;
1578
1579 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1580}
1581
1582#endif
1583
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001584struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1585 unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586{
Oleg Nesterov8d902742014-10-09 15:27:45 -07001587 struct mempolicy *pol = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588
1589 if (vma) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001590 if (vma->vm_ops && vma->vm_ops->get_policy) {
Oleg Nesterov8d902742014-10-09 15:27:45 -07001591 pol = vma->vm_ops->get_policy(vma, addr);
Mel Gorman00442ad2012-10-08 16:29:20 -07001592 } else if (vma->vm_policy) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593 pol = vma->vm_policy;
Mel Gorman00442ad2012-10-08 16:29:20 -07001594
1595 /*
1596 * shmem_alloc_page() passes MPOL_F_SHARED policy with
1597 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1598 * count on these policies which will be dropped by
1599 * mpol_cond_put() later
1600 */
1601 if (mpol_needs_cond_ref(pol))
1602 mpol_get(pol);
1603 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604 }
Oleg Nesterovf15ca782014-10-09 15:27:43 -07001605
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001606 return pol;
1607}
1608
1609/*
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001610 * get_vma_policy(@vma, @addr)
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001611 * @vma: virtual memory area whose policy is sought
1612 * @addr: address in @vma for shared policy lookup
1613 *
1614 * Returns effective policy for a VMA at specified address.
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001615 * Falls back to current->mempolicy or system default policy, as necessary.
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001616 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1617 * count--added by the get_policy() vm_op, as appropriate--to protect against
1618 * freeing by another task. It is the caller's responsibility to free the
1619 * extra reference for shared policies.
1620 */
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001621static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
1622 unsigned long addr)
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001623{
1624 struct mempolicy *pol = __get_vma_policy(vma, addr);
1625
Oleg Nesterov8d902742014-10-09 15:27:45 -07001626 if (!pol)
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001627 pol = get_task_policy(current);
Oleg Nesterov8d902742014-10-09 15:27:45 -07001628
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 return pol;
1630}
1631
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001632bool vma_policy_mof(struct vm_area_struct *vma)
Mel Gormanfc3147242013-10-07 11:29:09 +01001633{
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001634 struct mempolicy *pol;
Oleg Nesterovf15ca782014-10-09 15:27:43 -07001635
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001636 if (vma->vm_ops && vma->vm_ops->get_policy) {
1637 bool ret = false;
Mel Gormanfc3147242013-10-07 11:29:09 +01001638
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001639 pol = vma->vm_ops->get_policy(vma, vma->vm_start);
1640 if (pol && (pol->flags & MPOL_F_MOF))
1641 ret = true;
1642 mpol_cond_put(pol);
Mel Gormanfc3147242013-10-07 11:29:09 +01001643
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001644 return ret;
Mel Gormanfc3147242013-10-07 11:29:09 +01001645 }
1646
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001647 pol = vma->vm_policy;
Oleg Nesterov8d902742014-10-09 15:27:45 -07001648 if (!pol)
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001649 pol = get_task_policy(current);
Oleg Nesterov8d902742014-10-09 15:27:45 -07001650
Mel Gormanfc3147242013-10-07 11:29:09 +01001651 return pol->flags & MPOL_F_MOF;
1652}
1653
Lai Jiangshand3eb1572013-02-22 16:33:22 -08001654static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1655{
1656 enum zone_type dynamic_policy_zone = policy_zone;
1657
1658 BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1659
1660 /*
1661 * if policy->v.nodes has movable memory only,
1662 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1663 *
1664 * policy->v.nodes is intersect with node_states[N_MEMORY].
1665 * so if the following test faile, it implies
1666 * policy->v.nodes has movable memory only.
1667 */
1668 if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY]))
1669 dynamic_policy_zone = ZONE_MOVABLE;
1670
1671 return zone >= dynamic_policy_zone;
1672}
1673
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001674/*
1675 * Return a nodemask representing a mempolicy for filtering nodes for
1676 * page allocation
1677 */
1678static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
Mel Gorman19770b32008-04-28 02:12:18 -07001679{
1680 /* Lower zones don't get a nodemask applied for MPOL_BIND */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001681 if (unlikely(policy->mode == MPOL_BIND) &&
Lai Jiangshand3eb1572013-02-22 16:33:22 -08001682 apply_policy_zone(policy, gfp_zone(gfp)) &&
Mel Gorman19770b32008-04-28 02:12:18 -07001683 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1684 return &policy->v.nodes;
1685
1686 return NULL;
1687}
1688
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001689/* Return a zonelist indicated by gfp for node representing a mempolicy */
Andi Kleen2f5f9482011-03-04 17:36:29 -08001690static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
1691 int nd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001692{
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001693 switch (policy->mode) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001695 if (!(policy->flags & MPOL_F_LOCAL))
1696 nd = policy->v.preferred_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697 break;
1698 case MPOL_BIND:
Mel Gorman19770b32008-04-28 02:12:18 -07001699 /*
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001700 * Normally, MPOL_BIND allocations are node-local within the
1701 * allowed nodemask. However, if __GFP_THISNODE is set and the
Bob Liu6eb27e12010-05-24 14:32:00 -07001702 * current node isn't part of the mask, we use the zonelist for
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001703 * the first node in the mask instead.
Mel Gorman19770b32008-04-28 02:12:18 -07001704 */
Mel Gorman19770b32008-04-28 02:12:18 -07001705 if (unlikely(gfp & __GFP_THISNODE) &&
1706 unlikely(!node_isset(nd, policy->v.nodes)))
1707 nd = first_node(policy->v.nodes);
1708 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001709 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001710 BUG();
1711 }
Mel Gorman0e884602008-04-28 02:12:14 -07001712 return node_zonelist(nd, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001713}
1714
1715/* Do dynamic interleaving for a process */
1716static unsigned interleave_nodes(struct mempolicy *policy)
1717{
1718 unsigned nid, next;
1719 struct task_struct *me = current;
1720
1721 nid = me->il_next;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001722 next = next_node(nid, policy->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001723 if (next >= MAX_NUMNODES)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001724 next = first_node(policy->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001725 if (next < MAX_NUMNODES)
1726 me->il_next = next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727 return nid;
1728}
1729
Christoph Lameterdc85da12006-01-18 17:42:36 -08001730/*
1731 * Depending on the memory policy provide a node from which to allocate the
1732 * next slab entry.
1733 */
David Rientjes2a389612014-04-07 15:37:29 -07001734unsigned int mempolicy_slab_node(void)
Christoph Lameterdc85da12006-01-18 17:42:36 -08001735{
Andi Kleene7b691b2012-06-09 02:40:03 -07001736 struct mempolicy *policy;
David Rientjes2a389612014-04-07 15:37:29 -07001737 int node = numa_mem_id();
Andi Kleene7b691b2012-06-09 02:40:03 -07001738
1739 if (in_interrupt())
David Rientjes2a389612014-04-07 15:37:29 -07001740 return node;
Andi Kleene7b691b2012-06-09 02:40:03 -07001741
1742 policy = current->mempolicy;
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001743 if (!policy || policy->flags & MPOL_F_LOCAL)
David Rientjes2a389612014-04-07 15:37:29 -07001744 return node;
Christoph Lameter765c4502006-09-27 01:50:08 -07001745
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001746 switch (policy->mode) {
1747 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001748 /*
1749 * handled MPOL_F_LOCAL above
1750 */
1751 return policy->v.preferred_node;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001752
Christoph Lameterdc85da12006-01-18 17:42:36 -08001753 case MPOL_INTERLEAVE:
1754 return interleave_nodes(policy);
1755
Mel Gormandd1a2392008-04-28 02:12:17 -07001756 case MPOL_BIND: {
Christoph Lameterdc85da12006-01-18 17:42:36 -08001757 /*
1758 * Follow bind policy behavior and start allocation at the
1759 * first node.
1760 */
Mel Gorman19770b32008-04-28 02:12:18 -07001761 struct zonelist *zonelist;
1762 struct zone *zone;
1763 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
David Rientjes2a389612014-04-07 15:37:29 -07001764 zonelist = &NODE_DATA(node)->node_zonelists[0];
Mel Gorman19770b32008-04-28 02:12:18 -07001765 (void)first_zones_zonelist(zonelist, highest_zoneidx,
1766 &policy->v.nodes,
1767 &zone);
David Rientjes2a389612014-04-07 15:37:29 -07001768 return zone ? zone->node : node;
Mel Gormandd1a2392008-04-28 02:12:17 -07001769 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08001770
Christoph Lameterdc85da12006-01-18 17:42:36 -08001771 default:
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001772 BUG();
Christoph Lameterdc85da12006-01-18 17:42:36 -08001773 }
1774}
1775
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776/* Do static interleaving for a VMA with known offset. */
1777static unsigned offset_il_node(struct mempolicy *pol,
1778 struct vm_area_struct *vma, unsigned long off)
1779{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001780 unsigned nnodes = nodes_weight(pol->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001781 unsigned target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782 int c;
Jianguo Wub76ac7e2013-11-12 15:07:39 -08001783 int nid = NUMA_NO_NODE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001784
David Rientjesf5b087b2008-04-28 02:12:27 -07001785 if (!nnodes)
1786 return numa_node_id();
1787 target = (unsigned int)off % nnodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001788 c = 0;
1789 do {
Andi Kleendfcd3c02005-10-29 18:15:48 -07001790 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 c++;
1792 } while (c <= target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793 return nid;
1794}
1795
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001796/* Determine a node number for interleave */
1797static inline unsigned interleave_nid(struct mempolicy *pol,
1798 struct vm_area_struct *vma, unsigned long addr, int shift)
1799{
1800 if (vma) {
1801 unsigned long off;
1802
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001803 /*
1804 * for small pages, there is no difference between
1805 * shift and PAGE_SHIFT, so the bit-shift is safe.
1806 * for huge pages, since vm_pgoff is in units of small
1807 * pages, we need to shift off the always 0 bits to get
1808 * a useful offset.
1809 */
1810 BUG_ON(shift < PAGE_SHIFT);
1811 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001812 off += (addr - vma->vm_start) >> shift;
1813 return offset_il_node(pol, vma, off);
1814 } else
1815 return interleave_nodes(pol);
1816}
1817
Michal Hocko778d3b02011-07-26 16:08:30 -07001818/*
1819 * Return the bit number of a random bit set in the nodemask.
Jianguo Wub76ac7e2013-11-12 15:07:39 -08001820 * (returns NUMA_NO_NODE if nodemask is empty)
Michal Hocko778d3b02011-07-26 16:08:30 -07001821 */
1822int node_random(const nodemask_t *maskp)
1823{
Jianguo Wub76ac7e2013-11-12 15:07:39 -08001824 int w, bit = NUMA_NO_NODE;
Michal Hocko778d3b02011-07-26 16:08:30 -07001825
1826 w = nodes_weight(*maskp);
1827 if (w)
1828 bit = bitmap_ord_to_pos(maskp->bits,
1829 get_random_int() % w, MAX_NUMNODES);
1830 return bit;
1831}
1832
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001833#ifdef CONFIG_HUGETLBFS
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001834/*
1835 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
Fabian Frederickb46e14a2014-06-04 16:08:18 -07001836 * @vma: virtual memory area whose policy is sought
1837 * @addr: address in @vma for shared policy lookup and interleave policy
1838 * @gfp_flags: for requested zone
1839 * @mpol: pointer to mempolicy pointer for reference counted mempolicy
1840 * @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001841 *
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001842 * Returns a zonelist suitable for a huge page allocation and a pointer
1843 * to the struct mempolicy for conditional unref after allocation.
1844 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1845 * @nodemask for filtering the zonelist.
Miao Xiec0ff7452010-05-24 14:32:08 -07001846 *
Mel Gormand26914d2014-04-03 14:47:24 -07001847 * Must be protected by read_mems_allowed_begin()
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001848 */
Mel Gorman396faf02007-07-17 04:03:13 -07001849struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
Mel Gorman19770b32008-04-28 02:12:18 -07001850 gfp_t gfp_flags, struct mempolicy **mpol,
1851 nodemask_t **nodemask)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001852{
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001853 struct zonelist *zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001854
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001855 *mpol = get_vma_policy(vma, addr);
Mel Gorman19770b32008-04-28 02:12:18 -07001856 *nodemask = NULL; /* assume !MPOL_BIND */
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001857
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001858 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1859 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
Andi Kleena5516432008-07-23 21:27:41 -07001860 huge_page_shift(hstate_vma(vma))), gfp_flags);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001861 } else {
Andi Kleen2f5f9482011-03-04 17:36:29 -08001862 zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001863 if ((*mpol)->mode == MPOL_BIND)
1864 *nodemask = &(*mpol)->v.nodes;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001865 }
1866 return zl;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001867}
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001868
1869/*
1870 * init_nodemask_of_mempolicy
1871 *
1872 * If the current task's mempolicy is "default" [NULL], return 'false'
1873 * to indicate default policy. Otherwise, extract the policy nodemask
1874 * for 'bind' or 'interleave' policy into the argument nodemask, or
1875 * initialize the argument nodemask to contain the single node for
1876 * 'preferred' or 'local' policy and return 'true' to indicate presence
1877 * of non-default mempolicy.
1878 *
1879 * We don't bother with reference counting the mempolicy [mpol_get/put]
1880 * because the current task is examining it's own mempolicy and a task's
1881 * mempolicy is only ever changed by the task itself.
1882 *
1883 * N.B., it is the caller's responsibility to free a returned nodemask.
1884 */
1885bool init_nodemask_of_mempolicy(nodemask_t *mask)
1886{
1887 struct mempolicy *mempolicy;
1888 int nid;
1889
1890 if (!(mask && current->mempolicy))
1891 return false;
1892
Miao Xiec0ff7452010-05-24 14:32:08 -07001893 task_lock(current);
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001894 mempolicy = current->mempolicy;
1895 switch (mempolicy->mode) {
1896 case MPOL_PREFERRED:
1897 if (mempolicy->flags & MPOL_F_LOCAL)
1898 nid = numa_node_id();
1899 else
1900 nid = mempolicy->v.preferred_node;
1901 init_nodemask_of_node(mask, nid);
1902 break;
1903
1904 case MPOL_BIND:
1905 /* Fall through */
1906 case MPOL_INTERLEAVE:
1907 *mask = mempolicy->v.nodes;
1908 break;
1909
1910 default:
1911 BUG();
1912 }
Miao Xiec0ff7452010-05-24 14:32:08 -07001913 task_unlock(current);
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001914
1915 return true;
1916}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001917#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001918
David Rientjes6f48d0eb2010-08-09 17:18:52 -07001919/*
1920 * mempolicy_nodemask_intersects
1921 *
1922 * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
1923 * policy. Otherwise, check for intersection between mask and the policy
1924 * nodemask for 'bind' or 'interleave' policy. For 'perferred' or 'local'
1925 * policy, always return true since it may allocate elsewhere on fallback.
1926 *
1927 * Takes task_lock(tsk) to prevent freeing of its mempolicy.
1928 */
1929bool mempolicy_nodemask_intersects(struct task_struct *tsk,
1930 const nodemask_t *mask)
1931{
1932 struct mempolicy *mempolicy;
1933 bool ret = true;
1934
1935 if (!mask)
1936 return ret;
1937 task_lock(tsk);
1938 mempolicy = tsk->mempolicy;
1939 if (!mempolicy)
1940 goto out;
1941
1942 switch (mempolicy->mode) {
1943 case MPOL_PREFERRED:
1944 /*
1945 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
1946 * allocate from, they may fallback to other nodes when oom.
1947 * Thus, it's possible for tsk to have allocated memory from
1948 * nodes in mask.
1949 */
1950 break;
1951 case MPOL_BIND:
1952 case MPOL_INTERLEAVE:
1953 ret = nodes_intersects(mempolicy->v.nodes, *mask);
1954 break;
1955 default:
1956 BUG();
1957 }
1958out:
1959 task_unlock(tsk);
1960 return ret;
1961}
1962
Linus Torvalds1da177e2005-04-16 15:20:36 -07001963/* Allocate a page in interleaved policy.
1964 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07001965static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1966 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967{
1968 struct zonelist *zl;
1969 struct page *page;
1970
Mel Gorman0e884602008-04-28 02:12:14 -07001971 zl = node_zonelist(nid, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972 page = __alloc_pages(gfp, order, zl);
Mel Gormandd1a2392008-04-28 02:12:17 -07001973 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
Christoph Lameterca889e62006-06-30 01:55:44 -07001974 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001975 return page;
1976}
1977
1978/**
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08001979 * alloc_pages_vma - Allocate a page for a VMA.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001980 *
1981 * @gfp:
1982 * %GFP_USER user allocation.
1983 * %GFP_KERNEL kernel allocations,
1984 * %GFP_HIGHMEM highmem/user allocations,
1985 * %GFP_FS allocation should not call back into a file system.
1986 * %GFP_ATOMIC don't sleep.
1987 *
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08001988 * @order:Order of the GFP allocation.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989 * @vma: Pointer to VMA or NULL if not available.
1990 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1991 *
1992 * This function allocates a page from the kernel page pool and applies
1993 * a NUMA policy associated with the VMA or the current process.
1994 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1995 * mm_struct of the VMA to prevent it from going away. Should be used for
1996 * all allocations for pages that will be mapped into
1997 * user space. Returns NULL when no page can be allocated.
1998 *
1999 * Should be called with the mm_sem of the vma hold.
2000 */
2001struct page *
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002002alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
Andi Kleen2f5f9482011-03-04 17:36:29 -08002003 unsigned long addr, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002004{
Mel Gormancc9a6c82012-03-21 16:34:11 -07002005 struct mempolicy *pol;
Miao Xiec0ff7452010-05-24 14:32:08 -07002006 struct page *page;
Mel Gormancc9a6c82012-03-21 16:34:11 -07002007 unsigned int cpuset_mems_cookie;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002008
Mel Gormancc9a6c82012-03-21 16:34:11 -07002009retry_cpuset:
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07002010 pol = get_vma_policy(vma, addr);
Mel Gormand26914d2014-04-03 14:47:24 -07002011 cpuset_mems_cookie = read_mems_allowed_begin();
Mel Gormancc9a6c82012-03-21 16:34:11 -07002012
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002013 if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002014 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08002015
Andi Kleen8eac5632011-02-25 14:44:28 -08002016 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002017 mpol_cond_put(pol);
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002018 page = alloc_page_interleave(gfp, order, nid);
Mel Gormand26914d2014-04-03 14:47:24 -07002019 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
Mel Gormancc9a6c82012-03-21 16:34:11 -07002020 goto retry_cpuset;
2021
Miao Xiec0ff7452010-05-24 14:32:08 -07002022 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023 }
David Rientjes212a0a62012-12-11 16:02:51 -08002024 page = __alloc_pages_nodemask(gfp, order,
2025 policy_zonelist(gfp, pol, node),
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002026 policy_nodemask(gfp, pol));
Oleg Nesterov23867402014-10-09 15:27:41 -07002027 mpol_cond_put(pol);
Mel Gormand26914d2014-04-03 14:47:24 -07002028 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
Mel Gormancc9a6c82012-03-21 16:34:11 -07002029 goto retry_cpuset;
Miao Xiec0ff7452010-05-24 14:32:08 -07002030 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031}
2032
2033/**
2034 * alloc_pages_current - Allocate pages.
2035 *
2036 * @gfp:
2037 * %GFP_USER user allocation,
2038 * %GFP_KERNEL kernel allocation,
2039 * %GFP_HIGHMEM highmem allocation,
2040 * %GFP_FS don't call back into a file system.
2041 * %GFP_ATOMIC don't sleep.
2042 * @order: Power of two of allocation size in pages. 0 is a single page.
2043 *
2044 * Allocate a page from the kernel page pool. When not in
2045 * interrupt context and apply the current process NUMA policy.
2046 * Returns NULL when no page can be allocated.
2047 *
Paul Jacksoncf2a473c2006-01-08 01:01:54 -08002048 * Don't call cpuset_update_task_memory_state() unless
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049 * 1) it's ok to take cpuset_sem (can WAIT), and
2050 * 2) allocating for current task (not interrupt).
2051 */
Al Virodd0fc662005-10-07 07:46:04 +01002052struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053{
Oleg Nesterov8d902742014-10-09 15:27:45 -07002054 struct mempolicy *pol = &default_policy;
Miao Xiec0ff7452010-05-24 14:32:08 -07002055 struct page *page;
Mel Gormancc9a6c82012-03-21 16:34:11 -07002056 unsigned int cpuset_mems_cookie;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002057
Oleg Nesterov8d902742014-10-09 15:27:45 -07002058 if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2059 pol = get_task_policy(current);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002060
Mel Gormancc9a6c82012-03-21 16:34:11 -07002061retry_cpuset:
Mel Gormand26914d2014-04-03 14:47:24 -07002062 cpuset_mems_cookie = read_mems_allowed_begin();
Mel Gormancc9a6c82012-03-21 16:34:11 -07002063
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002064 /*
2065 * No reference counting needed for current->mempolicy
2066 * nor system default_policy
2067 */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002068 if (pol->mode == MPOL_INTERLEAVE)
Miao Xiec0ff7452010-05-24 14:32:08 -07002069 page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2070 else
2071 page = __alloc_pages_nodemask(gfp, order,
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002072 policy_zonelist(gfp, pol, numa_node_id()),
2073 policy_nodemask(gfp, pol));
Mel Gormancc9a6c82012-03-21 16:34:11 -07002074
Mel Gormand26914d2014-04-03 14:47:24 -07002075 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
Mel Gormancc9a6c82012-03-21 16:34:11 -07002076 goto retry_cpuset;
2077
Miao Xiec0ff7452010-05-24 14:32:08 -07002078 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002079}
2080EXPORT_SYMBOL(alloc_pages_current);
2081
Oleg Nesterovef0855d2013-09-11 14:20:14 -07002082int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2083{
2084 struct mempolicy *pol = mpol_dup(vma_policy(src));
2085
2086 if (IS_ERR(pol))
2087 return PTR_ERR(pol);
2088 dst->vm_policy = pol;
2089 return 0;
2090}
2091
Paul Jackson42253992006-01-08 01:01:59 -08002092/*
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07002093 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
Paul Jackson42253992006-01-08 01:01:59 -08002094 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2095 * with the mems_allowed returned by cpuset_mems_allowed(). This
2096 * keeps mempolicies cpuset relative after its cpuset moves. See
2097 * further kernel/cpuset.c update_nodemask().
Miao Xie708c1bb2010-05-24 14:32:07 -07002098 *
2099 * current's mempolicy may be rebinded by the other task(the task that changes
2100 * cpuset's mems), so we needn't do rebind work for current task.
Paul Jackson42253992006-01-08 01:01:59 -08002101 */
Paul Jackson42253992006-01-08 01:01:59 -08002102
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07002103/* Slow path of a mempolicy duplicate */
2104struct mempolicy *__mpol_dup(struct mempolicy *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002105{
2106 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2107
2108 if (!new)
2109 return ERR_PTR(-ENOMEM);
Miao Xie708c1bb2010-05-24 14:32:07 -07002110
2111 /* task's mempolicy is protected by alloc_lock */
2112 if (old == current->mempolicy) {
2113 task_lock(current);
2114 *new = *old;
2115 task_unlock(current);
2116 } else
2117 *new = *old;
2118
Paul Jackson42253992006-01-08 01:01:59 -08002119 if (current_cpuset_is_being_rebound()) {
2120 nodemask_t mems = cpuset_mems_allowed(current);
Miao Xie708c1bb2010-05-24 14:32:07 -07002121 if (new->flags & MPOL_F_REBINDING)
2122 mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
2123 else
2124 mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
Paul Jackson42253992006-01-08 01:01:59 -08002125 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002126 atomic_set(&new->refcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127 return new;
2128}
2129
2130/* Slow path of a mempolicy comparison */
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002131bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002132{
2133 if (!a || !b)
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002134 return false;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002135 if (a->mode != b->mode)
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002136 return false;
Bob Liu19800502010-05-24 14:32:01 -07002137 if (a->flags != b->flags)
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002138 return false;
Bob Liu19800502010-05-24 14:32:01 -07002139 if (mpol_store_user_nodemask(a))
2140 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002141 return false;
Bob Liu19800502010-05-24 14:32:01 -07002142
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002143 switch (a->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -07002144 case MPOL_BIND:
2145 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002146 case MPOL_INTERLEAVE:
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002147 return !!nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148 case MPOL_PREFERRED:
Namhyung Kim75719662011-03-22 16:33:02 -07002149 return a->v.preferred_node == b->v.preferred_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002150 default:
2151 BUG();
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002152 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002153 }
2154}
2155
Linus Torvalds1da177e2005-04-16 15:20:36 -07002156/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002157 * Shared memory backing store policy support.
2158 *
2159 * Remember policies even when nobody has shared memory mapped.
2160 * The policies are kept in Red-Black tree linked from the inode.
2161 * They are protected by the sp->lock spinlock, which should be held
2162 * for any accesses to the tree.
2163 */
2164
2165/* lookup first element intersecting start-end */
Mel Gorman42288fe2012-12-21 23:10:25 +00002166/* Caller holds sp->lock */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167static struct sp_node *
2168sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2169{
2170 struct rb_node *n = sp->root.rb_node;
2171
2172 while (n) {
2173 struct sp_node *p = rb_entry(n, struct sp_node, nd);
2174
2175 if (start >= p->end)
2176 n = n->rb_right;
2177 else if (end <= p->start)
2178 n = n->rb_left;
2179 else
2180 break;
2181 }
2182 if (!n)
2183 return NULL;
2184 for (;;) {
2185 struct sp_node *w = NULL;
2186 struct rb_node *prev = rb_prev(n);
2187 if (!prev)
2188 break;
2189 w = rb_entry(prev, struct sp_node, nd);
2190 if (w->end <= start)
2191 break;
2192 n = prev;
2193 }
2194 return rb_entry(n, struct sp_node, nd);
2195}
2196
2197/* Insert a new shared policy into the list. */
2198/* Caller holds sp->lock */
2199static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2200{
2201 struct rb_node **p = &sp->root.rb_node;
2202 struct rb_node *parent = NULL;
2203 struct sp_node *nd;
2204
2205 while (*p) {
2206 parent = *p;
2207 nd = rb_entry(parent, struct sp_node, nd);
2208 if (new->start < nd->start)
2209 p = &(*p)->rb_left;
2210 else if (new->end > nd->end)
2211 p = &(*p)->rb_right;
2212 else
2213 BUG();
2214 }
2215 rb_link_node(&new->nd, parent, p);
2216 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07002217 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002218 new->policy ? new->policy->mode : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002219}
2220
2221/* Find shared policy intersecting idx */
2222struct mempolicy *
2223mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2224{
2225 struct mempolicy *pol = NULL;
2226 struct sp_node *sn;
2227
2228 if (!sp->root.rb_node)
2229 return NULL;
Mel Gorman42288fe2012-12-21 23:10:25 +00002230 spin_lock(&sp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002231 sn = sp_lookup(sp, idx, idx+1);
2232 if (sn) {
2233 mpol_get(sn->policy);
2234 pol = sn->policy;
2235 }
Mel Gorman42288fe2012-12-21 23:10:25 +00002236 spin_unlock(&sp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002237 return pol;
2238}
2239
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002240static void sp_free(struct sp_node *n)
2241{
2242 mpol_put(n->policy);
2243 kmem_cache_free(sn_cache, n);
2244}
2245
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002246/**
2247 * mpol_misplaced - check whether current page node is valid in policy
2248 *
Fabian Frederickb46e14a2014-06-04 16:08:18 -07002249 * @page: page to be checked
2250 * @vma: vm area where page mapped
2251 * @addr: virtual address where page mapped
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002252 *
2253 * Lookup current policy node id for vma,addr and "compare to" page's
2254 * node id.
2255 *
2256 * Returns:
2257 * -1 - not misplaced, page is in the right node
2258 * node - node id where the page should be
2259 *
2260 * Policy determination "mimics" alloc_page_vma().
2261 * Called from fault path where we know the vma and faulting address.
2262 */
2263int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2264{
2265 struct mempolicy *pol;
2266 struct zone *zone;
2267 int curnid = page_to_nid(page);
2268 unsigned long pgoff;
Peter Zijlstra90572892013-10-07 11:29:20 +01002269 int thiscpu = raw_smp_processor_id();
2270 int thisnid = cpu_to_node(thiscpu);
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002271 int polnid = -1;
2272 int ret = -1;
2273
2274 BUG_ON(!vma);
2275
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07002276 pol = get_vma_policy(vma, addr);
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002277 if (!(pol->flags & MPOL_F_MOF))
2278 goto out;
2279
2280 switch (pol->mode) {
2281 case MPOL_INTERLEAVE:
2282 BUG_ON(addr >= vma->vm_end);
2283 BUG_ON(addr < vma->vm_start);
2284
2285 pgoff = vma->vm_pgoff;
2286 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
2287 polnid = offset_il_node(pol, vma, pgoff);
2288 break;
2289
2290 case MPOL_PREFERRED:
2291 if (pol->flags & MPOL_F_LOCAL)
2292 polnid = numa_node_id();
2293 else
2294 polnid = pol->v.preferred_node;
2295 break;
2296
2297 case MPOL_BIND:
2298 /*
2299 * allows binding to multiple nodes.
2300 * use current page if in policy nodemask,
2301 * else select nearest allowed node, if any.
2302 * If no allowed nodes, use current [!misplaced].
2303 */
2304 if (node_isset(curnid, pol->v.nodes))
2305 goto out;
2306 (void)first_zones_zonelist(
2307 node_zonelist(numa_node_id(), GFP_HIGHUSER),
2308 gfp_zone(GFP_HIGHUSER),
2309 &pol->v.nodes, &zone);
2310 polnid = zone->node;
2311 break;
2312
2313 default:
2314 BUG();
2315 }
Mel Gorman5606e382012-11-02 18:19:13 +00002316
2317 /* Migrate the page towards the node whose CPU is referencing it */
Mel Gormane42c8ff2012-11-12 09:17:07 +00002318 if (pol->flags & MPOL_F_MORON) {
Peter Zijlstra90572892013-10-07 11:29:20 +01002319 polnid = thisnid;
Mel Gorman5606e382012-11-02 18:19:13 +00002320
Rik van Riel10f39042014-01-27 17:03:44 -05002321 if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
Rik van Rielde1c9ce2013-10-07 11:29:39 +01002322 goto out;
Mel Gormane42c8ff2012-11-12 09:17:07 +00002323 }
2324
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002325 if (curnid != polnid)
2326 ret = polnid;
2327out:
2328 mpol_cond_put(pol);
2329
2330 return ret;
2331}
2332
Linus Torvalds1da177e2005-04-16 15:20:36 -07002333static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2334{
Paul Mundt140d5a42007-07-15 23:38:16 -07002335 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002336 rb_erase(&n->nd, &sp->root);
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002337 sp_free(n);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338}
2339
Mel Gorman42288fe2012-12-21 23:10:25 +00002340static void sp_node_init(struct sp_node *node, unsigned long start,
2341 unsigned long end, struct mempolicy *pol)
2342{
2343 node->start = start;
2344 node->end = end;
2345 node->policy = pol;
2346}
2347
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07002348static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2349 struct mempolicy *pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002350{
KOSAKI Motohiro869833f22012-10-08 16:29:16 -07002351 struct sp_node *n;
2352 struct mempolicy *newpol;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002353
KOSAKI Motohiro869833f22012-10-08 16:29:16 -07002354 n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002355 if (!n)
2356 return NULL;
KOSAKI Motohiro869833f22012-10-08 16:29:16 -07002357
2358 newpol = mpol_dup(pol);
2359 if (IS_ERR(newpol)) {
2360 kmem_cache_free(sn_cache, n);
2361 return NULL;
2362 }
2363 newpol->flags |= MPOL_F_SHARED;
Mel Gorman42288fe2012-12-21 23:10:25 +00002364 sp_node_init(n, start, end, newpol);
KOSAKI Motohiro869833f22012-10-08 16:29:16 -07002365
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366 return n;
2367}
2368
2369/* Replace a policy range. */
2370static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2371 unsigned long end, struct sp_node *new)
2372{
Mel Gormanb22d1272012-10-08 16:29:17 -07002373 struct sp_node *n;
Mel Gorman42288fe2012-12-21 23:10:25 +00002374 struct sp_node *n_new = NULL;
2375 struct mempolicy *mpol_new = NULL;
Mel Gormanb22d1272012-10-08 16:29:17 -07002376 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002377
Mel Gorman42288fe2012-12-21 23:10:25 +00002378restart:
2379 spin_lock(&sp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002380 n = sp_lookup(sp, start, end);
2381 /* Take care of old policies in the same range. */
2382 while (n && n->start < end) {
2383 struct rb_node *next = rb_next(&n->nd);
2384 if (n->start >= start) {
2385 if (n->end <= end)
2386 sp_delete(sp, n);
2387 else
2388 n->start = end;
2389 } else {
2390 /* Old policy spanning whole new range. */
2391 if (n->end > end) {
Mel Gorman42288fe2012-12-21 23:10:25 +00002392 if (!n_new)
2393 goto alloc_new;
2394
2395 *mpol_new = *n->policy;
2396 atomic_set(&mpol_new->refcnt, 1);
KOSAKI Motohiro78806392013-03-08 12:43:29 -08002397 sp_node_init(n_new, end, n->end, mpol_new);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398 n->end = start;
Hillf Danton5ca39572013-03-08 12:43:28 -08002399 sp_insert(sp, n_new);
Mel Gorman42288fe2012-12-21 23:10:25 +00002400 n_new = NULL;
2401 mpol_new = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002402 break;
2403 } else
2404 n->end = start;
2405 }
2406 if (!next)
2407 break;
2408 n = rb_entry(next, struct sp_node, nd);
2409 }
2410 if (new)
2411 sp_insert(sp, new);
Mel Gorman42288fe2012-12-21 23:10:25 +00002412 spin_unlock(&sp->lock);
2413 ret = 0;
2414
2415err_out:
2416 if (mpol_new)
2417 mpol_put(mpol_new);
2418 if (n_new)
2419 kmem_cache_free(sn_cache, n_new);
2420
Mel Gormanb22d1272012-10-08 16:29:17 -07002421 return ret;
Mel Gorman42288fe2012-12-21 23:10:25 +00002422
2423alloc_new:
2424 spin_unlock(&sp->lock);
2425 ret = -ENOMEM;
2426 n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2427 if (!n_new)
2428 goto err_out;
2429 mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2430 if (!mpol_new)
2431 goto err_out;
2432 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002433}
2434
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002435/**
2436 * mpol_shared_policy_init - initialize shared policy for inode
2437 * @sp: pointer to inode shared policy
2438 * @mpol: struct mempolicy to install
2439 *
2440 * Install non-NULL @mpol in inode's shared policy rb-tree.
2441 * On entry, the current task has a reference on a non-NULL @mpol.
2442 * This must be released on exit.
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002443 * This is called at get_inode() calls and we can use GFP_KERNEL.
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002444 */
2445void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
Robin Holt7339ff82006-01-14 13:20:48 -08002446{
Miao Xie58568d22009-06-16 15:31:49 -07002447 int ret;
2448
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002449 sp->root = RB_ROOT; /* empty tree == default mempolicy */
Mel Gorman42288fe2012-12-21 23:10:25 +00002450 spin_lock_init(&sp->lock);
Robin Holt7339ff82006-01-14 13:20:48 -08002451
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002452 if (mpol) {
2453 struct vm_area_struct pvma;
2454 struct mempolicy *new;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002455 NODEMASK_SCRATCH(scratch);
Robin Holt7339ff82006-01-14 13:20:48 -08002456
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002457 if (!scratch)
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002458 goto put_mpol;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002459 /* contextualize the tmpfs mount point mempolicy */
2460 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
Lee Schermerhorn15d77832010-05-24 14:32:04 -07002461 if (IS_ERR(new))
Dan Carpenter0cae3452010-05-25 23:42:58 -07002462 goto free_scratch; /* no valid nodemask intersection */
Miao Xie58568d22009-06-16 15:31:49 -07002463
2464 task_lock(current);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002465 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
Miao Xie58568d22009-06-16 15:31:49 -07002466 task_unlock(current);
Lee Schermerhorn15d77832010-05-24 14:32:04 -07002467 if (ret)
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002468 goto put_new;
Robin Holt7339ff82006-01-14 13:20:48 -08002469
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002470 /* Create pseudo-vma that contains just the policy */
2471 memset(&pvma, 0, sizeof(struct vm_area_struct));
2472 pvma.vm_end = TASK_SIZE; /* policy covers entire file */
2473 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
Lee Schermerhorn15d77832010-05-24 14:32:04 -07002474
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002475put_new:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002476 mpol_put(new); /* drop initial ref */
Dan Carpenter0cae3452010-05-25 23:42:58 -07002477free_scratch:
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002478 NODEMASK_SCRATCH_FREE(scratch);
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002479put_mpol:
2480 mpol_put(mpol); /* drop our incoming ref on sb mpol */
Robin Holt7339ff82006-01-14 13:20:48 -08002481 }
2482}
2483
Linus Torvalds1da177e2005-04-16 15:20:36 -07002484int mpol_set_shared_policy(struct shared_policy *info,
2485 struct vm_area_struct *vma, struct mempolicy *npol)
2486{
2487 int err;
2488 struct sp_node *new = NULL;
2489 unsigned long sz = vma_pages(vma);
2490
David Rientjes028fec42008-04-28 02:12:25 -07002491 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492 vma->vm_pgoff,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002493 sz, npol ? npol->mode : -1,
David Rientjes028fec42008-04-28 02:12:25 -07002494 npol ? npol->flags : -1,
David Rientjes00ef2d22013-02-22 16:35:36 -08002495 npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002496
2497 if (npol) {
2498 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2499 if (!new)
2500 return -ENOMEM;
2501 }
2502 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2503 if (err && new)
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002504 sp_free(new);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505 return err;
2506}
2507
2508/* Free a backing policy store on inode delete. */
2509void mpol_free_shared_policy(struct shared_policy *p)
2510{
2511 struct sp_node *n;
2512 struct rb_node *next;
2513
2514 if (!p->root.rb_node)
2515 return;
Mel Gorman42288fe2012-12-21 23:10:25 +00002516 spin_lock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002517 next = rb_first(&p->root);
2518 while (next) {
2519 n = rb_entry(next, struct sp_node, nd);
2520 next = rb_next(&n->nd);
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002521 sp_delete(p, n);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002522 }
Mel Gorman42288fe2012-12-21 23:10:25 +00002523 spin_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002524}
2525
Mel Gorman1a687c22012-11-22 11:16:36 +00002526#ifdef CONFIG_NUMA_BALANCING
Mel Gormanc2976632014-01-29 14:05:42 -08002527static int __initdata numabalancing_override;
Mel Gorman1a687c22012-11-22 11:16:36 +00002528
2529static void __init check_numabalancing_enable(void)
2530{
2531 bool numabalancing_default = false;
2532
2533 if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
2534 numabalancing_default = true;
2535
Mel Gormanc2976632014-01-29 14:05:42 -08002536 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2537 if (numabalancing_override)
2538 set_numabalancing_state(numabalancing_override == 1);
2539
Mel Gorman1a687c22012-11-22 11:16:36 +00002540 if (nr_node_ids > 1 && !numabalancing_override) {
Andrew Morton4a404be2014-01-29 14:05:43 -08002541 pr_info("%s automatic NUMA balancing. "
Mel Gormanc2976632014-01-29 14:05:42 -08002542 "Configure with numa_balancing= or the "
2543 "kernel.numa_balancing sysctl",
2544 numabalancing_default ? "Enabling" : "Disabling");
Mel Gorman1a687c22012-11-22 11:16:36 +00002545 set_numabalancing_state(numabalancing_default);
2546 }
2547}
2548
2549static int __init setup_numabalancing(char *str)
2550{
2551 int ret = 0;
2552 if (!str)
2553 goto out;
Mel Gorman1a687c22012-11-22 11:16:36 +00002554
2555 if (!strcmp(str, "enable")) {
Mel Gormanc2976632014-01-29 14:05:42 -08002556 numabalancing_override = 1;
Mel Gorman1a687c22012-11-22 11:16:36 +00002557 ret = 1;
2558 } else if (!strcmp(str, "disable")) {
Mel Gormanc2976632014-01-29 14:05:42 -08002559 numabalancing_override = -1;
Mel Gorman1a687c22012-11-22 11:16:36 +00002560 ret = 1;
2561 }
2562out:
2563 if (!ret)
Andrew Morton4a404be2014-01-29 14:05:43 -08002564 pr_warn("Unable to parse numa_balancing=\n");
Mel Gorman1a687c22012-11-22 11:16:36 +00002565
2566 return ret;
2567}
2568__setup("numa_balancing=", setup_numabalancing);
2569#else
2570static inline void __init check_numabalancing_enable(void)
2571{
2572}
2573#endif /* CONFIG_NUMA_BALANCING */
2574
Linus Torvalds1da177e2005-04-16 15:20:36 -07002575/* assumes fs == KERNEL_DS */
2576void __init numa_policy_init(void)
2577{
Paul Mundtb71636e2007-07-15 23:38:15 -07002578 nodemask_t interleave_nodes;
2579 unsigned long largest = 0;
2580 int nid, prefer = 0;
2581
Linus Torvalds1da177e2005-04-16 15:20:36 -07002582 policy_cache = kmem_cache_create("numa_policy",
2583 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09002584 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002585
2586 sn_cache = kmem_cache_create("shared_policy_node",
2587 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09002588 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002589
Mel Gorman5606e382012-11-02 18:19:13 +00002590 for_each_node(nid) {
2591 preferred_node_policy[nid] = (struct mempolicy) {
2592 .refcnt = ATOMIC_INIT(1),
2593 .mode = MPOL_PREFERRED,
2594 .flags = MPOL_F_MOF | MPOL_F_MORON,
2595 .v = { .preferred_node = nid, },
2596 };
2597 }
2598
Paul Mundtb71636e2007-07-15 23:38:15 -07002599 /*
2600 * Set interleaving policy for system init. Interleaving is only
2601 * enabled across suitably sized nodes (default is >= 16MB), or
2602 * fall back to the largest node if they're all smaller.
2603 */
2604 nodes_clear(interleave_nodes);
Lai Jiangshan01f13bd2012-12-12 13:51:33 -08002605 for_each_node_state(nid, N_MEMORY) {
Paul Mundtb71636e2007-07-15 23:38:15 -07002606 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002607
Paul Mundtb71636e2007-07-15 23:38:15 -07002608 /* Preserve the largest node */
2609 if (largest < total_pages) {
2610 largest = total_pages;
2611 prefer = nid;
2612 }
2613
2614 /* Interleave this node? */
2615 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2616 node_set(nid, interleave_nodes);
2617 }
2618
2619 /* All too small, use the largest */
2620 if (unlikely(nodes_empty(interleave_nodes)))
2621 node_set(prefer, interleave_nodes);
2622
David Rientjes028fec42008-04-28 02:12:25 -07002623 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -07002624 pr_err("%s: interleaving failed\n", __func__);
Mel Gorman1a687c22012-11-22 11:16:36 +00002625
2626 check_numabalancing_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002627}
2628
Christoph Lameter8bccd852005-10-29 18:16:59 -07002629/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630void numa_default_policy(void)
2631{
David Rientjes028fec42008-04-28 02:12:25 -07002632 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002633}
Paul Jackson68860ec2005-10-30 15:02:36 -08002634
Paul Jackson42253992006-01-08 01:01:59 -08002635/*
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002636 * Parse and format mempolicy from/to strings
2637 */
2638
2639/*
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002640 * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag.
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002641 */
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002642static const char * const policy_modes[] =
2643{
2644 [MPOL_DEFAULT] = "default",
2645 [MPOL_PREFERRED] = "prefer",
2646 [MPOL_BIND] = "bind",
2647 [MPOL_INTERLEAVE] = "interleave",
Lee Schermerhornd3a71032012-10-25 14:16:29 +02002648 [MPOL_LOCAL] = "local",
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002649};
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002650
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002651
2652#ifdef CONFIG_TMPFS
2653/**
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002654 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002655 * @str: string containing mempolicy to parse
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002656 * @mpol: pointer to struct mempolicy pointer, returned on success.
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002657 *
2658 * Format of input:
2659 * <mode>[=<flags>][:<nodelist>]
2660 *
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002661 * On success, returns 0, else 1
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002662 */
Hugh Dickinsa7a88b22013-01-02 02:04:23 -08002663int mpol_parse_str(char *str, struct mempolicy **mpol)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002664{
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002665 struct mempolicy *new = NULL;
Lee Schermerhornb4652e82010-05-24 14:32:03 -07002666 unsigned short mode;
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002667 unsigned short mode_flags;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002668 nodemask_t nodes;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002669 char *nodelist = strchr(str, ':');
2670 char *flags = strchr(str, '=');
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002671 int err = 1;
2672
2673 if (nodelist) {
2674 /* NUL-terminate mode or flags string */
2675 *nodelist++ = '\0';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002676 if (nodelist_parse(nodelist, nodes))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002677 goto out;
Lai Jiangshan01f13bd2012-12-12 13:51:33 -08002678 if (!nodes_subset(nodes, node_states[N_MEMORY]))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002679 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002680 } else
2681 nodes_clear(nodes);
2682
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002683 if (flags)
2684 *flags++ = '\0'; /* terminate mode string */
2685
Peter Zijlstra479e2802012-10-25 14:16:28 +02002686 for (mode = 0; mode < MPOL_MAX; mode++) {
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002687 if (!strcmp(str, policy_modes[mode])) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002688 break;
2689 }
2690 }
Mel Gormana7200942012-11-16 09:37:58 +00002691 if (mode >= MPOL_MAX)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002692 goto out;
2693
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002694 switch (mode) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002695 case MPOL_PREFERRED:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002696 /*
2697 * Insist on a nodelist of one node only
2698 */
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002699 if (nodelist) {
2700 char *rest = nodelist;
2701 while (isdigit(*rest))
2702 rest++;
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002703 if (*rest)
2704 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002705 }
2706 break;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002707 case MPOL_INTERLEAVE:
2708 /*
2709 * Default to online nodes with memory if no nodelist
2710 */
2711 if (!nodelist)
Lai Jiangshan01f13bd2012-12-12 13:51:33 -08002712 nodes = node_states[N_MEMORY];
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002713 break;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002714 case MPOL_LOCAL:
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002715 /*
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002716 * Don't allow a nodelist; mpol_new() checks flags
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002717 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002718 if (nodelist)
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002719 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002720 mode = MPOL_PREFERRED;
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002721 break;
Ravikiran G Thirumalai413b43d2010-03-23 13:35:28 -07002722 case MPOL_DEFAULT:
2723 /*
2724 * Insist on a empty nodelist
2725 */
2726 if (!nodelist)
2727 err = 0;
2728 goto out;
KOSAKI Motohirod69b2e62010-03-23 13:35:30 -07002729 case MPOL_BIND:
2730 /*
2731 * Insist on a nodelist
2732 */
2733 if (!nodelist)
2734 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002735 }
2736
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002737 mode_flags = 0;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002738 if (flags) {
2739 /*
2740 * Currently, we only support two mutually exclusive
2741 * mode flags.
2742 */
2743 if (!strcmp(flags, "static"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002744 mode_flags |= MPOL_F_STATIC_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002745 else if (!strcmp(flags, "relative"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002746 mode_flags |= MPOL_F_RELATIVE_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002747 else
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002748 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002749 }
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002750
2751 new = mpol_new(mode, mode_flags, &nodes);
2752 if (IS_ERR(new))
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002753 goto out;
2754
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002755 /*
2756 * Save nodes for mpol_to_str() to show the tmpfs mount options
2757 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
2758 */
2759 if (mode != MPOL_PREFERRED)
2760 new->v.nodes = nodes;
2761 else if (nodelist)
2762 new->v.preferred_node = first_node(nodes);
2763 else
2764 new->flags |= MPOL_F_LOCAL;
2765
2766 /*
2767 * Save nodes for contextualization: this will be used to "clone"
2768 * the mempolicy in a specific context [cpuset] at a later time.
2769 */
2770 new->w.user_nodemask = nodes;
2771
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002772 err = 0;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002773
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002774out:
2775 /* Restore string for error message */
2776 if (nodelist)
2777 *--nodelist = ':';
2778 if (flags)
2779 *--flags = '=';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002780 if (!err)
2781 *mpol = new;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002782 return err;
2783}
2784#endif /* CONFIG_TMPFS */
2785
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002786/**
2787 * mpol_to_str - format a mempolicy structure for printing
2788 * @buffer: to contain formatted mempolicy string
2789 * @maxlen: length of @buffer
2790 * @pol: pointer to mempolicy to be formatted
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002791 *
David Rientjes948927e2013-11-12 15:07:28 -08002792 * Convert @pol into a string. If @buffer is too short, truncate the string.
2793 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
2794 * longest flag, "relative", and to display at least a few node ids.
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002795 */
David Rientjes948927e2013-11-12 15:07:28 -08002796void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002797{
2798 char *p = buffer;
David Rientjes948927e2013-11-12 15:07:28 -08002799 nodemask_t nodes = NODE_MASK_NONE;
2800 unsigned short mode = MPOL_DEFAULT;
2801 unsigned short flags = 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002802
David Rientjes8790c712014-01-30 15:46:08 -08002803 if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002804 mode = pol->mode;
David Rientjes948927e2013-11-12 15:07:28 -08002805 flags = pol->flags;
2806 }
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002807
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002808 switch (mode) {
2809 case MPOL_DEFAULT:
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002810 break;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002811 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002812 if (flags & MPOL_F_LOCAL)
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002813 mode = MPOL_LOCAL;
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002814 else
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002815 node_set(pol->v.preferred_node, nodes);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002816 break;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002817 case MPOL_BIND:
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002818 case MPOL_INTERLEAVE:
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002819 nodes = pol->v.nodes;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002820 break;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002821 default:
David Rientjes948927e2013-11-12 15:07:28 -08002822 WARN_ON_ONCE(1);
2823 snprintf(p, maxlen, "unknown");
2824 return;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002825 }
2826
David Rientjesb7a9f422013-11-21 14:32:06 -08002827 p += snprintf(p, maxlen, "%s", policy_modes[mode]);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002828
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002829 if (flags & MPOL_MODE_FLAGS) {
David Rientjes948927e2013-11-12 15:07:28 -08002830 p += snprintf(p, buffer + maxlen - p, "=");
David Rientjesf5b087b2008-04-28 02:12:27 -07002831
Lee Schermerhorn22919902008-04-28 02:13:22 -07002832 /*
2833 * Currently, the only defined flags are mutually exclusive
2834 */
David Rientjesf5b087b2008-04-28 02:12:27 -07002835 if (flags & MPOL_F_STATIC_NODES)
Lee Schermerhorn22919902008-04-28 02:13:22 -07002836 p += snprintf(p, buffer + maxlen - p, "static");
2837 else if (flags & MPOL_F_RELATIVE_NODES)
2838 p += snprintf(p, buffer + maxlen - p, "relative");
David Rientjesf5b087b2008-04-28 02:12:27 -07002839 }
2840
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002841 if (!nodes_empty(nodes)) {
David Rientjes948927e2013-11-12 15:07:28 -08002842 p += snprintf(p, buffer + maxlen - p, ":");
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002843 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
2844 }
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002845}