blob: 22874d7cf2c017c2be6b03b5c00eeff3250d411c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * kernel/cpuset.c
3 *
4 * Processor and Memory placement constraints for sets of tasks.
5 *
6 * Copyright (C) 2003 BULL SA.
Paul Jackson029190c2007-10-18 23:40:20 -07007 * Copyright (C) 2004-2007 Silicon Graphics, Inc.
Paul Menage8793d852007-10-18 23:39:39 -07008 * Copyright (C) 2006 Google, Inc
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 *
10 * Portions derived from Patrick Mochel's sysfs code.
11 * sysfs is Copyright (c) 2001-3 Patrick Mochel
Linus Torvalds1da177e2005-04-16 15:20:36 -070012 *
Paul Jackson825a46a2006-03-24 03:16:03 -080013 * 2003-10-10 Written by Simon Derr.
Linus Torvalds1da177e2005-04-16 15:20:36 -070014 * 2003-10-22 Updates by Stephen Hemminger.
Paul Jackson825a46a2006-03-24 03:16:03 -080015 * 2004 May-July Rework by Paul Jackson.
Paul Menage8793d852007-10-18 23:39:39 -070016 * 2006 Rework by Paul Menage to use generic cgroups
Max Krasnyanskycf417142008-08-11 14:33:53 -070017 * 2008 Rework of the scheduler domains and CPU hotplug handling
18 * by Max Krasnyansky
Linus Torvalds1da177e2005-04-16 15:20:36 -070019 *
20 * This file is subject to the terms and conditions of the GNU General Public
21 * License. See the file COPYING in the main directory of the Linux
22 * distribution for more details.
23 */
24
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/cpu.h>
26#include <linux/cpumask.h>
27#include <linux/cpuset.h>
28#include <linux/err.h>
29#include <linux/errno.h>
30#include <linux/file.h>
31#include <linux/fs.h>
32#include <linux/init.h>
33#include <linux/interrupt.h>
34#include <linux/kernel.h>
35#include <linux/kmod.h>
36#include <linux/list.h>
Paul Jackson68860ec2005-10-30 15:02:36 -080037#include <linux/mempolicy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#include <linux/mm.h>
Miao Xief4818912008-11-19 15:36:30 -080039#include <linux/memory.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040040#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/mount.h>
42#include <linux/namei.h>
43#include <linux/pagemap.h>
44#include <linux/proc_fs.h>
Paul Jackson6b9c2602006-01-08 01:02:02 -080045#include <linux/rcupdate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070046#include <linux/sched.h>
47#include <linux/seq_file.h>
David Quigley22fb52d2006-06-23 02:04:00 -070048#include <linux/security.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050#include <linux/spinlock.h>
51#include <linux/stat.h>
52#include <linux/string.h>
53#include <linux/time.h>
54#include <linux/backing-dev.h>
55#include <linux/sort.h>
56
57#include <asm/uaccess.h>
Arun Sharma600634972011-07-26 16:09:06 -070058#include <linux/atomic.h>
Ingo Molnar3d3f26a2006-03-23 03:00:18 -080059#include <linux/mutex.h>
Cliff Wickman956db3c2008-02-07 00:14:43 -080060#include <linux/workqueue.h>
61#include <linux/cgroup.h>
Li Zefane44193d2013-06-09 17:14:22 +080062#include <linux/wait.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Mel Gorman664eedd2014-06-04 16:10:08 -070064struct static_key cpusets_enabled_key __read_mostly = STATIC_KEY_INIT_FALSE;
Paul Jackson202f72d2006-01-08 01:01:57 -080065
Paul Jackson3e0d98b2006-01-08 01:01:49 -080066/* See "Frequency meter" comments, below. */
67
68struct fmeter {
69 int cnt; /* unprocessed events count */
70 int val; /* most recent output value */
71 time_t time; /* clock (secs) when val computed */
72 spinlock_t lock; /* guards read or write of above */
73};
74
Linus Torvalds1da177e2005-04-16 15:20:36 -070075struct cpuset {
Paul Menage8793d852007-10-18 23:39:39 -070076 struct cgroup_subsys_state css;
77
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 unsigned long flags; /* "unsigned long" so bitops work */
Li Zefane2b9a3d2014-07-09 16:47:03 +080079
Li Zefan7e882912014-07-09 16:48:42 +080080 /*
81 * On default hierarchy:
82 *
83 * The user-configured masks can only be changed by writing to
84 * cpuset.cpus and cpuset.mems, and won't be limited by the
85 * parent masks.
86 *
87 * The effective masks is the real masks that apply to the tasks
88 * in the cpuset. They may be changed if the configured masks are
89 * changed or hotplug happens.
90 *
91 * effective_mask == configured_mask & parent's effective_mask,
92 * and if it ends up empty, it will inherit the parent's mask.
93 *
94 *
95 * On legacy hierachy:
96 *
97 * The user-configured masks are always the same with effective masks.
98 */
99
Li Zefane2b9a3d2014-07-09 16:47:03 +0800100 /* user-configured CPUs and Memory Nodes allow to tasks */
101 cpumask_var_t cpus_allowed;
102 nodemask_t mems_allowed;
103
104 /* effective CPUs and Memory Nodes allow to tasks */
105 cpumask_var_t effective_cpus;
106 nodemask_t effective_mems;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
Li Zefan33ad8012013-06-09 17:15:08 +0800108 /*
109 * This is old Memory Nodes tasks took on.
110 *
111 * - top_cpuset.old_mems_allowed is initialized to mems_allowed.
112 * - A new cpuset's old_mems_allowed is initialized when some
113 * task is moved into it.
114 * - old_mems_allowed is used in cpuset_migrate_mm() when we change
115 * cpuset.mems_allowed and have tasks' nodemask updated, and
116 * then old_mems_allowed is updated to mems_allowed.
117 */
118 nodemask_t old_mems_allowed;
119
Paul Jackson3e0d98b2006-01-08 01:01:49 -0800120 struct fmeter fmeter; /* memory_pressure filter */
Paul Jackson029190c2007-10-18 23:40:20 -0700121
Tejun Heo452477f2013-01-07 08:51:07 -0800122 /*
123 * Tasks are being attached to this cpuset. Used to prevent
124 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
125 */
126 int attach_in_progress;
127
Paul Jackson029190c2007-10-18 23:40:20 -0700128 /* partition number for rebuild_sched_domains() */
129 int pn;
Cliff Wickman956db3c2008-02-07 00:14:43 -0800130
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900131 /* for custom sched domain */
132 int relax_domain_level;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133};
134
Tejun Heoa7c6d552013-08-08 20:11:23 -0400135static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
Paul Menage8793d852007-10-18 23:39:39 -0700136{
Tejun Heoa7c6d552013-08-08 20:11:23 -0400137 return css ? container_of(css, struct cpuset, css) : NULL;
Paul Menage8793d852007-10-18 23:39:39 -0700138}
139
140/* Retrieve the cpuset for a task */
141static inline struct cpuset *task_cs(struct task_struct *task)
142{
Tejun Heo073219e2014-02-08 10:36:58 -0500143 return css_cs(task_css(task, cpuset_cgrp_id));
Paul Menage8793d852007-10-18 23:39:39 -0700144}
Paul Menage8793d852007-10-18 23:39:39 -0700145
Tejun Heoc9710d82013-08-08 20:11:22 -0400146static inline struct cpuset *parent_cs(struct cpuset *cs)
Tejun Heoc4310692013-01-07 08:51:08 -0800147{
Tejun Heo5c9d5352014-05-16 13:22:48 -0400148 return css_cs(cs->css.parent);
Tejun Heoc4310692013-01-07 08:51:08 -0800149}
150
David Rientjesb2462722011-12-19 17:11:52 -0800151#ifdef CONFIG_NUMA
152static inline bool task_has_mempolicy(struct task_struct *task)
153{
154 return task->mempolicy;
155}
156#else
157static inline bool task_has_mempolicy(struct task_struct *task)
158{
159 return false;
160}
161#endif
162
163
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164/* bits in struct cpuset flags field */
165typedef enum {
Tejun Heoefeb77b2013-01-07 08:51:07 -0800166 CS_ONLINE,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 CS_CPU_EXCLUSIVE,
168 CS_MEM_EXCLUSIVE,
Paul Menage78608362008-04-29 01:00:26 -0700169 CS_MEM_HARDWALL,
Paul Jackson45b07ef2006-01-08 01:00:56 -0800170 CS_MEMORY_MIGRATE,
Paul Jackson029190c2007-10-18 23:40:20 -0700171 CS_SCHED_LOAD_BALANCE,
Paul Jackson825a46a2006-03-24 03:16:03 -0800172 CS_SPREAD_PAGE,
173 CS_SPREAD_SLAB,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174} cpuset_flagbits_t;
175
176/* convenient tests for these bits */
Tejun Heoefeb77b2013-01-07 08:51:07 -0800177static inline bool is_cpuset_online(const struct cpuset *cs)
178{
179 return test_bit(CS_ONLINE, &cs->flags);
180}
181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182static inline int is_cpu_exclusive(const struct cpuset *cs)
183{
Paul Jackson7b5b9ef2006-03-24 03:16:00 -0800184 return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185}
186
187static inline int is_mem_exclusive(const struct cpuset *cs)
188{
Paul Jackson7b5b9ef2006-03-24 03:16:00 -0800189 return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190}
191
Paul Menage78608362008-04-29 01:00:26 -0700192static inline int is_mem_hardwall(const struct cpuset *cs)
193{
194 return test_bit(CS_MEM_HARDWALL, &cs->flags);
195}
196
Paul Jackson029190c2007-10-18 23:40:20 -0700197static inline int is_sched_load_balance(const struct cpuset *cs)
198{
199 return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
200}
201
Paul Jackson45b07ef2006-01-08 01:00:56 -0800202static inline int is_memory_migrate(const struct cpuset *cs)
203{
Paul Jackson7b5b9ef2006-03-24 03:16:00 -0800204 return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
Paul Jackson45b07ef2006-01-08 01:00:56 -0800205}
206
Paul Jackson825a46a2006-03-24 03:16:03 -0800207static inline int is_spread_page(const struct cpuset *cs)
208{
209 return test_bit(CS_SPREAD_PAGE, &cs->flags);
210}
211
212static inline int is_spread_slab(const struct cpuset *cs)
213{
214 return test_bit(CS_SPREAD_SLAB, &cs->flags);
215}
216
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217static struct cpuset top_cpuset = {
Tejun Heoefeb77b2013-01-07 08:51:07 -0800218 .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |
219 (1 << CS_MEM_EXCLUSIVE)),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220};
221
Tejun Heoae8086c2013-01-07 08:51:07 -0800222/**
223 * cpuset_for_each_child - traverse online children of a cpuset
224 * @child_cs: loop cursor pointing to the current child
Tejun Heo492eb212013-08-08 20:11:25 -0400225 * @pos_css: used for iteration
Tejun Heoae8086c2013-01-07 08:51:07 -0800226 * @parent_cs: target cpuset to walk children of
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 *
Tejun Heoae8086c2013-01-07 08:51:07 -0800228 * Walk @child_cs through the online children of @parent_cs. Must be used
229 * with RCU read locked.
230 */
Tejun Heo492eb212013-08-08 20:11:25 -0400231#define cpuset_for_each_child(child_cs, pos_css, parent_cs) \
232 css_for_each_child((pos_css), &(parent_cs)->css) \
233 if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
Tejun Heoae8086c2013-01-07 08:51:07 -0800234
Tejun Heofc560a22013-01-07 08:51:08 -0800235/**
236 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
237 * @des_cs: loop cursor pointing to the current descendant
Tejun Heo492eb212013-08-08 20:11:25 -0400238 * @pos_css: used for iteration
Tejun Heofc560a22013-01-07 08:51:08 -0800239 * @root_cs: target cpuset to walk ancestor of
240 *
241 * Walk @des_cs through the online descendants of @root_cs. Must be used
Tejun Heo492eb212013-08-08 20:11:25 -0400242 * with RCU read locked. The caller may modify @pos_css by calling
Tejun Heobd8815a2013-08-08 20:11:27 -0400243 * css_rightmost_descendant() to skip subtree. @root_cs is included in the
244 * iteration and the first node to be visited.
Tejun Heofc560a22013-01-07 08:51:08 -0800245 */
Tejun Heo492eb212013-08-08 20:11:25 -0400246#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \
247 css_for_each_descendant_pre((pos_css), &(root_cs)->css) \
248 if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
Tejun Heofc560a22013-01-07 08:51:08 -0800249
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250/*
Tejun Heo5d21cc22013-01-07 08:51:08 -0800251 * There are two global mutexes guarding cpuset structures - cpuset_mutex
252 * and callback_mutex. The latter may nest inside the former. We also
253 * require taking task_lock() when dereferencing a task's cpuset pointer.
254 * See "The task_lock() exception", at the end of this comment.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 *
Tejun Heo5d21cc22013-01-07 08:51:08 -0800256 * A task must hold both mutexes to modify cpusets. If a task holds
257 * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it
258 * is the only task able to also acquire callback_mutex and be able to
259 * modify cpusets. It can perform various checks on the cpuset structure
260 * first, knowing nothing will change. It can also allocate memory while
261 * just holding cpuset_mutex. While it is performing these checks, various
262 * callback routines can briefly acquire callback_mutex to query cpusets.
263 * Once it is ready to make the changes, it takes callback_mutex, blocking
264 * everyone else.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 *
Paul Jackson053199e2005-10-30 15:02:30 -0800266 * Calls to the kernel memory allocator can not be made while holding
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800267 * callback_mutex, as that would risk double tripping on callback_mutex
Paul Jackson053199e2005-10-30 15:02:30 -0800268 * from one of the callbacks into the cpuset code from within
269 * __alloc_pages().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270 *
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800271 * If a task is only holding callback_mutex, then it has read-only
Paul Jackson053199e2005-10-30 15:02:30 -0800272 * access to cpusets.
273 *
Miao Xie58568d22009-06-16 15:31:49 -0700274 * Now, the task_struct fields mems_allowed and mempolicy may be changed
275 * by other task, we use alloc_lock in the task_struct fields to protect
276 * them.
Paul Jackson053199e2005-10-30 15:02:30 -0800277 *
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800278 * The cpuset_common_file_read() handlers only hold callback_mutex across
Paul Jackson053199e2005-10-30 15:02:30 -0800279 * small pieces of code, such as when reading out possibly multi-word
280 * cpumasks and nodemasks.
281 *
Paul Menage2df167a2008-02-07 00:14:45 -0800282 * Accessing a task's cpuset should be done in accordance with the
283 * guidelines for accessing subsystem state in kernel/cgroup.c
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 */
285
Tejun Heo5d21cc22013-01-07 08:51:08 -0800286static DEFINE_MUTEX(cpuset_mutex);
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800287static DEFINE_MUTEX(callback_mutex);
Paul Jackson4247bdc2005-09-10 00:26:06 -0700288
Max Krasnyanskycf417142008-08-11 14:33:53 -0700289/*
Tejun Heo3a5a6d02013-01-07 08:51:07 -0800290 * CPU / memory hotplug is handled asynchronously.
291 */
292static void cpuset_hotplug_workfn(struct work_struct *work);
Tejun Heo3a5a6d02013-01-07 08:51:07 -0800293static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn);
294
Li Zefane44193d2013-06-09 17:14:22 +0800295static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);
296
Tejun Heo3a5a6d02013-01-07 08:51:07 -0800297/*
Max Krasnyanskycf417142008-08-11 14:33:53 -0700298 * This is ugly, but preserves the userspace API for existing cpuset
Paul Menage8793d852007-10-18 23:39:39 -0700299 * users. If someone tries to mount the "cpuset" filesystem, we
Max Krasnyanskycf417142008-08-11 14:33:53 -0700300 * silently switch it to mount "cgroup" instead
301 */
Al Virof7e83572010-07-26 13:23:11 +0400302static struct dentry *cpuset_mount(struct file_system_type *fs_type,
303 int flags, const char *unused_dev_name, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304{
Paul Menage8793d852007-10-18 23:39:39 -0700305 struct file_system_type *cgroup_fs = get_fs_type("cgroup");
Al Virof7e83572010-07-26 13:23:11 +0400306 struct dentry *ret = ERR_PTR(-ENODEV);
Paul Menage8793d852007-10-18 23:39:39 -0700307 if (cgroup_fs) {
308 char mountopts[] =
309 "cpuset,noprefix,"
310 "release_agent=/sbin/cpuset_release_agent";
Al Virof7e83572010-07-26 13:23:11 +0400311 ret = cgroup_fs->mount(cgroup_fs, flags,
312 unused_dev_name, mountopts);
Paul Menage8793d852007-10-18 23:39:39 -0700313 put_filesystem(cgroup_fs);
314 }
315 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316}
317
318static struct file_system_type cpuset_fs_type = {
319 .name = "cpuset",
Al Virof7e83572010-07-26 13:23:11 +0400320 .mount = cpuset_mount,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321};
322
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323/*
Li Zefan300ed6c2009-01-07 18:08:44 -0800324 * Return in pmask the portion of a cpusets's cpus_allowed that
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 * are online. If none are online, walk up the cpuset hierarchy
Li Zefan40df2de2013-06-05 17:15:23 +0800326 * until we find one that does have some online cpus. The top
327 * cpuset always has some cpus online.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 *
329 * One way or another, we guarantee to return some non-empty subset
Rusty Russell5f054e32012-03-29 15:38:31 +1030330 * of cpu_online_mask.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 *
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800332 * Call with callback_mutex held.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 */
Tejun Heoc9710d82013-08-08 20:11:22 -0400334static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
Li Zefanae1c8022014-07-09 16:48:32 +0800336 while (!cpumask_intersects(cs->effective_cpus, cpu_online_mask))
Tejun Heoc4310692013-01-07 08:51:08 -0800337 cs = parent_cs(cs);
Li Zefanae1c8022014-07-09 16:48:32 +0800338 cpumask_and(pmask, cs->effective_cpus, cpu_online_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339}
340
341/*
342 * Return in *pmask the portion of a cpusets's mems_allowed that
Christoph Lameter0e1e7c72007-10-16 01:25:38 -0700343 * are online, with memory. If none are online with memory, walk
344 * up the cpuset hierarchy until we find one that does have some
Li Zefan40df2de2013-06-05 17:15:23 +0800345 * online mems. The top cpuset always has some mems online.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 *
347 * One way or another, we guarantee to return some non-empty subset
Lai Jiangshan38d7bee2012-12-12 13:51:24 -0800348 * of node_states[N_MEMORY].
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 *
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800350 * Call with callback_mutex held.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 */
Tejun Heoc9710d82013-08-08 20:11:22 -0400352static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353{
Li Zefanae1c8022014-07-09 16:48:32 +0800354 while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))
Tejun Heoc4310692013-01-07 08:51:08 -0800355 cs = parent_cs(cs);
Li Zefanae1c8022014-07-09 16:48:32 +0800356 nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357}
358
Miao Xief3b39d42009-06-16 15:31:46 -0700359/*
360 * update task's spread flag if cpuset's page/slab spread flag is set
361 *
Tejun Heo5d21cc22013-01-07 08:51:08 -0800362 * Called with callback_mutex/cpuset_mutex held
Miao Xief3b39d42009-06-16 15:31:46 -0700363 */
364static void cpuset_update_task_spread_flag(struct cpuset *cs,
365 struct task_struct *tsk)
366{
367 if (is_spread_page(cs))
368 tsk->flags |= PF_SPREAD_PAGE;
369 else
370 tsk->flags &= ~PF_SPREAD_PAGE;
371 if (is_spread_slab(cs))
372 tsk->flags |= PF_SPREAD_SLAB;
373 else
374 tsk->flags &= ~PF_SPREAD_SLAB;
375}
376
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377/*
378 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
379 *
380 * One cpuset is a subset of another if all its allowed CPUs and
381 * Memory Nodes are a subset of the other, and its exclusive flags
Tejun Heo5d21cc22013-01-07 08:51:08 -0800382 * are only set if the other's are set. Call holding cpuset_mutex.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 */
384
385static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
386{
Li Zefan300ed6c2009-01-07 18:08:44 -0800387 return cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 nodes_subset(p->mems_allowed, q->mems_allowed) &&
389 is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
390 is_mem_exclusive(p) <= is_mem_exclusive(q);
391}
392
Li Zefan645fcc92009-01-07 18:08:43 -0800393/**
394 * alloc_trial_cpuset - allocate a trial cpuset
395 * @cs: the cpuset that the trial cpuset duplicates
396 */
Tejun Heoc9710d82013-08-08 20:11:22 -0400397static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
Li Zefan645fcc92009-01-07 18:08:43 -0800398{
Li Zefan300ed6c2009-01-07 18:08:44 -0800399 struct cpuset *trial;
400
401 trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
402 if (!trial)
403 return NULL;
404
Li Zefane2b9a3d2014-07-09 16:47:03 +0800405 if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL))
406 goto free_cs;
407 if (!alloc_cpumask_var(&trial->effective_cpus, GFP_KERNEL))
408 goto free_cpus;
Li Zefan300ed6c2009-01-07 18:08:44 -0800409
Li Zefane2b9a3d2014-07-09 16:47:03 +0800410 cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
411 cpumask_copy(trial->effective_cpus, cs->effective_cpus);
Li Zefan300ed6c2009-01-07 18:08:44 -0800412 return trial;
Li Zefane2b9a3d2014-07-09 16:47:03 +0800413
414free_cpus:
415 free_cpumask_var(trial->cpus_allowed);
416free_cs:
417 kfree(trial);
418 return NULL;
Li Zefan645fcc92009-01-07 18:08:43 -0800419}
420
421/**
422 * free_trial_cpuset - free the trial cpuset
423 * @trial: the trial cpuset to be freed
424 */
425static void free_trial_cpuset(struct cpuset *trial)
426{
Li Zefane2b9a3d2014-07-09 16:47:03 +0800427 free_cpumask_var(trial->effective_cpus);
Li Zefan300ed6c2009-01-07 18:08:44 -0800428 free_cpumask_var(trial->cpus_allowed);
Li Zefan645fcc92009-01-07 18:08:43 -0800429 kfree(trial);
430}
431
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432/*
433 * validate_change() - Used to validate that any proposed cpuset change
434 * follows the structural rules for cpusets.
435 *
436 * If we replaced the flag and mask values of the current cpuset
437 * (cur) with those values in the trial cpuset (trial), would
438 * our various subset and exclusive rules still be valid? Presumes
Tejun Heo5d21cc22013-01-07 08:51:08 -0800439 * cpuset_mutex held.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 *
441 * 'cur' is the address of an actual, in-use cpuset. Operations
442 * such as list traversal that depend on the actual address of the
443 * cpuset in the list must use cur below, not trial.
444 *
445 * 'trial' is the address of bulk structure copy of cur, with
446 * perhaps one or more of the fields cpus_allowed, mems_allowed,
447 * or flags changed to new, trial values.
448 *
449 * Return 0 if valid, -errno if not.
450 */
451
Tejun Heoc9710d82013-08-08 20:11:22 -0400452static int validate_change(struct cpuset *cur, struct cpuset *trial)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453{
Tejun Heo492eb212013-08-08 20:11:25 -0400454 struct cgroup_subsys_state *css;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 struct cpuset *c, *par;
Tejun Heoae8086c2013-01-07 08:51:07 -0800456 int ret;
457
458 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459
460 /* Each of our child cpusets must be a subset of us */
Tejun Heoae8086c2013-01-07 08:51:07 -0800461 ret = -EBUSY;
Tejun Heo492eb212013-08-08 20:11:25 -0400462 cpuset_for_each_child(c, css, cur)
Tejun Heoae8086c2013-01-07 08:51:07 -0800463 if (!is_cpuset_subset(c, trial))
464 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
466 /* Remaining checks don't apply to root cpuset */
Tejun Heoae8086c2013-01-07 08:51:07 -0800467 ret = 0;
Paul Jackson69604062006-12-06 20:36:15 -0800468 if (cur == &top_cpuset)
Tejun Heoae8086c2013-01-07 08:51:07 -0800469 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470
Tejun Heoc4310692013-01-07 08:51:08 -0800471 par = parent_cs(cur);
Paul Jackson69604062006-12-06 20:36:15 -0800472
Li Zefan7e882912014-07-09 16:48:42 +0800473 /* On legacy hiearchy, we must be a subset of our parent cpuset. */
Tejun Heoae8086c2013-01-07 08:51:07 -0800474 ret = -EACCES;
Li Zefan7e882912014-07-09 16:48:42 +0800475 if (!cgroup_on_dfl(cur->css.cgroup) && !is_cpuset_subset(trial, par))
Tejun Heoae8086c2013-01-07 08:51:07 -0800476 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477
Paul Menage2df167a2008-02-07 00:14:45 -0800478 /*
479 * If either I or some sibling (!= me) is exclusive, we can't
480 * overlap
481 */
Tejun Heoae8086c2013-01-07 08:51:07 -0800482 ret = -EINVAL;
Tejun Heo492eb212013-08-08 20:11:25 -0400483 cpuset_for_each_child(c, css, par) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
485 c != cur &&
Li Zefan300ed6c2009-01-07 18:08:44 -0800486 cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
Tejun Heoae8086c2013-01-07 08:51:07 -0800487 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
489 c != cur &&
490 nodes_intersects(trial->mems_allowed, c->mems_allowed))
Tejun Heoae8086c2013-01-07 08:51:07 -0800491 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492 }
493
Tejun Heo452477f2013-01-07 08:51:07 -0800494 /*
495 * Cpusets with tasks - existing or newly being attached - can't
Li Zefan1c09b192013-08-21 10:22:28 +0800496 * be changed to have empty cpus_allowed or mems_allowed.
Tejun Heo452477f2013-01-07 08:51:07 -0800497 */
Tejun Heoae8086c2013-01-07 08:51:07 -0800498 ret = -ENOSPC;
Tejun Heo07bc3562014-02-13 06:58:39 -0500499 if ((cgroup_has_tasks(cur->css.cgroup) || cur->attach_in_progress)) {
Li Zefan1c09b192013-08-21 10:22:28 +0800500 if (!cpumask_empty(cur->cpus_allowed) &&
501 cpumask_empty(trial->cpus_allowed))
502 goto out;
503 if (!nodes_empty(cur->mems_allowed) &&
504 nodes_empty(trial->mems_allowed))
505 goto out;
506 }
Paul Jackson020958b2007-10-18 23:40:21 -0700507
Tejun Heoae8086c2013-01-07 08:51:07 -0800508 ret = 0;
509out:
510 rcu_read_unlock();
511 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512}
513
Paul Menagedb7f47c2009-04-02 16:57:55 -0700514#ifdef CONFIG_SMP
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700515/*
Max Krasnyanskycf417142008-08-11 14:33:53 -0700516 * Helper routine for generate_sched_domains().
Li Zefan8b5f1c52014-07-09 16:47:50 +0800517 * Do cpusets a, b have overlapping effective cpus_allowed masks?
Paul Jackson029190c2007-10-18 23:40:20 -0700518 */
Paul Jackson029190c2007-10-18 23:40:20 -0700519static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
520{
Li Zefan8b5f1c52014-07-09 16:47:50 +0800521 return cpumask_intersects(a->effective_cpus, b->effective_cpus);
Paul Jackson029190c2007-10-18 23:40:20 -0700522}
523
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900524static void
525update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
526{
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900527 if (dattr->relax_domain_level < c->relax_domain_level)
528 dattr->relax_domain_level = c->relax_domain_level;
529 return;
530}
531
Tejun Heofc560a22013-01-07 08:51:08 -0800532static void update_domain_attr_tree(struct sched_domain_attr *dattr,
533 struct cpuset *root_cs)
Lai Jiangshanf5393692008-07-29 22:33:22 -0700534{
Tejun Heofc560a22013-01-07 08:51:08 -0800535 struct cpuset *cp;
Tejun Heo492eb212013-08-08 20:11:25 -0400536 struct cgroup_subsys_state *pos_css;
Lai Jiangshanf5393692008-07-29 22:33:22 -0700537
Tejun Heofc560a22013-01-07 08:51:08 -0800538 rcu_read_lock();
Tejun Heo492eb212013-08-08 20:11:25 -0400539 cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
Tejun Heobd8815a2013-08-08 20:11:27 -0400540 if (cp == root_cs)
541 continue;
542
Tejun Heofc560a22013-01-07 08:51:08 -0800543 /* skip the whole subtree if @cp doesn't have any CPU */
544 if (cpumask_empty(cp->cpus_allowed)) {
Tejun Heo492eb212013-08-08 20:11:25 -0400545 pos_css = css_rightmost_descendant(pos_css);
Lai Jiangshanf5393692008-07-29 22:33:22 -0700546 continue;
Tejun Heofc560a22013-01-07 08:51:08 -0800547 }
Lai Jiangshanf5393692008-07-29 22:33:22 -0700548
549 if (is_sched_load_balance(cp))
550 update_domain_attr(dattr, cp);
Lai Jiangshanf5393692008-07-29 22:33:22 -0700551 }
Tejun Heofc560a22013-01-07 08:51:08 -0800552 rcu_read_unlock();
Lai Jiangshanf5393692008-07-29 22:33:22 -0700553}
554
Paul Jackson029190c2007-10-18 23:40:20 -0700555/*
Max Krasnyanskycf417142008-08-11 14:33:53 -0700556 * generate_sched_domains()
Paul Jackson029190c2007-10-18 23:40:20 -0700557 *
Max Krasnyanskycf417142008-08-11 14:33:53 -0700558 * This function builds a partial partition of the systems CPUs
559 * A 'partial partition' is a set of non-overlapping subsets whose
560 * union is a subset of that set.
Viresh Kumar0a0fca92013-06-04 13:10:24 +0530561 * The output of this function needs to be passed to kernel/sched/core.c
Max Krasnyanskycf417142008-08-11 14:33:53 -0700562 * partition_sched_domains() routine, which will rebuild the scheduler's
563 * load balancing domains (sched domains) as specified by that partial
564 * partition.
Paul Jackson029190c2007-10-18 23:40:20 -0700565 *
Li Zefan45ce80f2009-01-15 13:50:59 -0800566 * See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt
Paul Jackson029190c2007-10-18 23:40:20 -0700567 * for a background explanation of this.
568 *
569 * Does not return errors, on the theory that the callers of this
570 * routine would rather not worry about failures to rebuild sched
571 * domains when operating in the severe memory shortage situations
572 * that could cause allocation failures below.
573 *
Tejun Heo5d21cc22013-01-07 08:51:08 -0800574 * Must be called with cpuset_mutex held.
Paul Jackson029190c2007-10-18 23:40:20 -0700575 *
576 * The three key local variables below are:
Li Zefanaeed6822008-07-29 22:33:24 -0700577 * q - a linked-list queue of cpuset pointers, used to implement a
Paul Jackson029190c2007-10-18 23:40:20 -0700578 * top-down scan of all cpusets. This scan loads a pointer
579 * to each cpuset marked is_sched_load_balance into the
580 * array 'csa'. For our purposes, rebuilding the schedulers
581 * sched domains, we can ignore !is_sched_load_balance cpusets.
582 * csa - (for CpuSet Array) Array of pointers to all the cpusets
583 * that need to be load balanced, for convenient iterative
584 * access by the subsequent code that finds the best partition,
585 * i.e the set of domains (subsets) of CPUs such that the
586 * cpus_allowed of every cpuset marked is_sched_load_balance
587 * is a subset of one of these domains, while there are as
588 * many such domains as possible, each as small as possible.
589 * doms - Conversion of 'csa' to an array of cpumasks, for passing to
Viresh Kumar0a0fca92013-06-04 13:10:24 +0530590 * the kernel/sched/core.c routine partition_sched_domains() in a
Paul Jackson029190c2007-10-18 23:40:20 -0700591 * convenient format, that can be easily compared to the prior
592 * value to determine what partition elements (sched domains)
593 * were changed (added or removed.)
594 *
595 * Finding the best partition (set of domains):
596 * The triple nested loops below over i, j, k scan over the
597 * load balanced cpusets (using the array of cpuset pointers in
598 * csa[]) looking for pairs of cpusets that have overlapping
599 * cpus_allowed, but which don't have the same 'pn' partition
600 * number and gives them in the same partition number. It keeps
601 * looping on the 'restart' label until it can no longer find
602 * any such pairs.
603 *
604 * The union of the cpus_allowed masks from the set of
605 * all cpusets having the same 'pn' value then form the one
606 * element of the partition (one sched domain) to be passed to
607 * partition_sched_domains().
608 */
Rusty Russellacc3f5d2009-11-03 14:53:40 +1030609static int generate_sched_domains(cpumask_var_t **domains,
Max Krasnyanskycf417142008-08-11 14:33:53 -0700610 struct sched_domain_attr **attributes)
Paul Jackson029190c2007-10-18 23:40:20 -0700611{
Paul Jackson029190c2007-10-18 23:40:20 -0700612 struct cpuset *cp; /* scans q */
613 struct cpuset **csa; /* array of all cpuset ptrs */
614 int csn; /* how many cpuset ptrs in csa so far */
615 int i, j, k; /* indices for partition finding loops */
Rusty Russellacc3f5d2009-11-03 14:53:40 +1030616 cpumask_var_t *doms; /* resulting partition; i.e. sched domains */
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900617 struct sched_domain_attr *dattr; /* attributes for custom domains */
Ingo Molnar15837152008-11-25 10:27:49 +0100618 int ndoms = 0; /* number of sched domains in result */
Li Zefan6af866a2009-01-07 18:08:45 -0800619 int nslot; /* next empty doms[] struct cpumask slot */
Tejun Heo492eb212013-08-08 20:11:25 -0400620 struct cgroup_subsys_state *pos_css;
Paul Jackson029190c2007-10-18 23:40:20 -0700621
Paul Jackson029190c2007-10-18 23:40:20 -0700622 doms = NULL;
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900623 dattr = NULL;
Max Krasnyanskycf417142008-08-11 14:33:53 -0700624 csa = NULL;
Paul Jackson029190c2007-10-18 23:40:20 -0700625
626 /* Special case for the 99% of systems with one, full, sched domain */
627 if (is_sched_load_balance(&top_cpuset)) {
Rusty Russellacc3f5d2009-11-03 14:53:40 +1030628 ndoms = 1;
629 doms = alloc_sched_domains(ndoms);
Paul Jackson029190c2007-10-18 23:40:20 -0700630 if (!doms)
Max Krasnyanskycf417142008-08-11 14:33:53 -0700631 goto done;
632
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900633 dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
634 if (dattr) {
635 *dattr = SD_ATTR_INIT;
Li Zefan93a65572008-07-29 22:33:23 -0700636 update_domain_attr_tree(dattr, &top_cpuset);
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900637 }
Li Zefan8b5f1c52014-07-09 16:47:50 +0800638 cpumask_copy(doms[0], top_cpuset.effective_cpus);
Max Krasnyanskycf417142008-08-11 14:33:53 -0700639
Max Krasnyanskycf417142008-08-11 14:33:53 -0700640 goto done;
Paul Jackson029190c2007-10-18 23:40:20 -0700641 }
642
Mel Gorman664eedd2014-06-04 16:10:08 -0700643 csa = kmalloc(nr_cpusets() * sizeof(cp), GFP_KERNEL);
Paul Jackson029190c2007-10-18 23:40:20 -0700644 if (!csa)
645 goto done;
646 csn = 0;
647
Tejun Heofc560a22013-01-07 08:51:08 -0800648 rcu_read_lock();
Tejun Heo492eb212013-08-08 20:11:25 -0400649 cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
Tejun Heobd8815a2013-08-08 20:11:27 -0400650 if (cp == &top_cpuset)
651 continue;
Lai Jiangshanf5393692008-07-29 22:33:22 -0700652 /*
Tejun Heofc560a22013-01-07 08:51:08 -0800653 * Continue traversing beyond @cp iff @cp has some CPUs and
654 * isn't load balancing. The former is obvious. The
655 * latter: All child cpusets contain a subset of the
656 * parent's cpus, so just skip them, and then we call
657 * update_domain_attr_tree() to calc relax_domain_level of
658 * the corresponding sched domain.
Lai Jiangshanf5393692008-07-29 22:33:22 -0700659 */
Tejun Heofc560a22013-01-07 08:51:08 -0800660 if (!cpumask_empty(cp->cpus_allowed) &&
661 !is_sched_load_balance(cp))
Lai Jiangshanf5393692008-07-29 22:33:22 -0700662 continue;
Lai Jiangshan489a5392008-07-25 01:47:23 -0700663
Tejun Heofc560a22013-01-07 08:51:08 -0800664 if (is_sched_load_balance(cp))
665 csa[csn++] = cp;
666
667 /* skip @cp's subtree */
Tejun Heo492eb212013-08-08 20:11:25 -0400668 pos_css = css_rightmost_descendant(pos_css);
Tejun Heofc560a22013-01-07 08:51:08 -0800669 }
670 rcu_read_unlock();
Paul Jackson029190c2007-10-18 23:40:20 -0700671
672 for (i = 0; i < csn; i++)
673 csa[i]->pn = i;
674 ndoms = csn;
675
676restart:
677 /* Find the best partition (set of sched domains) */
678 for (i = 0; i < csn; i++) {
679 struct cpuset *a = csa[i];
680 int apn = a->pn;
681
682 for (j = 0; j < csn; j++) {
683 struct cpuset *b = csa[j];
684 int bpn = b->pn;
685
686 if (apn != bpn && cpusets_overlap(a, b)) {
687 for (k = 0; k < csn; k++) {
688 struct cpuset *c = csa[k];
689
690 if (c->pn == bpn)
691 c->pn = apn;
692 }
693 ndoms--; /* one less element */
694 goto restart;
695 }
696 }
697 }
698
Max Krasnyanskycf417142008-08-11 14:33:53 -0700699 /*
700 * Now we know how many domains to create.
701 * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
702 */
Rusty Russellacc3f5d2009-11-03 14:53:40 +1030703 doms = alloc_sched_domains(ndoms);
Li Zefan700018e2008-11-18 14:02:03 +0800704 if (!doms)
Max Krasnyanskycf417142008-08-11 14:33:53 -0700705 goto done;
Max Krasnyanskycf417142008-08-11 14:33:53 -0700706
707 /*
708 * The rest of the code, including the scheduler, can deal with
709 * dattr==NULL case. No need to abort if alloc fails.
710 */
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +0900711 dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);
Paul Jackson029190c2007-10-18 23:40:20 -0700712
713 for (nslot = 0, i = 0; i < csn; i++) {
714 struct cpuset *a = csa[i];
Li Zefan6af866a2009-01-07 18:08:45 -0800715 struct cpumask *dp;
Paul Jackson029190c2007-10-18 23:40:20 -0700716 int apn = a->pn;
717
Max Krasnyanskycf417142008-08-11 14:33:53 -0700718 if (apn < 0) {
719 /* Skip completed partitions */
720 continue;
Paul Jackson029190c2007-10-18 23:40:20 -0700721 }
Max Krasnyanskycf417142008-08-11 14:33:53 -0700722
Rusty Russellacc3f5d2009-11-03 14:53:40 +1030723 dp = doms[nslot];
Max Krasnyanskycf417142008-08-11 14:33:53 -0700724
725 if (nslot == ndoms) {
726 static int warnings = 10;
727 if (warnings) {
Fabian Frederick12d30892014-05-05 19:49:00 +0200728 pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n",
729 nslot, ndoms, csn, i, apn);
Max Krasnyanskycf417142008-08-11 14:33:53 -0700730 warnings--;
731 }
732 continue;
733 }
734
Li Zefan6af866a2009-01-07 18:08:45 -0800735 cpumask_clear(dp);
Max Krasnyanskycf417142008-08-11 14:33:53 -0700736 if (dattr)
737 *(dattr + nslot) = SD_ATTR_INIT;
738 for (j = i; j < csn; j++) {
739 struct cpuset *b = csa[j];
740
741 if (apn == b->pn) {
Li Zefan8b5f1c52014-07-09 16:47:50 +0800742 cpumask_or(dp, dp, b->effective_cpus);
Max Krasnyanskycf417142008-08-11 14:33:53 -0700743 if (dattr)
744 update_domain_attr_tree(dattr + nslot, b);
745
746 /* Done with this partition */
747 b->pn = -1;
748 }
749 }
750 nslot++;
Paul Jackson029190c2007-10-18 23:40:20 -0700751 }
752 BUG_ON(nslot != ndoms);
753
Paul Jackson029190c2007-10-18 23:40:20 -0700754done:
Paul Jackson029190c2007-10-18 23:40:20 -0700755 kfree(csa);
Max Krasnyanskycf417142008-08-11 14:33:53 -0700756
Li Zefan700018e2008-11-18 14:02:03 +0800757 /*
758 * Fallback to the default domain if kmalloc() failed.
759 * See comments in partition_sched_domains().
760 */
761 if (doms == NULL)
762 ndoms = 1;
763
Max Krasnyanskycf417142008-08-11 14:33:53 -0700764 *domains = doms;
765 *attributes = dattr;
766 return ndoms;
767}
768
769/*
770 * Rebuild scheduler domains.
771 *
Tejun Heo699140b2013-01-07 08:51:07 -0800772 * If the flag 'sched_load_balance' of any cpuset with non-empty
773 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
774 * which has that flag enabled, or if any cpuset with a non-empty
775 * 'cpus' is removed, then call this routine to rebuild the
776 * scheduler's dynamic sched domains.
Max Krasnyanskycf417142008-08-11 14:33:53 -0700777 *
Tejun Heo5d21cc22013-01-07 08:51:08 -0800778 * Call with cpuset_mutex held. Takes get_online_cpus().
Max Krasnyanskycf417142008-08-11 14:33:53 -0700779 */
Tejun Heo699140b2013-01-07 08:51:07 -0800780static void rebuild_sched_domains_locked(void)
Max Krasnyanskycf417142008-08-11 14:33:53 -0700781{
782 struct sched_domain_attr *attr;
Rusty Russellacc3f5d2009-11-03 14:53:40 +1030783 cpumask_var_t *doms;
Max Krasnyanskycf417142008-08-11 14:33:53 -0700784 int ndoms;
785
Tejun Heo5d21cc22013-01-07 08:51:08 -0800786 lockdep_assert_held(&cpuset_mutex);
Max Krasnyanskycf417142008-08-11 14:33:53 -0700787 get_online_cpus();
788
Li Zefan5b16c2a2013-04-27 06:52:43 -0700789 /*
790 * We have raced with CPU hotplug. Don't do anything to avoid
791 * passing doms with offlined cpu to partition_sched_domains().
792 * Anyways, hotplug work item will rebuild sched domains.
793 */
Li Zefan8b5f1c52014-07-09 16:47:50 +0800794 if (!cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask))
Li Zefan5b16c2a2013-04-27 06:52:43 -0700795 goto out;
796
Max Krasnyanskycf417142008-08-11 14:33:53 -0700797 /* Generate domain masks and attrs */
Max Krasnyanskycf417142008-08-11 14:33:53 -0700798 ndoms = generate_sched_domains(&doms, &attr);
Max Krasnyanskycf417142008-08-11 14:33:53 -0700799
800 /* Have scheduler rebuild the domains */
801 partition_sched_domains(ndoms, doms, attr);
Li Zefan5b16c2a2013-04-27 06:52:43 -0700802out:
Max Krasnyanskycf417142008-08-11 14:33:53 -0700803 put_online_cpus();
804}
Paul Menagedb7f47c2009-04-02 16:57:55 -0700805#else /* !CONFIG_SMP */
Tejun Heo699140b2013-01-07 08:51:07 -0800806static void rebuild_sched_domains_locked(void)
Paul Menagedb7f47c2009-04-02 16:57:55 -0700807{
808}
Paul Menagedb7f47c2009-04-02 16:57:55 -0700809#endif /* CONFIG_SMP */
Max Krasnyanskycf417142008-08-11 14:33:53 -0700810
Max Krasnyanskycf417142008-08-11 14:33:53 -0700811void rebuild_sched_domains(void)
812{
Tejun Heo5d21cc22013-01-07 08:51:08 -0800813 mutex_lock(&cpuset_mutex);
Tejun Heo699140b2013-01-07 08:51:07 -0800814 rebuild_sched_domains_locked();
Tejun Heo5d21cc22013-01-07 08:51:08 -0800815 mutex_unlock(&cpuset_mutex);
Paul Jackson029190c2007-10-18 23:40:20 -0700816}
817
Cliff Wickman58f47902008-02-07 00:14:44 -0800818/**
Miao Xie0b2f6302008-07-25 01:47:21 -0700819 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
820 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
821 *
Tejun Heod66393e2014-02-13 06:58:40 -0500822 * Iterate through each task of @cs updating its cpus_allowed to the
823 * effective cpuset's. As this function is called with cpuset_mutex held,
824 * cpuset membership stays stable.
Miao Xie0b2f6302008-07-25 01:47:21 -0700825 */
Tejun Heod66393e2014-02-13 06:58:40 -0500826static void update_tasks_cpumask(struct cpuset *cs)
Miao Xie0b2f6302008-07-25 01:47:21 -0700827{
Tejun Heod66393e2014-02-13 06:58:40 -0500828 struct css_task_iter it;
829 struct task_struct *task;
830
831 css_task_iter_start(&cs->css, &it);
832 while ((task = css_task_iter_next(&it)))
Li Zefanae1c8022014-07-09 16:48:32 +0800833 set_cpus_allowed_ptr(task, cs->effective_cpus);
Tejun Heod66393e2014-02-13 06:58:40 -0500834 css_task_iter_end(&it);
Miao Xie0b2f6302008-07-25 01:47:21 -0700835}
836
Li Zefan5c5cc622013-06-09 17:16:29 +0800837/*
Li Zefan734d4512014-07-09 16:47:29 +0800838 * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
839 * @cs: the cpuset to consider
840 * @new_cpus: temp variable for calculating new effective_cpus
Li Zefan5c5cc622013-06-09 17:16:29 +0800841 *
Li Zefan734d4512014-07-09 16:47:29 +0800842 * When congifured cpumask is changed, the effective cpumasks of this cpuset
843 * and all its descendants need to be updated.
844 *
845 * On legacy hierachy, effective_cpus will be the same with cpu_allowed.
Li Zefan5c5cc622013-06-09 17:16:29 +0800846 *
847 * Called with cpuset_mutex held
848 */
Li Zefan734d4512014-07-09 16:47:29 +0800849static void update_cpumasks_hier(struct cpuset *cs, struct cpumask *new_cpus)
Li Zefan5c5cc622013-06-09 17:16:29 +0800850{
851 struct cpuset *cp;
Tejun Heo492eb212013-08-08 20:11:25 -0400852 struct cgroup_subsys_state *pos_css;
Li Zefan8b5f1c52014-07-09 16:47:50 +0800853 bool need_rebuild_sched_domains = false;
Li Zefan5c5cc622013-06-09 17:16:29 +0800854
855 rcu_read_lock();
Li Zefan734d4512014-07-09 16:47:29 +0800856 cpuset_for_each_descendant_pre(cp, pos_css, cs) {
857 struct cpuset *parent = parent_cs(cp);
858
859 cpumask_and(new_cpus, cp->cpus_allowed, parent->effective_cpus);
860
Li Zefan554b0d12014-07-09 16:47:41 +0800861 /*
862 * If it becomes empty, inherit the effective mask of the
863 * parent, which is guaranteed to have some CPUs.
864 */
865 if (cpumask_empty(new_cpus))
866 cpumask_copy(new_cpus, parent->effective_cpus);
867
Li Zefan734d4512014-07-09 16:47:29 +0800868 /* Skip the whole subtree if the cpumask remains the same. */
869 if (cpumask_equal(new_cpus, cp->effective_cpus)) {
870 pos_css = css_rightmost_descendant(pos_css);
871 continue;
Li Zefan5c5cc622013-06-09 17:16:29 +0800872 }
Li Zefan734d4512014-07-09 16:47:29 +0800873
Tejun Heoec903c02014-05-13 12:11:01 -0400874 if (!css_tryget_online(&cp->css))
Li Zefan5c5cc622013-06-09 17:16:29 +0800875 continue;
876 rcu_read_unlock();
877
Li Zefan734d4512014-07-09 16:47:29 +0800878 mutex_lock(&callback_mutex);
879 cpumask_copy(cp->effective_cpus, new_cpus);
880 mutex_unlock(&callback_mutex);
881
882 WARN_ON(!cgroup_on_dfl(cp->css.cgroup) &&
883 !cpumask_equal(cp->cpus_allowed, cp->effective_cpus));
884
Tejun Heod66393e2014-02-13 06:58:40 -0500885 update_tasks_cpumask(cp);
Li Zefan5c5cc622013-06-09 17:16:29 +0800886
Li Zefan8b5f1c52014-07-09 16:47:50 +0800887 /*
888 * If the effective cpumask of any non-empty cpuset is changed,
889 * we need to rebuild sched domains.
890 */
891 if (!cpumask_empty(cp->cpus_allowed) &&
892 is_sched_load_balance(cp))
893 need_rebuild_sched_domains = true;
894
Li Zefan5c5cc622013-06-09 17:16:29 +0800895 rcu_read_lock();
896 css_put(&cp->css);
897 }
898 rcu_read_unlock();
Li Zefan8b5f1c52014-07-09 16:47:50 +0800899
900 if (need_rebuild_sched_domains)
901 rebuild_sched_domains_locked();
Li Zefan5c5cc622013-06-09 17:16:29 +0800902}
903
Miao Xie0b2f6302008-07-25 01:47:21 -0700904/**
Cliff Wickman58f47902008-02-07 00:14:44 -0800905 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
906 * @cs: the cpuset to consider
Fabian Frederickfc34ac12014-05-05 19:46:55 +0200907 * @trialcs: trial cpuset
Cliff Wickman58f47902008-02-07 00:14:44 -0800908 * @buf: buffer of cpu numbers written to this cpuset
909 */
Li Zefan645fcc92009-01-07 18:08:43 -0800910static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
911 const char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912{
Cliff Wickman58f47902008-02-07 00:14:44 -0800913 int retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
Rusty Russell5f054e32012-03-29 15:38:31 +1030915 /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
Paul Jackson4c4d50f2006-08-27 01:23:51 -0700916 if (cs == &top_cpuset)
917 return -EACCES;
918
David Rientjes6f7f02e2007-05-08 00:31:43 -0700919 /*
Paul Jacksonc8d9c902008-02-07 00:14:46 -0800920 * An empty cpus_allowed is ok only if the cpuset has no tasks.
Paul Jackson020958b2007-10-18 23:40:21 -0700921 * Since cpulist_parse() fails on an empty mask, we special case
922 * that parsing. The validate_change() call ensures that cpusets
923 * with tasks have cpus.
David Rientjes6f7f02e2007-05-08 00:31:43 -0700924 */
Paul Jackson020958b2007-10-18 23:40:21 -0700925 if (!*buf) {
Li Zefan300ed6c2009-01-07 18:08:44 -0800926 cpumask_clear(trialcs->cpus_allowed);
David Rientjes6f7f02e2007-05-08 00:31:43 -0700927 } else {
Li Zefan300ed6c2009-01-07 18:08:44 -0800928 retval = cpulist_parse(buf, trialcs->cpus_allowed);
David Rientjes6f7f02e2007-05-08 00:31:43 -0700929 if (retval < 0)
930 return retval;
Lai Jiangshan37340742008-06-05 22:46:32 -0700931
Li Zefan5d8ba822014-07-09 16:49:12 +0800932 if (!cpumask_subset(trialcs->cpus_allowed,
933 top_cpuset.cpus_allowed))
Lai Jiangshan37340742008-06-05 22:46:32 -0700934 return -EINVAL;
David Rientjes6f7f02e2007-05-08 00:31:43 -0700935 }
Paul Jackson029190c2007-10-18 23:40:20 -0700936
Paul Menage8707d8b2007-10-18 23:40:22 -0700937 /* Nothing to do if the cpus didn't change */
Li Zefan300ed6c2009-01-07 18:08:44 -0800938 if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
Paul Menage8707d8b2007-10-18 23:40:22 -0700939 return 0;
Cliff Wickman58f47902008-02-07 00:14:44 -0800940
Li Zefana73456f2013-06-05 17:15:59 +0800941 retval = validate_change(cs, trialcs);
942 if (retval < 0)
943 return retval;
944
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800945 mutex_lock(&callback_mutex);
Li Zefan300ed6c2009-01-07 18:08:44 -0800946 cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
Ingo Molnar3d3f26a2006-03-23 03:00:18 -0800947 mutex_unlock(&callback_mutex);
Paul Jackson029190c2007-10-18 23:40:20 -0700948
Li Zefan734d4512014-07-09 16:47:29 +0800949 /* use trialcs->cpus_allowed as a temp variable */
950 update_cpumasks_hier(cs, trialcs->cpus_allowed);
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700951 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952}
953
Paul Jackson053199e2005-10-30 15:02:30 -0800954/*
Paul Jacksone4e364e2006-03-31 02:30:52 -0800955 * cpuset_migrate_mm
956 *
957 * Migrate memory region from one set of nodes to another.
958 *
959 * Temporarilly set tasks mems_allowed to target nodes of migration,
960 * so that the migration code can allocate pages on these nodes.
961 *
Paul Jacksone4e364e2006-03-31 02:30:52 -0800962 * While the mm_struct we are migrating is typically from some
963 * other task, the task_struct mems_allowed that we are hacking
964 * is for our current task, which must allocate new pages for that
965 * migrating memory region.
Paul Jacksone4e364e2006-03-31 02:30:52 -0800966 */
967
968static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
969 const nodemask_t *to)
970{
971 struct task_struct *tsk = current;
972
Paul Jacksone4e364e2006-03-31 02:30:52 -0800973 tsk->mems_allowed = *to;
Paul Jacksone4e364e2006-03-31 02:30:52 -0800974
975 do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);
976
Li Zefan47295832014-02-27 18:19:03 +0800977 rcu_read_lock();
Li Zefanae1c8022014-07-09 16:48:32 +0800978 guarantee_online_mems(task_cs(tsk), &tsk->mems_allowed);
Li Zefan47295832014-02-27 18:19:03 +0800979 rcu_read_unlock();
Paul Jacksone4e364e2006-03-31 02:30:52 -0800980}
981
Li Zefan3b6766f2009-04-02 16:57:51 -0700982/*
Miao Xie58568d22009-06-16 15:31:49 -0700983 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
984 * @tsk: the task to change
985 * @newmems: new nodes that the task will be set
986 *
987 * In order to avoid seeing no nodes if the old and new nodes are disjoint,
988 * we structure updates as setting all new allowed nodes, then clearing newly
989 * disallowed ones.
Miao Xie58568d22009-06-16 15:31:49 -0700990 */
991static void cpuset_change_task_nodemask(struct task_struct *tsk,
992 nodemask_t *newmems)
993{
David Rientjesb2462722011-12-19 17:11:52 -0800994 bool need_loop;
David Rientjes89e8a242011-11-02 13:38:39 -0700995
Miao Xiec0ff7452010-05-24 14:32:08 -0700996 /*
997 * Allow tasks that have access to memory reserves because they have
998 * been OOM killed to get memory anywhere.
999 */
1000 if (unlikely(test_thread_flag(TIF_MEMDIE)))
1001 return;
1002 if (current->flags & PF_EXITING) /* Let dying task have memory */
1003 return;
1004
1005 task_lock(tsk);
David Rientjesb2462722011-12-19 17:11:52 -08001006 /*
1007 * Determine if a loop is necessary if another thread is doing
Mel Gormand26914d2014-04-03 14:47:24 -07001008 * read_mems_allowed_begin(). If at least one node remains unchanged and
David Rientjesb2462722011-12-19 17:11:52 -08001009 * tsk does not have a mempolicy, then an empty nodemask will not be
1010 * possible when mems_allowed is larger than a word.
1011 */
1012 need_loop = task_has_mempolicy(tsk) ||
1013 !nodes_intersects(*newmems, tsk->mems_allowed);
Mel Gormancc9a6c82012-03-21 16:34:11 -07001014
Peter Zijlstra0fc02872013-11-26 15:03:41 +01001015 if (need_loop) {
1016 local_irq_disable();
Mel Gormancc9a6c82012-03-21 16:34:11 -07001017 write_seqcount_begin(&tsk->mems_allowed_seq);
Peter Zijlstra0fc02872013-11-26 15:03:41 +01001018 }
Mel Gormancc9a6c82012-03-21 16:34:11 -07001019
Miao Xie58568d22009-06-16 15:31:49 -07001020 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
Miao Xiec0ff7452010-05-24 14:32:08 -07001021 mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1);
1022
Miao Xiec0ff7452010-05-24 14:32:08 -07001023 mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2);
Miao Xie58568d22009-06-16 15:31:49 -07001024 tsk->mems_allowed = *newmems;
Mel Gormancc9a6c82012-03-21 16:34:11 -07001025
Peter Zijlstra0fc02872013-11-26 15:03:41 +01001026 if (need_loop) {
Mel Gormancc9a6c82012-03-21 16:34:11 -07001027 write_seqcount_end(&tsk->mems_allowed_seq);
Peter Zijlstra0fc02872013-11-26 15:03:41 +01001028 local_irq_enable();
1029 }
Mel Gormancc9a6c82012-03-21 16:34:11 -07001030
Miao Xiec0ff7452010-05-24 14:32:08 -07001031 task_unlock(tsk);
Miao Xie58568d22009-06-16 15:31:49 -07001032}
1033
Paul Menage8793d852007-10-18 23:39:39 -07001034static void *cpuset_being_rebound;
1035
Miao Xie0b2f6302008-07-25 01:47:21 -07001036/**
1037 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
1038 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
Miao Xie0b2f6302008-07-25 01:47:21 -07001039 *
Tejun Heod66393e2014-02-13 06:58:40 -05001040 * Iterate through each task of @cs updating its mems_allowed to the
1041 * effective cpuset's. As this function is called with cpuset_mutex held,
1042 * cpuset membership stays stable.
Miao Xie0b2f6302008-07-25 01:47:21 -07001043 */
Tejun Heod66393e2014-02-13 06:58:40 -05001044static void update_tasks_nodemask(struct cpuset *cs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045{
Li Zefan33ad8012013-06-09 17:15:08 +08001046 static nodemask_t newmems; /* protected by cpuset_mutex */
Tejun Heod66393e2014-02-13 06:58:40 -05001047 struct css_task_iter it;
1048 struct task_struct *task;
Paul Jackson59dac162006-01-08 01:01:52 -08001049
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001050 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
Paul Jackson42253992006-01-08 01:01:59 -08001051
Li Zefanae1c8022014-07-09 16:48:32 +08001052 guarantee_online_mems(cs, &newmems);
Li Zefan33ad8012013-06-09 17:15:08 +08001053
Paul Jackson42253992006-01-08 01:01:59 -08001054 /*
Li Zefan3b6766f2009-04-02 16:57:51 -07001055 * The mpol_rebind_mm() call takes mmap_sem, which we couldn't
1056 * take while holding tasklist_lock. Forks can happen - the
1057 * mpol_dup() cpuset_being_rebound check will catch such forks,
1058 * and rebind their vma mempolicies too. Because we still hold
Tejun Heo5d21cc22013-01-07 08:51:08 -08001059 * the global cpuset_mutex, we know that no other rebind effort
Li Zefan3b6766f2009-04-02 16:57:51 -07001060 * will be contending for the global variable cpuset_being_rebound.
Paul Jackson42253992006-01-08 01:01:59 -08001061 * It's ok if we rebind the same mm twice; mpol_rebind_mm()
Paul Jackson04c19fa2006-01-08 01:02:00 -08001062 * is idempotent. Also migrate pages in each mm to new nodes.
Paul Jackson42253992006-01-08 01:01:59 -08001063 */
Tejun Heod66393e2014-02-13 06:58:40 -05001064 css_task_iter_start(&cs->css, &it);
1065 while ((task = css_task_iter_next(&it))) {
1066 struct mm_struct *mm;
1067 bool migrate;
1068
1069 cpuset_change_task_nodemask(task, &newmems);
1070
1071 mm = get_task_mm(task);
1072 if (!mm)
1073 continue;
1074
1075 migrate = is_memory_migrate(cs);
1076
1077 mpol_rebind_mm(mm, &cs->mems_allowed);
1078 if (migrate)
1079 cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
1080 mmput(mm);
1081 }
1082 css_task_iter_end(&it);
Paul Jackson42253992006-01-08 01:01:59 -08001083
Li Zefan33ad8012013-06-09 17:15:08 +08001084 /*
1085 * All the tasks' nodemasks have been updated, update
1086 * cs->old_mems_allowed.
1087 */
1088 cs->old_mems_allowed = newmems;
1089
Paul Menage2df167a2008-02-07 00:14:45 -08001090 /* We're done rebinding vmas to this cpuset's new mems_allowed. */
Paul Menage8793d852007-10-18 23:39:39 -07001091 cpuset_being_rebound = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092}
1093
Miao Xie0b2f6302008-07-25 01:47:21 -07001094/*
Li Zefan734d4512014-07-09 16:47:29 +08001095 * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
1096 * @cs: the cpuset to consider
1097 * @new_mems: a temp variable for calculating new effective_mems
Li Zefan5c5cc622013-06-09 17:16:29 +08001098 *
Li Zefan734d4512014-07-09 16:47:29 +08001099 * When configured nodemask is changed, the effective nodemasks of this cpuset
1100 * and all its descendants need to be updated.
1101 *
1102 * On legacy hiearchy, effective_mems will be the same with mems_allowed.
Li Zefan5c5cc622013-06-09 17:16:29 +08001103 *
1104 * Called with cpuset_mutex held
1105 */
Li Zefan734d4512014-07-09 16:47:29 +08001106static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
Li Zefan5c5cc622013-06-09 17:16:29 +08001107{
1108 struct cpuset *cp;
Tejun Heo492eb212013-08-08 20:11:25 -04001109 struct cgroup_subsys_state *pos_css;
Li Zefan5c5cc622013-06-09 17:16:29 +08001110
1111 rcu_read_lock();
Li Zefan734d4512014-07-09 16:47:29 +08001112 cpuset_for_each_descendant_pre(cp, pos_css, cs) {
1113 struct cpuset *parent = parent_cs(cp);
1114
1115 nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems);
1116
Li Zefan554b0d12014-07-09 16:47:41 +08001117 /*
1118 * If it becomes empty, inherit the effective mask of the
1119 * parent, which is guaranteed to have some MEMs.
1120 */
1121 if (nodes_empty(*new_mems))
1122 *new_mems = parent->effective_mems;
1123
Li Zefan734d4512014-07-09 16:47:29 +08001124 /* Skip the whole subtree if the nodemask remains the same. */
1125 if (nodes_equal(*new_mems, cp->effective_mems)) {
1126 pos_css = css_rightmost_descendant(pos_css);
1127 continue;
Li Zefan5c5cc622013-06-09 17:16:29 +08001128 }
Li Zefan734d4512014-07-09 16:47:29 +08001129
Tejun Heoec903c02014-05-13 12:11:01 -04001130 if (!css_tryget_online(&cp->css))
Li Zefan5c5cc622013-06-09 17:16:29 +08001131 continue;
1132 rcu_read_unlock();
1133
Li Zefan734d4512014-07-09 16:47:29 +08001134 mutex_lock(&callback_mutex);
1135 cp->effective_mems = *new_mems;
1136 mutex_unlock(&callback_mutex);
1137
1138 WARN_ON(!cgroup_on_dfl(cp->css.cgroup) &&
Li Zefana1381262014-07-30 15:07:13 +08001139 !nodes_equal(cp->mems_allowed, cp->effective_mems));
Li Zefan734d4512014-07-09 16:47:29 +08001140
Tejun Heod66393e2014-02-13 06:58:40 -05001141 update_tasks_nodemask(cp);
Li Zefan5c5cc622013-06-09 17:16:29 +08001142
1143 rcu_read_lock();
1144 css_put(&cp->css);
1145 }
1146 rcu_read_unlock();
1147}
1148
1149/*
Miao Xie0b2f6302008-07-25 01:47:21 -07001150 * Handle user request to change the 'mems' memory placement
1151 * of a cpuset. Needs to validate the request, update the
Miao Xie58568d22009-06-16 15:31:49 -07001152 * cpusets mems_allowed, and for each task in the cpuset,
1153 * update mems_allowed and rebind task's mempolicy and any vma
1154 * mempolicies and if the cpuset is marked 'memory_migrate',
1155 * migrate the tasks pages to the new memory.
Miao Xie0b2f6302008-07-25 01:47:21 -07001156 *
Tejun Heo5d21cc22013-01-07 08:51:08 -08001157 * Call with cpuset_mutex held. May take callback_mutex during call.
Miao Xie0b2f6302008-07-25 01:47:21 -07001158 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
1159 * lock each such tasks mm->mmap_sem, scan its vma's and rebind
1160 * their mempolicies to the cpusets new mems_allowed.
1161 */
Li Zefan645fcc92009-01-07 18:08:43 -08001162static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
1163 const char *buf)
Miao Xie0b2f6302008-07-25 01:47:21 -07001164{
Miao Xie0b2f6302008-07-25 01:47:21 -07001165 int retval;
1166
1167 /*
Lai Jiangshan38d7bee2012-12-12 13:51:24 -08001168 * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
Miao Xie0b2f6302008-07-25 01:47:21 -07001169 * it's read-only
1170 */
Miao Xie53feb292010-03-23 13:35:35 -07001171 if (cs == &top_cpuset) {
1172 retval = -EACCES;
1173 goto done;
1174 }
Miao Xie0b2f6302008-07-25 01:47:21 -07001175
Miao Xie0b2f6302008-07-25 01:47:21 -07001176 /*
1177 * An empty mems_allowed is ok iff there are no tasks in the cpuset.
1178 * Since nodelist_parse() fails on an empty mask, we special case
1179 * that parsing. The validate_change() call ensures that cpusets
1180 * with tasks have memory.
1181 */
1182 if (!*buf) {
Li Zefan645fcc92009-01-07 18:08:43 -08001183 nodes_clear(trialcs->mems_allowed);
Miao Xie0b2f6302008-07-25 01:47:21 -07001184 } else {
Li Zefan645fcc92009-01-07 18:08:43 -08001185 retval = nodelist_parse(buf, trialcs->mems_allowed);
Miao Xie0b2f6302008-07-25 01:47:21 -07001186 if (retval < 0)
1187 goto done;
1188
Li Zefan645fcc92009-01-07 18:08:43 -08001189 if (!nodes_subset(trialcs->mems_allowed,
Li Zefan5d8ba822014-07-09 16:49:12 +08001190 top_cpuset.mems_allowed)) {
1191 retval = -EINVAL;
Miao Xie53feb292010-03-23 13:35:35 -07001192 goto done;
1193 }
Miao Xie0b2f6302008-07-25 01:47:21 -07001194 }
Li Zefan33ad8012013-06-09 17:15:08 +08001195
1196 if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) {
Miao Xie0b2f6302008-07-25 01:47:21 -07001197 retval = 0; /* Too easy - nothing to do */
1198 goto done;
1199 }
Li Zefan645fcc92009-01-07 18:08:43 -08001200 retval = validate_change(cs, trialcs);
Miao Xie0b2f6302008-07-25 01:47:21 -07001201 if (retval < 0)
1202 goto done;
1203
1204 mutex_lock(&callback_mutex);
Li Zefan645fcc92009-01-07 18:08:43 -08001205 cs->mems_allowed = trialcs->mems_allowed;
Miao Xie0b2f6302008-07-25 01:47:21 -07001206 mutex_unlock(&callback_mutex);
1207
Li Zefan734d4512014-07-09 16:47:29 +08001208 /* use trialcs->mems_allowed as a temp variable */
1209 update_nodemasks_hier(cs, &cs->mems_allowed);
Miao Xie0b2f6302008-07-25 01:47:21 -07001210done:
1211 return retval;
1212}
1213
Paul Menage8793d852007-10-18 23:39:39 -07001214int current_cpuset_is_being_rebound(void)
1215{
Gu Zheng391acf92014-06-25 09:57:18 +08001216 int ret;
1217
1218 rcu_read_lock();
1219 ret = task_cs(current) == cpuset_being_rebound;
1220 rcu_read_unlock();
1221
1222 return ret;
Paul Menage8793d852007-10-18 23:39:39 -07001223}
1224
Paul Menage5be7a472008-05-06 20:42:41 -07001225static int update_relax_domain_level(struct cpuset *cs, s64 val)
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09001226{
Paul Menagedb7f47c2009-04-02 16:57:55 -07001227#ifdef CONFIG_SMP
Peter Zijlstra60495e72011-04-07 14:10:04 +02001228 if (val < -1 || val >= sched_domain_level_max)
Li Zefan30e0e172008-05-13 10:27:17 +08001229 return -EINVAL;
Paul Menagedb7f47c2009-04-02 16:57:55 -07001230#endif
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09001231
1232 if (val != cs->relax_domain_level) {
1233 cs->relax_domain_level = val;
Li Zefan300ed6c2009-01-07 18:08:44 -08001234 if (!cpumask_empty(cs->cpus_allowed) &&
1235 is_sched_load_balance(cs))
Tejun Heo699140b2013-01-07 08:51:07 -08001236 rebuild_sched_domains_locked();
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09001237 }
1238
1239 return 0;
1240}
1241
Tejun Heo72ec7022013-08-08 20:11:26 -04001242/**
Miao Xie950592f2009-06-16 15:31:47 -07001243 * update_tasks_flags - update the spread flags of tasks in the cpuset.
1244 * @cs: the cpuset in which each task's spread flags needs to be changed
Miao Xie950592f2009-06-16 15:31:47 -07001245 *
Tejun Heod66393e2014-02-13 06:58:40 -05001246 * Iterate through each task of @cs updating its spread flags. As this
1247 * function is called with cpuset_mutex held, cpuset membership stays
1248 * stable.
Miao Xie950592f2009-06-16 15:31:47 -07001249 */
Tejun Heod66393e2014-02-13 06:58:40 -05001250static void update_tasks_flags(struct cpuset *cs)
Miao Xie950592f2009-06-16 15:31:47 -07001251{
Tejun Heod66393e2014-02-13 06:58:40 -05001252 struct css_task_iter it;
1253 struct task_struct *task;
1254
1255 css_task_iter_start(&cs->css, &it);
1256 while ((task = css_task_iter_next(&it)))
1257 cpuset_update_task_spread_flag(cs, task);
1258 css_task_iter_end(&it);
Miao Xie950592f2009-06-16 15:31:47 -07001259}
1260
1261/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262 * update_flag - read a 0 or a 1 in a file and update associated flag
Paul Menage78608362008-04-29 01:00:26 -07001263 * bit: the bit to update (see cpuset_flagbits_t)
1264 * cs: the cpuset to update
1265 * turning_on: whether the flag is being set or cleared
Paul Jackson053199e2005-10-30 15:02:30 -08001266 *
Tejun Heo5d21cc22013-01-07 08:51:08 -08001267 * Call with cpuset_mutex held.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 */
1269
Paul Menage700fe1a2008-04-29 01:00:00 -07001270static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1271 int turning_on)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272{
Li Zefan645fcc92009-01-07 18:08:43 -08001273 struct cpuset *trialcs;
Rakib Mullick40b6a762008-10-18 20:28:18 -07001274 int balance_flag_changed;
Miao Xie950592f2009-06-16 15:31:47 -07001275 int spread_flag_changed;
Miao Xie950592f2009-06-16 15:31:47 -07001276 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277
Li Zefan645fcc92009-01-07 18:08:43 -08001278 trialcs = alloc_trial_cpuset(cs);
1279 if (!trialcs)
1280 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281
Li Zefan645fcc92009-01-07 18:08:43 -08001282 if (turning_on)
1283 set_bit(bit, &trialcs->flags);
1284 else
1285 clear_bit(bit, &trialcs->flags);
1286
1287 err = validate_change(cs, trialcs);
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -07001288 if (err < 0)
Li Zefan645fcc92009-01-07 18:08:43 -08001289 goto out;
Paul Jackson029190c2007-10-18 23:40:20 -07001290
Paul Jackson029190c2007-10-18 23:40:20 -07001291 balance_flag_changed = (is_sched_load_balance(cs) !=
Li Zefan645fcc92009-01-07 18:08:43 -08001292 is_sched_load_balance(trialcs));
Paul Jackson029190c2007-10-18 23:40:20 -07001293
Miao Xie950592f2009-06-16 15:31:47 -07001294 spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
1295 || (is_spread_page(cs) != is_spread_page(trialcs)));
1296
Ingo Molnar3d3f26a2006-03-23 03:00:18 -08001297 mutex_lock(&callback_mutex);
Li Zefan645fcc92009-01-07 18:08:43 -08001298 cs->flags = trialcs->flags;
Ingo Molnar3d3f26a2006-03-23 03:00:18 -08001299 mutex_unlock(&callback_mutex);
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -07001300
Li Zefan300ed6c2009-01-07 18:08:44 -08001301 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
Tejun Heo699140b2013-01-07 08:51:07 -08001302 rebuild_sched_domains_locked();
Paul Jackson029190c2007-10-18 23:40:20 -07001303
Miao Xie950592f2009-06-16 15:31:47 -07001304 if (spread_flag_changed)
Tejun Heod66393e2014-02-13 06:58:40 -05001305 update_tasks_flags(cs);
Li Zefan645fcc92009-01-07 18:08:43 -08001306out:
1307 free_trial_cpuset(trialcs);
1308 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309}
1310
Paul Jackson053199e2005-10-30 15:02:30 -08001311/*
Adrian Bunk80f72282006-06-30 18:27:16 +02001312 * Frequency meter - How fast is some event occurring?
Paul Jackson3e0d98b2006-01-08 01:01:49 -08001313 *
1314 * These routines manage a digitally filtered, constant time based,
1315 * event frequency meter. There are four routines:
1316 * fmeter_init() - initialize a frequency meter.
1317 * fmeter_markevent() - called each time the event happens.
1318 * fmeter_getrate() - returns the recent rate of such events.
1319 * fmeter_update() - internal routine used to update fmeter.
1320 *
1321 * A common data structure is passed to each of these routines,
1322 * which is used to keep track of the state required to manage the
1323 * frequency meter and its digital filter.
1324 *
1325 * The filter works on the number of events marked per unit time.
1326 * The filter is single-pole low-pass recursive (IIR). The time unit
1327 * is 1 second. Arithmetic is done using 32-bit integers scaled to
1328 * simulate 3 decimal digits of precision (multiplied by 1000).
1329 *
1330 * With an FM_COEF of 933, and a time base of 1 second, the filter
1331 * has a half-life of 10 seconds, meaning that if the events quit
1332 * happening, then the rate returned from the fmeter_getrate()
1333 * will be cut in half each 10 seconds, until it converges to zero.
1334 *
1335 * It is not worth doing a real infinitely recursive filter. If more
1336 * than FM_MAXTICKS ticks have elapsed since the last filter event,
1337 * just compute FM_MAXTICKS ticks worth, by which point the level
1338 * will be stable.
1339 *
1340 * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
1341 * arithmetic overflow in the fmeter_update() routine.
1342 *
1343 * Given the simple 32 bit integer arithmetic used, this meter works
1344 * best for reporting rates between one per millisecond (msec) and
1345 * one per 32 (approx) seconds. At constant rates faster than one
1346 * per msec it maxes out at values just under 1,000,000. At constant
1347 * rates between one per msec, and one per second it will stabilize
1348 * to a value N*1000, where N is the rate of events per second.
1349 * At constant rates between one per second and one per 32 seconds,
1350 * it will be choppy, moving up on the seconds that have an event,
1351 * and then decaying until the next event. At rates slower than
1352 * about one in 32 seconds, it decays all the way back to zero between
1353 * each event.
1354 */
1355
1356#define FM_COEF 933 /* coefficient for half-life of 10 secs */
1357#define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */
1358#define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */
1359#define FM_SCALE 1000 /* faux fixed point scale */
1360
1361/* Initialize a frequency meter */
1362static void fmeter_init(struct fmeter *fmp)
1363{
1364 fmp->cnt = 0;
1365 fmp->val = 0;
1366 fmp->time = 0;
1367 spin_lock_init(&fmp->lock);
1368}
1369
1370/* Internal meter update - process cnt events and update value */
1371static void fmeter_update(struct fmeter *fmp)
1372{
1373 time_t now = get_seconds();
1374 time_t ticks = now - fmp->time;
1375
1376 if (ticks == 0)
1377 return;
1378
1379 ticks = min(FM_MAXTICKS, ticks);
1380 while (ticks-- > 0)
1381 fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
1382 fmp->time = now;
1383
1384 fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
1385 fmp->cnt = 0;
1386}
1387
1388/* Process any previous ticks, then bump cnt by one (times scale). */
1389static void fmeter_markevent(struct fmeter *fmp)
1390{
1391 spin_lock(&fmp->lock);
1392 fmeter_update(fmp);
1393 fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
1394 spin_unlock(&fmp->lock);
1395}
1396
1397/* Process any previous ticks, then return current value. */
1398static int fmeter_getrate(struct fmeter *fmp)
1399{
1400 int val;
1401
1402 spin_lock(&fmp->lock);
1403 fmeter_update(fmp);
1404 val = fmp->val;
1405 spin_unlock(&fmp->lock);
1406 return val;
1407}
1408
Tejun Heo57fce0a2014-02-13 06:58:41 -05001409static struct cpuset *cpuset_attach_old_cs;
1410
Tejun Heo5d21cc22013-01-07 08:51:08 -08001411/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
Tejun Heoeb954192013-08-08 20:11:23 -04001412static int cpuset_can_attach(struct cgroup_subsys_state *css,
1413 struct cgroup_taskset *tset)
Ben Blumf780bdb2011-05-26 16:25:19 -07001414{
Tejun Heoeb954192013-08-08 20:11:23 -04001415 struct cpuset *cs = css_cs(css);
Tejun Heobb9d97b2011-12-12 18:12:21 -08001416 struct task_struct *task;
1417 int ret;
Ben Blumf780bdb2011-05-26 16:25:19 -07001418
Tejun Heo57fce0a2014-02-13 06:58:41 -05001419 /* used later by cpuset_attach() */
1420 cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset));
1421
Tejun Heo5d21cc22013-01-07 08:51:08 -08001422 mutex_lock(&cpuset_mutex);
1423
Tejun Heoaa6ec292014-07-09 10:08:08 -04001424 /* allow moving tasks into an empty cpuset if on default hierarchy */
Tejun Heo5d21cc22013-01-07 08:51:08 -08001425 ret = -ENOSPC;
Tejun Heoaa6ec292014-07-09 10:08:08 -04001426 if (!cgroup_on_dfl(css->cgroup) &&
Li Zefan88fa5232013-06-09 17:16:46 +08001427 (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)))
Tejun Heo5d21cc22013-01-07 08:51:08 -08001428 goto out_unlock;
Ben Blumbe367d02009-09-23 15:56:31 -07001429
Tejun Heo924f0d92014-02-13 06:58:41 -05001430 cgroup_taskset_for_each(task, tset) {
Tejun Heobb9d97b2011-12-12 18:12:21 -08001431 /*
Tejun Heo14a40ff2013-03-19 13:45:20 -07001432 * Kthreads which disallow setaffinity shouldn't be moved
1433 * to a new cpuset; we don't want to change their cpu
1434 * affinity and isolating such threads by their set of
1435 * allowed nodes is unnecessary. Thus, cpusets are not
1436 * applicable for such threads. This prevents checking for
1437 * success of set_cpus_allowed_ptr() on all attached tasks
1438 * before cpus_allowed may be changed.
Tejun Heobb9d97b2011-12-12 18:12:21 -08001439 */
Tejun Heo5d21cc22013-01-07 08:51:08 -08001440 ret = -EINVAL;
Tejun Heo14a40ff2013-03-19 13:45:20 -07001441 if (task->flags & PF_NO_SETAFFINITY)
Tejun Heo5d21cc22013-01-07 08:51:08 -08001442 goto out_unlock;
1443 ret = security_task_setscheduler(task);
1444 if (ret)
1445 goto out_unlock;
Tejun Heobb9d97b2011-12-12 18:12:21 -08001446 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447
Tejun Heo452477f2013-01-07 08:51:07 -08001448 /*
1449 * Mark attach is in progress. This makes validate_change() fail
1450 * changes which zero cpus/mems_allowed.
1451 */
1452 cs->attach_in_progress++;
Tejun Heo5d21cc22013-01-07 08:51:08 -08001453 ret = 0;
1454out_unlock:
1455 mutex_unlock(&cpuset_mutex);
1456 return ret;
Tejun Heo4e4c9a12013-01-07 08:51:07 -08001457}
1458
Tejun Heoeb954192013-08-08 20:11:23 -04001459static void cpuset_cancel_attach(struct cgroup_subsys_state *css,
Tejun Heo452477f2013-01-07 08:51:07 -08001460 struct cgroup_taskset *tset)
1461{
Tejun Heo5d21cc22013-01-07 08:51:08 -08001462 mutex_lock(&cpuset_mutex);
Tejun Heoeb954192013-08-08 20:11:23 -04001463 css_cs(css)->attach_in_progress--;
Tejun Heo5d21cc22013-01-07 08:51:08 -08001464 mutex_unlock(&cpuset_mutex);
Tejun Heo452477f2013-01-07 08:51:07 -08001465}
1466
Tejun Heo4e4c9a12013-01-07 08:51:07 -08001467/*
Tejun Heo5d21cc22013-01-07 08:51:08 -08001468 * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach()
Tejun Heo4e4c9a12013-01-07 08:51:07 -08001469 * but we can't allocate it dynamically there. Define it global and
1470 * allocate from cpuset_init().
1471 */
1472static cpumask_var_t cpus_attach;
1473
Tejun Heoeb954192013-08-08 20:11:23 -04001474static void cpuset_attach(struct cgroup_subsys_state *css,
1475 struct cgroup_taskset *tset)
Tejun Heo4e4c9a12013-01-07 08:51:07 -08001476{
Li Zefan67bd2c52013-06-05 17:15:35 +08001477 /* static buf protected by cpuset_mutex */
Tejun Heo4e4c9a12013-01-07 08:51:07 -08001478 static nodemask_t cpuset_attach_nodemask_to;
1479 struct mm_struct *mm;
1480 struct task_struct *task;
1481 struct task_struct *leader = cgroup_taskset_first(tset);
Tejun Heoeb954192013-08-08 20:11:23 -04001482 struct cpuset *cs = css_cs(css);
Tejun Heo57fce0a2014-02-13 06:58:41 -05001483 struct cpuset *oldcs = cpuset_attach_old_cs;
Tejun Heo4e4c9a12013-01-07 08:51:07 -08001484
Tejun Heo5d21cc22013-01-07 08:51:08 -08001485 mutex_lock(&cpuset_mutex);
1486
Tejun Heo94196f52011-12-12 18:12:22 -08001487 /* prepare for attach */
Ben Blumf780bdb2011-05-26 16:25:19 -07001488 if (cs == &top_cpuset)
1489 cpumask_copy(cpus_attach, cpu_possible_mask);
1490 else
Li Zefanae1c8022014-07-09 16:48:32 +08001491 guarantee_online_cpus(cs, cpus_attach);
Ben Blumf780bdb2011-05-26 16:25:19 -07001492
Li Zefanae1c8022014-07-09 16:48:32 +08001493 guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
Tejun Heo94196f52011-12-12 18:12:22 -08001494
Tejun Heo924f0d92014-02-13 06:58:41 -05001495 cgroup_taskset_for_each(task, tset) {
Tejun Heobb9d97b2011-12-12 18:12:21 -08001496 /*
1497 * can_attach beforehand should guarantee that this doesn't
1498 * fail. TODO: have a better way to handle failure here
1499 */
1500 WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));
1501
1502 cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
1503 cpuset_update_task_spread_flag(cs, task);
1504 }
David Quigley22fb52d2006-06-23 02:04:00 -07001505
Ben Blumf780bdb2011-05-26 16:25:19 -07001506 /*
1507 * Change mm, possibly for multiple threads in a threadgroup. This is
1508 * expensive and may sleep.
1509 */
Li Zefanae1c8022014-07-09 16:48:32 +08001510 cpuset_attach_nodemask_to = cs->effective_mems;
Tejun Heobb9d97b2011-12-12 18:12:21 -08001511 mm = get_task_mm(leader);
Paul Jackson42253992006-01-08 01:01:59 -08001512 if (mm) {
Ben Blumf780bdb2011-05-26 16:25:19 -07001513 mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
Li Zefanf047cec2013-06-13 15:11:44 +08001514
1515 /*
1516 * old_mems_allowed is the same with mems_allowed here, except
1517 * if this task is being moved automatically due to hotplug.
1518 * In that case @mems_allowed has been updated and is empty,
1519 * so @old_mems_allowed is the right nodesets that we migrate
1520 * mm from.
1521 */
1522 if (is_memory_migrate(cs)) {
Li Zefanae1c8022014-07-09 16:48:32 +08001523 cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
Ben Blumf780bdb2011-05-26 16:25:19 -07001524 &cpuset_attach_nodemask_to);
Li Zefanf047cec2013-06-13 15:11:44 +08001525 }
Paul Jackson42253992006-01-08 01:01:59 -08001526 mmput(mm);
1527 }
Tejun Heo452477f2013-01-07 08:51:07 -08001528
Li Zefan33ad8012013-06-09 17:15:08 +08001529 cs->old_mems_allowed = cpuset_attach_nodemask_to;
Tejun Heo02bb5862013-01-07 08:51:08 -08001530
Tejun Heo452477f2013-01-07 08:51:07 -08001531 cs->attach_in_progress--;
Li Zefane44193d2013-06-09 17:14:22 +08001532 if (!cs->attach_in_progress)
1533 wake_up(&cpuset_attach_wq);
Tejun Heo5d21cc22013-01-07 08:51:08 -08001534
1535 mutex_unlock(&cpuset_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001536}
1537
1538/* The various types of files and directories in a cpuset file system */
1539
1540typedef enum {
Paul Jackson45b07ef2006-01-08 01:00:56 -08001541 FILE_MEMORY_MIGRATE,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542 FILE_CPULIST,
1543 FILE_MEMLIST,
Li Zefanafd1a8b2014-07-09 16:49:25 +08001544 FILE_EFFECTIVE_CPULIST,
1545 FILE_EFFECTIVE_MEMLIST,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 FILE_CPU_EXCLUSIVE,
1547 FILE_MEM_EXCLUSIVE,
Paul Menage78608362008-04-29 01:00:26 -07001548 FILE_MEM_HARDWALL,
Paul Jackson029190c2007-10-18 23:40:20 -07001549 FILE_SCHED_LOAD_BALANCE,
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09001550 FILE_SCHED_RELAX_DOMAIN_LEVEL,
Paul Jackson3e0d98b2006-01-08 01:01:49 -08001551 FILE_MEMORY_PRESSURE_ENABLED,
1552 FILE_MEMORY_PRESSURE,
Paul Jackson825a46a2006-03-24 03:16:03 -08001553 FILE_SPREAD_PAGE,
1554 FILE_SPREAD_SLAB,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555} cpuset_filetype_t;
1556
Tejun Heo182446d2013-08-08 20:11:24 -04001557static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
1558 u64 val)
Paul Menage700fe1a2008-04-29 01:00:00 -07001559{
Tejun Heo182446d2013-08-08 20:11:24 -04001560 struct cpuset *cs = css_cs(css);
Paul Menage700fe1a2008-04-29 01:00:00 -07001561 cpuset_filetype_t type = cft->private;
Li Zefana903f082013-08-13 10:05:59 +08001562 int retval = 0;
Paul Menage700fe1a2008-04-29 01:00:00 -07001563
Tejun Heo5d21cc22013-01-07 08:51:08 -08001564 mutex_lock(&cpuset_mutex);
Li Zefana903f082013-08-13 10:05:59 +08001565 if (!is_cpuset_online(cs)) {
1566 retval = -ENODEV;
Tejun Heo5d21cc22013-01-07 08:51:08 -08001567 goto out_unlock;
Li Zefana903f082013-08-13 10:05:59 +08001568 }
Paul Menage700fe1a2008-04-29 01:00:00 -07001569
1570 switch (type) {
1571 case FILE_CPU_EXCLUSIVE:
1572 retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
1573 break;
1574 case FILE_MEM_EXCLUSIVE:
1575 retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
1576 break;
Paul Menage78608362008-04-29 01:00:26 -07001577 case FILE_MEM_HARDWALL:
1578 retval = update_flag(CS_MEM_HARDWALL, cs, val);
1579 break;
Paul Menage700fe1a2008-04-29 01:00:00 -07001580 case FILE_SCHED_LOAD_BALANCE:
1581 retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
1582 break;
1583 case FILE_MEMORY_MIGRATE:
1584 retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
1585 break;
1586 case FILE_MEMORY_PRESSURE_ENABLED:
1587 cpuset_memory_pressure_enabled = !!val;
1588 break;
1589 case FILE_MEMORY_PRESSURE:
1590 retval = -EACCES;
1591 break;
1592 case FILE_SPREAD_PAGE:
1593 retval = update_flag(CS_SPREAD_PAGE, cs, val);
Paul Menage700fe1a2008-04-29 01:00:00 -07001594 break;
1595 case FILE_SPREAD_SLAB:
1596 retval = update_flag(CS_SPREAD_SLAB, cs, val);
Paul Menage700fe1a2008-04-29 01:00:00 -07001597 break;
1598 default:
1599 retval = -EINVAL;
1600 break;
1601 }
Tejun Heo5d21cc22013-01-07 08:51:08 -08001602out_unlock:
1603 mutex_unlock(&cpuset_mutex);
Paul Menage700fe1a2008-04-29 01:00:00 -07001604 return retval;
1605}
1606
Tejun Heo182446d2013-08-08 20:11:24 -04001607static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
1608 s64 val)
Paul Menage5be7a472008-05-06 20:42:41 -07001609{
Tejun Heo182446d2013-08-08 20:11:24 -04001610 struct cpuset *cs = css_cs(css);
Paul Menage5be7a472008-05-06 20:42:41 -07001611 cpuset_filetype_t type = cft->private;
Tejun Heo5d21cc22013-01-07 08:51:08 -08001612 int retval = -ENODEV;
Paul Menage5be7a472008-05-06 20:42:41 -07001613
Tejun Heo5d21cc22013-01-07 08:51:08 -08001614 mutex_lock(&cpuset_mutex);
1615 if (!is_cpuset_online(cs))
1616 goto out_unlock;
Paul Menagee3712392008-07-25 01:47:02 -07001617
Paul Menage5be7a472008-05-06 20:42:41 -07001618 switch (type) {
1619 case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1620 retval = update_relax_domain_level(cs, val);
1621 break;
1622 default:
1623 retval = -EINVAL;
1624 break;
1625 }
Tejun Heo5d21cc22013-01-07 08:51:08 -08001626out_unlock:
1627 mutex_unlock(&cpuset_mutex);
Paul Menage5be7a472008-05-06 20:42:41 -07001628 return retval;
1629}
1630
Linus Torvalds1da177e2005-04-16 15:20:36 -07001631/*
Paul Menagee3712392008-07-25 01:47:02 -07001632 * Common handling for a write to a "cpus" or "mems" file.
1633 */
Tejun Heo451af502014-05-13 12:16:21 -04001634static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
1635 char *buf, size_t nbytes, loff_t off)
Paul Menagee3712392008-07-25 01:47:02 -07001636{
Tejun Heo451af502014-05-13 12:16:21 -04001637 struct cpuset *cs = css_cs(of_css(of));
Li Zefan645fcc92009-01-07 18:08:43 -08001638 struct cpuset *trialcs;
Tejun Heo5d21cc22013-01-07 08:51:08 -08001639 int retval = -ENODEV;
Paul Menagee3712392008-07-25 01:47:02 -07001640
Tejun Heo451af502014-05-13 12:16:21 -04001641 buf = strstrip(buf);
1642
Tejun Heo3a5a6d02013-01-07 08:51:07 -08001643 /*
1644 * CPU or memory hotunplug may leave @cs w/o any execution
1645 * resources, in which case the hotplug code asynchronously updates
1646 * configuration and transfers all tasks to the nearest ancestor
1647 * which can execute.
1648 *
1649 * As writes to "cpus" or "mems" may restore @cs's execution
1650 * resources, wait for the previously scheduled operations before
1651 * proceeding, so that we don't end up keep removing tasks added
1652 * after execution capability is restored.
Tejun Heo76bb5ab2014-06-30 15:47:32 -04001653 *
1654 * cpuset_hotplug_work calls back into cgroup core via
1655 * cgroup_transfer_tasks() and waiting for it from a cgroupfs
1656 * operation like this one can lead to a deadlock through kernfs
1657 * active_ref protection. Let's break the protection. Losing the
1658 * protection is okay as we check whether @cs is online after
1659 * grabbing cpuset_mutex anyway. This only happens on the legacy
1660 * hierarchies.
Tejun Heo3a5a6d02013-01-07 08:51:07 -08001661 */
Tejun Heo76bb5ab2014-06-30 15:47:32 -04001662 css_get(&cs->css);
1663 kernfs_break_active_protection(of->kn);
Tejun Heo3a5a6d02013-01-07 08:51:07 -08001664 flush_work(&cpuset_hotplug_work);
1665
Tejun Heo5d21cc22013-01-07 08:51:08 -08001666 mutex_lock(&cpuset_mutex);
1667 if (!is_cpuset_online(cs))
1668 goto out_unlock;
Paul Menagee3712392008-07-25 01:47:02 -07001669
Li Zefan645fcc92009-01-07 18:08:43 -08001670 trialcs = alloc_trial_cpuset(cs);
Li Zefanb75f38d2011-03-04 17:36:21 -08001671 if (!trialcs) {
1672 retval = -ENOMEM;
Tejun Heo5d21cc22013-01-07 08:51:08 -08001673 goto out_unlock;
Li Zefanb75f38d2011-03-04 17:36:21 -08001674 }
Li Zefan645fcc92009-01-07 18:08:43 -08001675
Tejun Heo451af502014-05-13 12:16:21 -04001676 switch (of_cft(of)->private) {
Paul Menagee3712392008-07-25 01:47:02 -07001677 case FILE_CPULIST:
Li Zefan645fcc92009-01-07 18:08:43 -08001678 retval = update_cpumask(cs, trialcs, buf);
Paul Menagee3712392008-07-25 01:47:02 -07001679 break;
1680 case FILE_MEMLIST:
Li Zefan645fcc92009-01-07 18:08:43 -08001681 retval = update_nodemask(cs, trialcs, buf);
Paul Menagee3712392008-07-25 01:47:02 -07001682 break;
1683 default:
1684 retval = -EINVAL;
1685 break;
1686 }
Li Zefan645fcc92009-01-07 18:08:43 -08001687
1688 free_trial_cpuset(trialcs);
Tejun Heo5d21cc22013-01-07 08:51:08 -08001689out_unlock:
1690 mutex_unlock(&cpuset_mutex);
Tejun Heo76bb5ab2014-06-30 15:47:32 -04001691 kernfs_unbreak_active_protection(of->kn);
1692 css_put(&cs->css);
Tejun Heo451af502014-05-13 12:16:21 -04001693 return retval ?: nbytes;
Paul Menagee3712392008-07-25 01:47:02 -07001694}
1695
1696/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697 * These ascii lists should be read in a single call, by using a user
1698 * buffer large enough to hold the entire map. If read in smaller
1699 * chunks, there is no guarantee of atomicity. Since the display format
1700 * used, list of ranges of sequential numbers, is variable length,
1701 * and since these maps can change value dynamically, one could read
1702 * gibberish by doing partial reads while a list was changing.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703 */
Tejun Heo2da8ca82013-12-05 12:28:04 -05001704static int cpuset_common_seq_show(struct seq_file *sf, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001706 struct cpuset *cs = css_cs(seq_css(sf));
1707 cpuset_filetype_t type = seq_cft(sf)->private;
Tejun Heo51ffe412013-12-05 12:28:02 -05001708 ssize_t count;
1709 char *buf, *s;
1710 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711
Tejun Heo51ffe412013-12-05 12:28:02 -05001712 count = seq_get_buf(sf, &buf);
1713 s = buf;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714
Tejun Heo51ffe412013-12-05 12:28:02 -05001715 mutex_lock(&callback_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716
1717 switch (type) {
1718 case FILE_CPULIST:
Tejun Heo51ffe412013-12-05 12:28:02 -05001719 s += cpulist_scnprintf(s, count, cs->cpus_allowed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001720 break;
1721 case FILE_MEMLIST:
Tejun Heo51ffe412013-12-05 12:28:02 -05001722 s += nodelist_scnprintf(s, count, cs->mems_allowed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001723 break;
Li Zefanafd1a8b2014-07-09 16:49:25 +08001724 case FILE_EFFECTIVE_CPULIST:
1725 s += cpulist_scnprintf(s, count, cs->effective_cpus);
1726 break;
1727 case FILE_EFFECTIVE_MEMLIST:
1728 s += nodelist_scnprintf(s, count, cs->effective_mems);
1729 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730 default:
Tejun Heo51ffe412013-12-05 12:28:02 -05001731 ret = -EINVAL;
1732 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001733 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001734
Tejun Heo51ffe412013-12-05 12:28:02 -05001735 if (s < buf + count - 1) {
1736 *s++ = '\n';
1737 seq_commit(sf, s - buf);
1738 } else {
1739 seq_commit(sf, -1);
1740 }
1741out_unlock:
1742 mutex_unlock(&callback_mutex);
1743 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744}
1745
Tejun Heo182446d2013-08-08 20:11:24 -04001746static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
Paul Menage700fe1a2008-04-29 01:00:00 -07001747{
Tejun Heo182446d2013-08-08 20:11:24 -04001748 struct cpuset *cs = css_cs(css);
Paul Menage700fe1a2008-04-29 01:00:00 -07001749 cpuset_filetype_t type = cft->private;
1750 switch (type) {
1751 case FILE_CPU_EXCLUSIVE:
1752 return is_cpu_exclusive(cs);
1753 case FILE_MEM_EXCLUSIVE:
1754 return is_mem_exclusive(cs);
Paul Menage78608362008-04-29 01:00:26 -07001755 case FILE_MEM_HARDWALL:
1756 return is_mem_hardwall(cs);
Paul Menage700fe1a2008-04-29 01:00:00 -07001757 case FILE_SCHED_LOAD_BALANCE:
1758 return is_sched_load_balance(cs);
1759 case FILE_MEMORY_MIGRATE:
1760 return is_memory_migrate(cs);
1761 case FILE_MEMORY_PRESSURE_ENABLED:
1762 return cpuset_memory_pressure_enabled;
1763 case FILE_MEMORY_PRESSURE:
1764 return fmeter_getrate(&cs->fmeter);
1765 case FILE_SPREAD_PAGE:
1766 return is_spread_page(cs);
1767 case FILE_SPREAD_SLAB:
1768 return is_spread_slab(cs);
1769 default:
1770 BUG();
1771 }
Max Krasnyanskycf417142008-08-11 14:33:53 -07001772
1773 /* Unreachable but makes gcc happy */
1774 return 0;
Paul Menage700fe1a2008-04-29 01:00:00 -07001775}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776
Tejun Heo182446d2013-08-08 20:11:24 -04001777static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
Paul Menage5be7a472008-05-06 20:42:41 -07001778{
Tejun Heo182446d2013-08-08 20:11:24 -04001779 struct cpuset *cs = css_cs(css);
Paul Menage5be7a472008-05-06 20:42:41 -07001780 cpuset_filetype_t type = cft->private;
1781 switch (type) {
1782 case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1783 return cs->relax_domain_level;
1784 default:
1785 BUG();
1786 }
Max Krasnyanskycf417142008-08-11 14:33:53 -07001787
1788 /* Unrechable but makes gcc happy */
1789 return 0;
Paul Menage5be7a472008-05-06 20:42:41 -07001790}
1791
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792
1793/*
1794 * for the common functions, 'private' gives the type of file
1795 */
1796
Paul Menageaddf2c72008-04-29 01:00:26 -07001797static struct cftype files[] = {
1798 {
1799 .name = "cpus",
Tejun Heo2da8ca82013-12-05 12:28:04 -05001800 .seq_show = cpuset_common_seq_show,
Tejun Heo451af502014-05-13 12:16:21 -04001801 .write = cpuset_write_resmask,
Paul Menagee3712392008-07-25 01:47:02 -07001802 .max_write_len = (100U + 6 * NR_CPUS),
Paul Menageaddf2c72008-04-29 01:00:26 -07001803 .private = FILE_CPULIST,
1804 },
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805
Paul Menageaddf2c72008-04-29 01:00:26 -07001806 {
1807 .name = "mems",
Tejun Heo2da8ca82013-12-05 12:28:04 -05001808 .seq_show = cpuset_common_seq_show,
Tejun Heo451af502014-05-13 12:16:21 -04001809 .write = cpuset_write_resmask,
Paul Menagee3712392008-07-25 01:47:02 -07001810 .max_write_len = (100U + 6 * MAX_NUMNODES),
Paul Menageaddf2c72008-04-29 01:00:26 -07001811 .private = FILE_MEMLIST,
1812 },
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813
Paul Menageaddf2c72008-04-29 01:00:26 -07001814 {
Li Zefanafd1a8b2014-07-09 16:49:25 +08001815 .name = "effective_cpus",
1816 .seq_show = cpuset_common_seq_show,
1817 .private = FILE_EFFECTIVE_CPULIST,
1818 },
1819
1820 {
1821 .name = "effective_mems",
1822 .seq_show = cpuset_common_seq_show,
1823 .private = FILE_EFFECTIVE_MEMLIST,
1824 },
1825
1826 {
Paul Menageaddf2c72008-04-29 01:00:26 -07001827 .name = "cpu_exclusive",
1828 .read_u64 = cpuset_read_u64,
1829 .write_u64 = cpuset_write_u64,
1830 .private = FILE_CPU_EXCLUSIVE,
1831 },
Linus Torvalds1da177e2005-04-16 15:20:36 -07001832
Paul Menageaddf2c72008-04-29 01:00:26 -07001833 {
1834 .name = "mem_exclusive",
1835 .read_u64 = cpuset_read_u64,
1836 .write_u64 = cpuset_write_u64,
1837 .private = FILE_MEM_EXCLUSIVE,
1838 },
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839
Paul Menageaddf2c72008-04-29 01:00:26 -07001840 {
Paul Menage78608362008-04-29 01:00:26 -07001841 .name = "mem_hardwall",
1842 .read_u64 = cpuset_read_u64,
1843 .write_u64 = cpuset_write_u64,
1844 .private = FILE_MEM_HARDWALL,
1845 },
1846
1847 {
Paul Menageaddf2c72008-04-29 01:00:26 -07001848 .name = "sched_load_balance",
1849 .read_u64 = cpuset_read_u64,
1850 .write_u64 = cpuset_write_u64,
1851 .private = FILE_SCHED_LOAD_BALANCE,
1852 },
Paul Jackson029190c2007-10-18 23:40:20 -07001853
Paul Menageaddf2c72008-04-29 01:00:26 -07001854 {
1855 .name = "sched_relax_domain_level",
Paul Menage5be7a472008-05-06 20:42:41 -07001856 .read_s64 = cpuset_read_s64,
1857 .write_s64 = cpuset_write_s64,
Paul Menageaddf2c72008-04-29 01:00:26 -07001858 .private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
1859 },
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09001860
Paul Menageaddf2c72008-04-29 01:00:26 -07001861 {
1862 .name = "memory_migrate",
1863 .read_u64 = cpuset_read_u64,
1864 .write_u64 = cpuset_write_u64,
1865 .private = FILE_MEMORY_MIGRATE,
1866 },
1867
1868 {
1869 .name = "memory_pressure",
1870 .read_u64 = cpuset_read_u64,
1871 .write_u64 = cpuset_write_u64,
1872 .private = FILE_MEMORY_PRESSURE,
Li Zefan099fca32009-04-02 16:57:29 -07001873 .mode = S_IRUGO,
Paul Menageaddf2c72008-04-29 01:00:26 -07001874 },
1875
1876 {
1877 .name = "memory_spread_page",
1878 .read_u64 = cpuset_read_u64,
1879 .write_u64 = cpuset_write_u64,
1880 .private = FILE_SPREAD_PAGE,
1881 },
1882
1883 {
1884 .name = "memory_spread_slab",
1885 .read_u64 = cpuset_read_u64,
1886 .write_u64 = cpuset_write_u64,
1887 .private = FILE_SPREAD_SLAB,
1888 },
Tejun Heo4baf6e32012-04-01 12:09:55 -07001889
1890 {
1891 .name = "memory_pressure_enabled",
1892 .flags = CFTYPE_ONLY_ON_ROOT,
1893 .read_u64 = cpuset_read_u64,
1894 .write_u64 = cpuset_write_u64,
1895 .private = FILE_MEMORY_PRESSURE_ENABLED,
1896 },
1897
1898 { } /* terminate */
Paul Jackson45b07ef2006-01-08 01:00:56 -08001899};
1900
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901/*
Tejun Heo92fb9742012-11-19 08:13:38 -08001902 * cpuset_css_alloc - allocate a cpuset css
Li Zefanc9e5fe62013-06-14 11:18:27 +08001903 * cgrp: control group that the new cpuset will be part of
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 */
1905
Tejun Heoeb954192013-08-08 20:11:23 -04001906static struct cgroup_subsys_state *
1907cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001908{
Tejun Heoc8f699b2013-01-07 08:51:07 -08001909 struct cpuset *cs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001910
Tejun Heoeb954192013-08-08 20:11:23 -04001911 if (!parent_css)
Paul Menage8793d852007-10-18 23:39:39 -07001912 return &top_cpuset.css;
Tejun Heo033fa1c2012-11-19 08:13:39 -08001913
Tejun Heoc8f699b2013-01-07 08:51:07 -08001914 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915 if (!cs)
Paul Menage8793d852007-10-18 23:39:39 -07001916 return ERR_PTR(-ENOMEM);
Li Zefane2b9a3d2014-07-09 16:47:03 +08001917 if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL))
1918 goto free_cs;
1919 if (!alloc_cpumask_var(&cs->effective_cpus, GFP_KERNEL))
1920 goto free_cpus;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921
Paul Jackson029190c2007-10-18 23:40:20 -07001922 set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
Li Zefan300ed6c2009-01-07 18:08:44 -08001923 cpumask_clear(cs->cpus_allowed);
Mike Travisf9a86fc2008-04-04 18:11:07 -07001924 nodes_clear(cs->mems_allowed);
Li Zefane2b9a3d2014-07-09 16:47:03 +08001925 cpumask_clear(cs->effective_cpus);
1926 nodes_clear(cs->effective_mems);
Paul Jackson3e0d98b2006-01-08 01:01:49 -08001927 fmeter_init(&cs->fmeter);
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09001928 cs->relax_domain_level = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001929
Tejun Heoc8f699b2013-01-07 08:51:07 -08001930 return &cs->css;
Li Zefane2b9a3d2014-07-09 16:47:03 +08001931
1932free_cpus:
1933 free_cpumask_var(cs->cpus_allowed);
1934free_cs:
1935 kfree(cs);
1936 return ERR_PTR(-ENOMEM);
Tejun Heoc8f699b2013-01-07 08:51:07 -08001937}
1938
Tejun Heoeb954192013-08-08 20:11:23 -04001939static int cpuset_css_online(struct cgroup_subsys_state *css)
Tejun Heoc8f699b2013-01-07 08:51:07 -08001940{
Tejun Heoeb954192013-08-08 20:11:23 -04001941 struct cpuset *cs = css_cs(css);
Tejun Heoc4310692013-01-07 08:51:08 -08001942 struct cpuset *parent = parent_cs(cs);
Tejun Heoae8086c2013-01-07 08:51:07 -08001943 struct cpuset *tmp_cs;
Tejun Heo492eb212013-08-08 20:11:25 -04001944 struct cgroup_subsys_state *pos_css;
Tejun Heoc8f699b2013-01-07 08:51:07 -08001945
1946 if (!parent)
1947 return 0;
1948
Tejun Heo5d21cc22013-01-07 08:51:08 -08001949 mutex_lock(&cpuset_mutex);
1950
Tejun Heoefeb77b2013-01-07 08:51:07 -08001951 set_bit(CS_ONLINE, &cs->flags);
Tejun Heoc8f699b2013-01-07 08:51:07 -08001952 if (is_spread_page(parent))
1953 set_bit(CS_SPREAD_PAGE, &cs->flags);
1954 if (is_spread_slab(parent))
1955 set_bit(CS_SPREAD_SLAB, &cs->flags);
1956
Mel Gorman664eedd2014-06-04 16:10:08 -07001957 cpuset_inc();
Tejun Heo033fa1c2012-11-19 08:13:39 -08001958
Li Zefane2b9a3d2014-07-09 16:47:03 +08001959 mutex_lock(&callback_mutex);
1960 if (cgroup_on_dfl(cs->css.cgroup)) {
1961 cpumask_copy(cs->effective_cpus, parent->effective_cpus);
1962 cs->effective_mems = parent->effective_mems;
1963 }
1964 mutex_unlock(&callback_mutex);
1965
Tejun Heoeb954192013-08-08 20:11:23 -04001966 if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
Tejun Heo5d21cc22013-01-07 08:51:08 -08001967 goto out_unlock;
Tejun Heo033fa1c2012-11-19 08:13:39 -08001968
1969 /*
1970 * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
1971 * set. This flag handling is implemented in cgroup core for
1972 * histrical reasons - the flag may be specified during mount.
1973 *
1974 * Currently, if any sibling cpusets have exclusive cpus or mem, we
1975 * refuse to clone the configuration - thereby refusing the task to
1976 * be entered, and as a result refusing the sys_unshare() or
1977 * clone() which initiated it. If this becomes a problem for some
1978 * users who wish to allow that scenario, then this could be
1979 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
1980 * (and likewise for mems) to the new cgroup.
1981 */
Tejun Heoae8086c2013-01-07 08:51:07 -08001982 rcu_read_lock();
Tejun Heo492eb212013-08-08 20:11:25 -04001983 cpuset_for_each_child(tmp_cs, pos_css, parent) {
Tejun Heoae8086c2013-01-07 08:51:07 -08001984 if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
1985 rcu_read_unlock();
Tejun Heo5d21cc22013-01-07 08:51:08 -08001986 goto out_unlock;
Tejun Heoae8086c2013-01-07 08:51:07 -08001987 }
Tejun Heo033fa1c2012-11-19 08:13:39 -08001988 }
Tejun Heoae8086c2013-01-07 08:51:07 -08001989 rcu_read_unlock();
Tejun Heo033fa1c2012-11-19 08:13:39 -08001990
1991 mutex_lock(&callback_mutex);
1992 cs->mems_allowed = parent->mems_allowed;
1993 cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
1994 mutex_unlock(&callback_mutex);
Tejun Heo5d21cc22013-01-07 08:51:08 -08001995out_unlock:
1996 mutex_unlock(&cpuset_mutex);
Tejun Heoc8f699b2013-01-07 08:51:07 -08001997 return 0;
1998}
1999
Zhao Hongjiang0b9e6962013-07-27 11:56:53 +08002000/*
2001 * If the cpuset being removed has its flag 'sched_load_balance'
2002 * enabled, then simulate turning sched_load_balance off, which
2003 * will call rebuild_sched_domains_locked().
2004 */
2005
Tejun Heoeb954192013-08-08 20:11:23 -04002006static void cpuset_css_offline(struct cgroup_subsys_state *css)
Tejun Heoc8f699b2013-01-07 08:51:07 -08002007{
Tejun Heoeb954192013-08-08 20:11:23 -04002008 struct cpuset *cs = css_cs(css);
Tejun Heoc8f699b2013-01-07 08:51:07 -08002009
Tejun Heo5d21cc22013-01-07 08:51:08 -08002010 mutex_lock(&cpuset_mutex);
Tejun Heoc8f699b2013-01-07 08:51:07 -08002011
2012 if (is_sched_load_balance(cs))
2013 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
2014
Mel Gorman664eedd2014-06-04 16:10:08 -07002015 cpuset_dec();
Tejun Heoefeb77b2013-01-07 08:51:07 -08002016 clear_bit(CS_ONLINE, &cs->flags);
Tejun Heoc8f699b2013-01-07 08:51:07 -08002017
Tejun Heo5d21cc22013-01-07 08:51:08 -08002018 mutex_unlock(&cpuset_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019}
2020
Tejun Heoeb954192013-08-08 20:11:23 -04002021static void cpuset_css_free(struct cgroup_subsys_state *css)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002022{
Tejun Heoeb954192013-08-08 20:11:23 -04002023 struct cpuset *cs = css_cs(css);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002024
Li Zefane2b9a3d2014-07-09 16:47:03 +08002025 free_cpumask_var(cs->effective_cpus);
Li Zefan300ed6c2009-01-07 18:08:44 -08002026 free_cpumask_var(cs->cpus_allowed);
Paul Menage8793d852007-10-18 23:39:39 -07002027 kfree(cs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002028}
2029
Li Zefan39bd0d12014-07-09 16:48:01 +08002030static void cpuset_bind(struct cgroup_subsys_state *root_css)
2031{
2032 mutex_lock(&cpuset_mutex);
2033 mutex_lock(&callback_mutex);
2034
2035 if (cgroup_on_dfl(root_css->cgroup)) {
2036 cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask);
2037 top_cpuset.mems_allowed = node_possible_map;
2038 } else {
2039 cpumask_copy(top_cpuset.cpus_allowed,
2040 top_cpuset.effective_cpus);
2041 top_cpuset.mems_allowed = top_cpuset.effective_mems;
2042 }
2043
2044 mutex_unlock(&callback_mutex);
2045 mutex_unlock(&cpuset_mutex);
2046}
2047
Tejun Heo073219e2014-02-08 10:36:58 -05002048struct cgroup_subsys cpuset_cgrp_subsys = {
Li Zefan39bd0d12014-07-09 16:48:01 +08002049 .css_alloc = cpuset_css_alloc,
2050 .css_online = cpuset_css_online,
2051 .css_offline = cpuset_css_offline,
2052 .css_free = cpuset_css_free,
2053 .can_attach = cpuset_can_attach,
2054 .cancel_attach = cpuset_cancel_attach,
2055 .attach = cpuset_attach,
2056 .bind = cpuset_bind,
Tejun Heo55779642014-07-15 11:05:09 -04002057 .legacy_cftypes = files,
Li Zefan39bd0d12014-07-09 16:48:01 +08002058 .early_init = 1,
Paul Menage8793d852007-10-18 23:39:39 -07002059};
2060
Linus Torvalds1da177e2005-04-16 15:20:36 -07002061/**
2062 * cpuset_init - initialize cpusets at system boot
2063 *
2064 * Description: Initialize top_cpuset and the cpuset internal file system,
2065 **/
2066
2067int __init cpuset_init(void)
2068{
Paul Menage8793d852007-10-18 23:39:39 -07002069 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070
Miao Xie58568d22009-06-16 15:31:49 -07002071 if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL))
2072 BUG();
Li Zefane2b9a3d2014-07-09 16:47:03 +08002073 if (!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL))
2074 BUG();
Miao Xie58568d22009-06-16 15:31:49 -07002075
Li Zefan300ed6c2009-01-07 18:08:44 -08002076 cpumask_setall(top_cpuset.cpus_allowed);
Mike Travisf9a86fc2008-04-04 18:11:07 -07002077 nodes_setall(top_cpuset.mems_allowed);
Li Zefane2b9a3d2014-07-09 16:47:03 +08002078 cpumask_setall(top_cpuset.effective_cpus);
2079 nodes_setall(top_cpuset.effective_mems);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080
Paul Jackson3e0d98b2006-01-08 01:01:49 -08002081 fmeter_init(&top_cpuset.fmeter);
Paul Jackson029190c2007-10-18 23:40:20 -07002082 set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09002083 top_cpuset.relax_domain_level = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002084
Linus Torvalds1da177e2005-04-16 15:20:36 -07002085 err = register_filesystem(&cpuset_fs_type);
2086 if (err < 0)
Paul Menage8793d852007-10-18 23:39:39 -07002087 return err;
2088
Li Zefan2341d1b2009-01-07 18:08:42 -08002089 if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
2090 BUG();
2091
Paul Menage8793d852007-10-18 23:39:39 -07002092 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002093}
2094
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002095/*
Max Krasnyanskycf417142008-08-11 14:33:53 -07002096 * If CPU and/or memory hotplug handlers, below, unplug any CPUs
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002097 * or memory nodes, we need to walk over the cpuset hierarchy,
2098 * removing that CPU or node from all cpusets. If this removes the
Cliff Wickman956db3c2008-02-07 00:14:43 -08002099 * last CPU or node from a cpuset, then move the tasks in the empty
2100 * cpuset to its next-highest non-empty parent.
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002101 */
Cliff Wickman956db3c2008-02-07 00:14:43 -08002102static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002103{
Cliff Wickman956db3c2008-02-07 00:14:43 -08002104 struct cpuset *parent;
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002105
Paul Jacksonc8d9c902008-02-07 00:14:46 -08002106 /*
Cliff Wickman956db3c2008-02-07 00:14:43 -08002107 * Find its next-highest non-empty parent, (top cpuset
2108 * has online cpus, so can't be empty).
2109 */
Tejun Heoc4310692013-01-07 08:51:08 -08002110 parent = parent_cs(cs);
Li Zefan300ed6c2009-01-07 18:08:44 -08002111 while (cpumask_empty(parent->cpus_allowed) ||
Paul Jacksonb4501292008-02-07 00:14:47 -08002112 nodes_empty(parent->mems_allowed))
Tejun Heoc4310692013-01-07 08:51:08 -08002113 parent = parent_cs(parent);
Cliff Wickman956db3c2008-02-07 00:14:43 -08002114
Tejun Heo8cc99342013-04-07 09:29:50 -07002115 if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {
Fabian Frederick12d30892014-05-05 19:49:00 +02002116 pr_err("cpuset: failed to transfer tasks out of empty cpuset ");
Tejun Heoe61734c2014-02-12 09:29:50 -05002117 pr_cont_cgroup_name(cs->css.cgroup);
2118 pr_cont("\n");
Tejun Heo8cc99342013-04-07 09:29:50 -07002119 }
Cliff Wickman956db3c2008-02-07 00:14:43 -08002120}
2121
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002122static void
2123hotplug_update_tasks_legacy(struct cpuset *cs,
2124 struct cpumask *new_cpus, nodemask_t *new_mems,
2125 bool cpus_updated, bool mems_updated)
Li Zefan390a36a2014-07-09 16:48:54 +08002126{
2127 bool is_empty;
2128
2129 mutex_lock(&callback_mutex);
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002130 cpumask_copy(cs->cpus_allowed, new_cpus);
2131 cpumask_copy(cs->effective_cpus, new_cpus);
2132 cs->mems_allowed = *new_mems;
2133 cs->effective_mems = *new_mems;
Li Zefan390a36a2014-07-09 16:48:54 +08002134 mutex_unlock(&callback_mutex);
2135
2136 /*
2137 * Don't call update_tasks_cpumask() if the cpuset becomes empty,
2138 * as the tasks will be migratecd to an ancestor.
2139 */
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002140 if (cpus_updated && !cpumask_empty(cs->cpus_allowed))
Li Zefan390a36a2014-07-09 16:48:54 +08002141 update_tasks_cpumask(cs);
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002142 if (mems_updated && !nodes_empty(cs->mems_allowed))
Li Zefan390a36a2014-07-09 16:48:54 +08002143 update_tasks_nodemask(cs);
2144
2145 is_empty = cpumask_empty(cs->cpus_allowed) ||
2146 nodes_empty(cs->mems_allowed);
2147
2148 mutex_unlock(&cpuset_mutex);
2149
2150 /*
2151 * Move tasks to the nearest ancestor with execution resources,
2152 * This is full cgroup operation which will also call back into
2153 * cpuset. Should be done outside any lock.
2154 */
2155 if (is_empty)
2156 remove_tasks_in_empty_cpuset(cs);
2157
2158 mutex_lock(&cpuset_mutex);
2159}
2160
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002161static void
2162hotplug_update_tasks(struct cpuset *cs,
2163 struct cpumask *new_cpus, nodemask_t *new_mems,
2164 bool cpus_updated, bool mems_updated)
Li Zefan390a36a2014-07-09 16:48:54 +08002165{
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002166 if (cpumask_empty(new_cpus))
2167 cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus);
2168 if (nodes_empty(*new_mems))
2169 *new_mems = parent_cs(cs)->effective_mems;
Li Zefan390a36a2014-07-09 16:48:54 +08002170
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002171 mutex_lock(&callback_mutex);
2172 cpumask_copy(cs->effective_cpus, new_cpus);
2173 cs->effective_mems = *new_mems;
Li Zefan390a36a2014-07-09 16:48:54 +08002174 mutex_unlock(&callback_mutex);
2175
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002176 if (cpus_updated)
Li Zefan390a36a2014-07-09 16:48:54 +08002177 update_tasks_cpumask(cs);
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002178 if (mems_updated)
Li Zefan390a36a2014-07-09 16:48:54 +08002179 update_tasks_nodemask(cs);
2180}
2181
Tejun Heodeb7aa32013-01-07 08:51:07 -08002182/**
Li Zefan388afd82013-06-09 17:14:47 +08002183 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
Tejun Heodeb7aa32013-01-07 08:51:07 -08002184 * @cs: cpuset in interest
Cliff Wickman956db3c2008-02-07 00:14:43 -08002185 *
Tejun Heodeb7aa32013-01-07 08:51:07 -08002186 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
2187 * offline, update @cs accordingly. If @cs ends up with no CPU or memory,
2188 * all its tasks are moved to the nearest ancestor with both resources.
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002189 */
Li Zefan388afd82013-06-09 17:14:47 +08002190static void cpuset_hotplug_update_tasks(struct cpuset *cs)
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002191{
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002192 static cpumask_t new_cpus;
2193 static nodemask_t new_mems;
2194 bool cpus_updated;
2195 bool mems_updated;
Li Zefane44193d2013-06-09 17:14:22 +08002196retry:
2197 wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002198
Tejun Heo5d21cc22013-01-07 08:51:08 -08002199 mutex_lock(&cpuset_mutex);
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002200
Li Zefane44193d2013-06-09 17:14:22 +08002201 /*
2202 * We have raced with task attaching. We wait until attaching
2203 * is finished, so we won't attach a task to an empty cpuset.
2204 */
2205 if (cs->attach_in_progress) {
2206 mutex_unlock(&cpuset_mutex);
2207 goto retry;
2208 }
2209
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002210 cpumask_and(&new_cpus, cs->cpus_allowed, parent_cs(cs)->effective_cpus);
2211 nodes_and(new_mems, cs->mems_allowed, parent_cs(cs)->effective_mems);
Paul Jacksonb4501292008-02-07 00:14:47 -08002212
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002213 cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus);
2214 mems_updated = !nodes_equal(new_mems, cs->effective_mems);
Cliff Wickman956db3c2008-02-07 00:14:43 -08002215
Li Zefan390a36a2014-07-09 16:48:54 +08002216 if (cgroup_on_dfl(cs->css.cgroup))
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002217 hotplug_update_tasks(cs, &new_cpus, &new_mems,
2218 cpus_updated, mems_updated);
Li Zefan390a36a2014-07-09 16:48:54 +08002219 else
Li Zefanbe4c9dd2014-07-09 16:49:04 +08002220 hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems,
2221 cpus_updated, mems_updated);
Tejun Heo8d033942013-01-07 08:51:07 -08002222
Tejun Heo5d21cc22013-01-07 08:51:08 -08002223 mutex_unlock(&cpuset_mutex);
Tejun Heodeb7aa32013-01-07 08:51:07 -08002224}
Paul Jacksonb4501292008-02-07 00:14:47 -08002225
Tejun Heodeb7aa32013-01-07 08:51:07 -08002226/**
Tejun Heo3a5a6d02013-01-07 08:51:07 -08002227 * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
Tejun Heodeb7aa32013-01-07 08:51:07 -08002228 *
2229 * This function is called after either CPU or memory configuration has
2230 * changed and updates cpuset accordingly. The top_cpuset is always
2231 * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
2232 * order to make cpusets transparent (of no affect) on systems that are
2233 * actively using CPU hotplug but making no active use of cpusets.
2234 *
2235 * Non-root cpusets are only affected by offlining. If any CPUs or memory
Li Zefan388afd82013-06-09 17:14:47 +08002236 * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
2237 * all descendants.
Tejun Heodeb7aa32013-01-07 08:51:07 -08002238 *
2239 * Note that CPU offlining during suspend is ignored. We don't modify
2240 * cpusets across suspend/resume cycles at all.
2241 */
Tejun Heo3a5a6d02013-01-07 08:51:07 -08002242static void cpuset_hotplug_workfn(struct work_struct *work)
Tejun Heodeb7aa32013-01-07 08:51:07 -08002243{
Li Zefan5c5cc622013-06-09 17:16:29 +08002244 static cpumask_t new_cpus;
2245 static nodemask_t new_mems;
Tejun Heodeb7aa32013-01-07 08:51:07 -08002246 bool cpus_updated, mems_updated;
Li Zefan7e882912014-07-09 16:48:42 +08002247 bool on_dfl = cgroup_on_dfl(top_cpuset.css.cgroup);
Cliff Wickman956db3c2008-02-07 00:14:43 -08002248
Tejun Heo5d21cc22013-01-07 08:51:08 -08002249 mutex_lock(&cpuset_mutex);
Cliff Wickman956db3c2008-02-07 00:14:43 -08002250
Tejun Heodeb7aa32013-01-07 08:51:07 -08002251 /* fetch the available cpus/mems and find out which changed how */
2252 cpumask_copy(&new_cpus, cpu_active_mask);
2253 new_mems = node_states[N_MEMORY];
Cliff Wickman956db3c2008-02-07 00:14:43 -08002254
Li Zefan7e882912014-07-09 16:48:42 +08002255 cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus);
2256 mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems);
Srivatsa S. Bhat7ddf96b2012-05-24 19:46:55 +05302257
Tejun Heodeb7aa32013-01-07 08:51:07 -08002258 /* synchronize cpus_allowed to cpu_active_mask */
2259 if (cpus_updated) {
2260 mutex_lock(&callback_mutex);
Li Zefan7e882912014-07-09 16:48:42 +08002261 if (!on_dfl)
2262 cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
Li Zefan1344ab92014-07-09 16:47:16 +08002263 cpumask_copy(top_cpuset.effective_cpus, &new_cpus);
Tejun Heodeb7aa32013-01-07 08:51:07 -08002264 mutex_unlock(&callback_mutex);
2265 /* we don't mess with cpumasks of tasks in top_cpuset */
2266 }
Srivatsa S. Bhat7ddf96b2012-05-24 19:46:55 +05302267
Tejun Heodeb7aa32013-01-07 08:51:07 -08002268 /* synchronize mems_allowed to N_MEMORY */
2269 if (mems_updated) {
Tejun Heodeb7aa32013-01-07 08:51:07 -08002270 mutex_lock(&callback_mutex);
Li Zefan7e882912014-07-09 16:48:42 +08002271 if (!on_dfl)
2272 top_cpuset.mems_allowed = new_mems;
Li Zefan1344ab92014-07-09 16:47:16 +08002273 top_cpuset.effective_mems = new_mems;
Tejun Heodeb7aa32013-01-07 08:51:07 -08002274 mutex_unlock(&callback_mutex);
Tejun Heod66393e2014-02-13 06:58:40 -05002275 update_tasks_nodemask(&top_cpuset);
Tejun Heodeb7aa32013-01-07 08:51:07 -08002276 }
2277
Tejun Heo5d21cc22013-01-07 08:51:08 -08002278 mutex_unlock(&cpuset_mutex);
Tejun Heodeb7aa32013-01-07 08:51:07 -08002279
Li Zefan5c5cc622013-06-09 17:16:29 +08002280 /* if cpus or mems changed, we need to propagate to descendants */
2281 if (cpus_updated || mems_updated) {
Lai Jiangshan38d7bee2012-12-12 13:51:24 -08002282 struct cpuset *cs;
Tejun Heo492eb212013-08-08 20:11:25 -04002283 struct cgroup_subsys_state *pos_css;
Paul Jacksonb4501292008-02-07 00:14:47 -08002284
Paul Jacksonb4501292008-02-07 00:14:47 -08002285 rcu_read_lock();
Tejun Heo492eb212013-08-08 20:11:25 -04002286 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
Tejun Heoec903c02014-05-13 12:11:01 -04002287 if (cs == &top_cpuset || !css_tryget_online(&cs->css))
Li Zefan388afd82013-06-09 17:14:47 +08002288 continue;
2289 rcu_read_unlock();
2290
2291 cpuset_hotplug_update_tasks(cs);
2292
2293 rcu_read_lock();
2294 css_put(&cs->css);
2295 }
Tejun Heodeb7aa32013-01-07 08:51:07 -08002296 rcu_read_unlock();
2297 }
Tejun Heo8d033942013-01-07 08:51:07 -08002298
Tejun Heodeb7aa32013-01-07 08:51:07 -08002299 /* rebuild sched domains if cpus_allowed has changed */
Li Zhonge0e80a02013-04-27 06:52:43 -07002300 if (cpus_updated)
2301 rebuild_sched_domains();
Paul Jacksonb1aac8b2006-09-29 02:01:17 -07002302}
2303
Srivatsa S. Bhat7ddf96b2012-05-24 19:46:55 +05302304void cpuset_update_active_cpus(bool cpu_online)
Paul Jackson4c4d50f2006-08-27 01:23:51 -07002305{
Tejun Heo3a5a6d02013-01-07 08:51:07 -08002306 /*
2307 * We're inside cpu hotplug critical region which usually nests
2308 * inside cgroup synchronization. Bounce actual hotplug processing
2309 * to a work item to avoid reverse locking order.
2310 *
2311 * We still need to do partition_sched_domains() synchronously;
2312 * otherwise, the scheduler will get confused and put tasks to the
2313 * dead CPU. Fall back to the default single domain.
2314 * cpuset_hotplug_workfn() will rebuild it as necessary.
2315 */
2316 partition_sched_domains(1, NULL, NULL);
2317 schedule_work(&cpuset_hotplug_work);
Paul Jackson4c4d50f2006-08-27 01:23:51 -07002318}
Paul Jackson4c4d50f2006-08-27 01:23:51 -07002319
Paul Jackson38837fc2006-09-29 02:01:16 -07002320/*
Lai Jiangshan38d7bee2012-12-12 13:51:24 -08002321 * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
2322 * Call this routine anytime after node_states[N_MEMORY] changes.
Srivatsa S. Bhata1cd2b12012-05-24 19:47:03 +05302323 * See cpuset_update_active_cpus() for CPU hotplug handling.
Paul Jackson38837fc2006-09-29 02:01:16 -07002324 */
Miao Xief4818912008-11-19 15:36:30 -08002325static int cpuset_track_online_nodes(struct notifier_block *self,
2326 unsigned long action, void *arg)
Paul Jackson38837fc2006-09-29 02:01:16 -07002327{
Tejun Heo3a5a6d02013-01-07 08:51:07 -08002328 schedule_work(&cpuset_hotplug_work);
Miao Xief4818912008-11-19 15:36:30 -08002329 return NOTIFY_OK;
Paul Jackson38837fc2006-09-29 02:01:16 -07002330}
Andrew Mortond8f10cb2013-04-29 15:08:08 -07002331
2332static struct notifier_block cpuset_track_online_nodes_nb = {
2333 .notifier_call = cpuset_track_online_nodes,
2334 .priority = 10, /* ??! */
2335};
Paul Jackson38837fc2006-09-29 02:01:16 -07002336
Linus Torvalds1da177e2005-04-16 15:20:36 -07002337/**
2338 * cpuset_init_smp - initialize cpus_allowed
2339 *
2340 * Description: Finish top cpuset after cpu, node maps are initialized
Andrew Mortond8f10cb2013-04-29 15:08:08 -07002341 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002342void __init cpuset_init_smp(void)
2343{
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01002344 cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
Lai Jiangshan38d7bee2012-12-12 13:51:24 -08002345 top_cpuset.mems_allowed = node_states[N_MEMORY];
Li Zefan33ad8012013-06-09 17:15:08 +08002346 top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
Paul Jackson4c4d50f2006-08-27 01:23:51 -07002347
Li Zefane2b9a3d2014-07-09 16:47:03 +08002348 cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
2349 top_cpuset.effective_mems = node_states[N_MEMORY];
2350
Andrew Mortond8f10cb2013-04-29 15:08:08 -07002351 register_hotmemory_notifier(&cpuset_track_online_nodes_nb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352}
2353
2354/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002355 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
2356 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
Li Zefan6af866a2009-01-07 18:08:45 -08002357 * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002358 *
Li Zefan300ed6c2009-01-07 18:08:44 -08002359 * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
Linus Torvalds1da177e2005-04-16 15:20:36 -07002360 * attached to the specified @tsk. Guaranteed to return some non-empty
Rusty Russell5f054e32012-03-29 15:38:31 +10302361 * subset of cpu_online_mask, even if this means going outside the
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362 * tasks cpuset.
2363 **/
2364
Li Zefan6af866a2009-01-07 18:08:45 -08002365void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366{
Ingo Molnar3d3f26a2006-03-23 03:00:18 -08002367 mutex_lock(&callback_mutex);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002368 rcu_read_lock();
Li Zefanae1c8022014-07-09 16:48:32 +08002369 guarantee_online_cpus(task_cs(tsk), pmask);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002370 rcu_read_unlock();
Oleg Nesterov897f0b32010-03-15 10:10:03 +01002371 mutex_unlock(&callback_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002372}
2373
Peter Zijlstra2baab4e2012-03-20 15:57:01 +01002374void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
Oleg Nesterov9084bb82010-03-15 10:10:27 +01002375{
Oleg Nesterov9084bb82010-03-15 10:10:27 +01002376 rcu_read_lock();
Li Zefanae1c8022014-07-09 16:48:32 +08002377 do_set_cpus_allowed(tsk, task_cs(tsk)->effective_cpus);
Oleg Nesterov9084bb82010-03-15 10:10:27 +01002378 rcu_read_unlock();
2379
2380 /*
2381 * We own tsk->cpus_allowed, nobody can change it under us.
2382 *
2383 * But we used cs && cs->cpus_allowed lockless and thus can
2384 * race with cgroup_attach_task() or update_cpumask() and get
2385 * the wrong tsk->cpus_allowed. However, both cases imply the
2386 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
2387 * which takes task_rq_lock().
2388 *
2389 * If we are called after it dropped the lock we must see all
2390 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
2391 * set any mask even if it is not right from task_cs() pov,
2392 * the pending set_cpus_allowed_ptr() will fix things.
Peter Zijlstra2baab4e2012-03-20 15:57:01 +01002393 *
2394 * select_fallback_rq() will fix things ups and set cpu_possible_mask
2395 * if required.
Oleg Nesterov9084bb82010-03-15 10:10:27 +01002396 */
Oleg Nesterov9084bb82010-03-15 10:10:27 +01002397}
2398
Linus Torvalds1da177e2005-04-16 15:20:36 -07002399void cpuset_init_current_mems_allowed(void)
2400{
Mike Travisf9a86fc2008-04-04 18:11:07 -07002401 nodes_setall(current->mems_allowed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002402}
2403
Randy Dunlapd9fd8a62005-07-27 11:45:11 -07002404/**
Paul Jackson909d75a2006-01-08 01:01:55 -08002405 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
2406 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
2407 *
2408 * Description: Returns the nodemask_t mems_allowed of the cpuset
2409 * attached to the specified @tsk. Guaranteed to return some non-empty
Lai Jiangshan38d7bee2012-12-12 13:51:24 -08002410 * subset of node_states[N_MEMORY], even if this means going outside the
Paul Jackson909d75a2006-01-08 01:01:55 -08002411 * tasks cpuset.
2412 **/
2413
2414nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
2415{
2416 nodemask_t mask;
2417
Ingo Molnar3d3f26a2006-03-23 03:00:18 -08002418 mutex_lock(&callback_mutex);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002419 rcu_read_lock();
Li Zefanae1c8022014-07-09 16:48:32 +08002420 guarantee_online_mems(task_cs(tsk), &mask);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002421 rcu_read_unlock();
Ingo Molnar3d3f26a2006-03-23 03:00:18 -08002422 mutex_unlock(&callback_mutex);
Paul Jackson909d75a2006-01-08 01:01:55 -08002423
2424 return mask;
2425}
2426
2427/**
Mel Gorman19770b32008-04-28 02:12:18 -07002428 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
2429 * @nodemask: the nodemask to be checked
Randy Dunlapd9fd8a62005-07-27 11:45:11 -07002430 *
Mel Gorman19770b32008-04-28 02:12:18 -07002431 * Are any of the nodes in the nodemask allowed in current->mems_allowed?
Linus Torvalds1da177e2005-04-16 15:20:36 -07002432 */
Mel Gorman19770b32008-04-28 02:12:18 -07002433int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002434{
Mel Gorman19770b32008-04-28 02:12:18 -07002435 return nodes_intersects(*nodemask, current->mems_allowed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002436}
2437
Paul Jackson9bf22292005-09-06 15:18:12 -07002438/*
Paul Menage78608362008-04-29 01:00:26 -07002439 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
2440 * mem_hardwall ancestor to the specified cpuset. Call holding
2441 * callback_mutex. If no ancestor is mem_exclusive or mem_hardwall
2442 * (an unusual configuration), then returns the root cpuset.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002443 */
Tejun Heoc9710d82013-08-08 20:11:22 -04002444static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002445{
Tejun Heoc4310692013-01-07 08:51:08 -08002446 while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
2447 cs = parent_cs(cs);
Paul Jackson9bf22292005-09-06 15:18:12 -07002448 return cs;
2449}
2450
2451/**
David Rientjesa1bc5a42009-04-02 16:57:54 -07002452 * cpuset_node_allowed_softwall - Can we allocate on a memory node?
2453 * @node: is this an allowed node?
Paul Jackson02a0e532006-12-13 00:34:25 -08002454 * @gfp_mask: memory allocation flags
Paul Jackson9bf22292005-09-06 15:18:12 -07002455 *
David Rientjesa1bc5a42009-04-02 16:57:54 -07002456 * If we're in interrupt, yes, we can always allocate. If __GFP_THISNODE is
2457 * set, yes, we can always allocate. If node is in our task's mems_allowed,
2458 * yes. If it's not a __GFP_HARDWALL request and this node is in the nearest
2459 * hardwalled cpuset ancestor to this task's cpuset, yes. If the task has been
2460 * OOM killed and has access to memory reserves as specified by the TIF_MEMDIE
2461 * flag, yes.
Paul Jackson9bf22292005-09-06 15:18:12 -07002462 * Otherwise, no.
2463 *
David Rientjesa1bc5a42009-04-02 16:57:54 -07002464 * If __GFP_HARDWALL is set, cpuset_node_allowed_softwall() reduces to
2465 * cpuset_node_allowed_hardwall(). Otherwise, cpuset_node_allowed_softwall()
2466 * might sleep, and might allow a node from an enclosing cpuset.
Paul Jackson02a0e532006-12-13 00:34:25 -08002467 *
David Rientjesa1bc5a42009-04-02 16:57:54 -07002468 * cpuset_node_allowed_hardwall() only handles the simpler case of hardwall
2469 * cpusets, and never sleeps.
Paul Jackson02a0e532006-12-13 00:34:25 -08002470 *
2471 * The __GFP_THISNODE placement logic is really handled elsewhere,
2472 * by forcibly using a zonelist starting at a specified node, and by
2473 * (in get_page_from_freelist()) refusing to consider the zones for
2474 * any node on the zonelist except the first. By the time any such
2475 * calls get to this routine, we should just shut up and say 'yes'.
2476 *
Paul Jackson9bf22292005-09-06 15:18:12 -07002477 * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
David Rientjesc596d9f2007-05-06 14:49:32 -07002478 * and do not allow allocations outside the current tasks cpuset
2479 * unless the task has been OOM killed as is marked TIF_MEMDIE.
Paul Jackson9bf22292005-09-06 15:18:12 -07002480 * GFP_KERNEL allocations are not so marked, so can escape to the
Paul Menage78608362008-04-29 01:00:26 -07002481 * nearest enclosing hardwalled ancestor cpuset.
Paul Jackson9bf22292005-09-06 15:18:12 -07002482 *
Paul Jackson02a0e532006-12-13 00:34:25 -08002483 * Scanning up parent cpusets requires callback_mutex. The
2484 * __alloc_pages() routine only calls here with __GFP_HARDWALL bit
2485 * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
2486 * current tasks mems_allowed came up empty on the first pass over
2487 * the zonelist. So only GFP_KERNEL allocations, if all nodes in the
2488 * cpuset are short of memory, might require taking the callback_mutex
2489 * mutex.
Paul Jackson9bf22292005-09-06 15:18:12 -07002490 *
Paul Jackson36be57f2006-05-20 15:00:10 -07002491 * The first call here from mm/page_alloc:get_page_from_freelist()
Paul Jackson02a0e532006-12-13 00:34:25 -08002492 * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
2493 * so no allocation on a node outside the cpuset is allowed (unless
2494 * in interrupt, of course).
Paul Jackson9bf22292005-09-06 15:18:12 -07002495 *
Paul Jackson36be57f2006-05-20 15:00:10 -07002496 * The second pass through get_page_from_freelist() doesn't even call
2497 * here for GFP_ATOMIC calls. For those calls, the __alloc_pages()
2498 * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
2499 * in alloc_flags. That logic and the checks below have the combined
2500 * affect that:
Paul Jackson9bf22292005-09-06 15:18:12 -07002501 * in_interrupt - any node ok (current task context irrelevant)
2502 * GFP_ATOMIC - any node ok
David Rientjesc596d9f2007-05-06 14:49:32 -07002503 * TIF_MEMDIE - any node ok
Paul Menage78608362008-04-29 01:00:26 -07002504 * GFP_KERNEL - any node in enclosing hardwalled cpuset ok
Paul Jackson9bf22292005-09-06 15:18:12 -07002505 * GFP_USER - only nodes in current tasks mems allowed ok.
Paul Jackson36be57f2006-05-20 15:00:10 -07002506 *
2507 * Rule:
David Rientjesa1bc5a42009-04-02 16:57:54 -07002508 * Don't call cpuset_node_allowed_softwall if you can't sleep, unless you
Paul Jackson36be57f2006-05-20 15:00:10 -07002509 * pass in the __GFP_HARDWALL flag set in gfp_flag, which disables
2510 * the code that might scan up ancestor cpusets and sleep.
Paul Jackson02a0e532006-12-13 00:34:25 -08002511 */
David Rientjesa1bc5a42009-04-02 16:57:54 -07002512int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
Paul Jackson9bf22292005-09-06 15:18:12 -07002513{
Tejun Heoc9710d82013-08-08 20:11:22 -04002514 struct cpuset *cs; /* current cpuset ancestors */
Paul Jackson29afd492006-03-24 03:16:12 -08002515 int allowed; /* is allocation in zone z allowed? */
Paul Jackson9bf22292005-09-06 15:18:12 -07002516
Christoph Lameter9b819d22006-09-25 23:31:40 -07002517 if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
Paul Jackson9bf22292005-09-06 15:18:12 -07002518 return 1;
Paul Jackson92d1dbd2006-05-20 15:00:11 -07002519 might_sleep_if(!(gfp_mask & __GFP_HARDWALL));
Paul Jackson9bf22292005-09-06 15:18:12 -07002520 if (node_isset(node, current->mems_allowed))
2521 return 1;
David Rientjesc596d9f2007-05-06 14:49:32 -07002522 /*
2523 * Allow tasks that have access to memory reserves because they have
2524 * been OOM killed to get memory anywhere.
2525 */
2526 if (unlikely(test_thread_flag(TIF_MEMDIE)))
2527 return 1;
Paul Jackson9bf22292005-09-06 15:18:12 -07002528 if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
2529 return 0;
2530
Bob Picco5563e772005-11-13 16:06:35 -08002531 if (current->flags & PF_EXITING) /* Let dying task have memory */
2532 return 1;
2533
Paul Jackson9bf22292005-09-06 15:18:12 -07002534 /* Not hardwall and node outside mems_allowed: scan up cpusets */
Ingo Molnar3d3f26a2006-03-23 03:00:18 -08002535 mutex_lock(&callback_mutex);
Paul Jackson053199e2005-10-30 15:02:30 -08002536
Li Zefanb8dadcb2014-03-03 17:28:36 -05002537 rcu_read_lock();
Paul Menage78608362008-04-29 01:00:26 -07002538 cs = nearest_hardwall_ancestor(task_cs(current));
Li Zefan99afb0f2014-02-27 18:19:36 +08002539 allowed = node_isset(node, cs->mems_allowed);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002540 rcu_read_unlock();
Paul Jackson053199e2005-10-30 15:02:30 -08002541
Ingo Molnar3d3f26a2006-03-23 03:00:18 -08002542 mutex_unlock(&callback_mutex);
Paul Jackson9bf22292005-09-06 15:18:12 -07002543 return allowed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002544}
2545
Paul Jackson02a0e532006-12-13 00:34:25 -08002546/*
David Rientjesa1bc5a42009-04-02 16:57:54 -07002547 * cpuset_node_allowed_hardwall - Can we allocate on a memory node?
2548 * @node: is this an allowed node?
Paul Jackson02a0e532006-12-13 00:34:25 -08002549 * @gfp_mask: memory allocation flags
2550 *
David Rientjesa1bc5a42009-04-02 16:57:54 -07002551 * If we're in interrupt, yes, we can always allocate. If __GFP_THISNODE is
2552 * set, yes, we can always allocate. If node is in our task's mems_allowed,
2553 * yes. If the task has been OOM killed and has access to memory reserves as
2554 * specified by the TIF_MEMDIE flag, yes.
2555 * Otherwise, no.
Paul Jackson02a0e532006-12-13 00:34:25 -08002556 *
2557 * The __GFP_THISNODE placement logic is really handled elsewhere,
2558 * by forcibly using a zonelist starting at a specified node, and by
2559 * (in get_page_from_freelist()) refusing to consider the zones for
2560 * any node on the zonelist except the first. By the time any such
2561 * calls get to this routine, we should just shut up and say 'yes'.
2562 *
David Rientjesa1bc5a42009-04-02 16:57:54 -07002563 * Unlike the cpuset_node_allowed_softwall() variant, above,
2564 * this variant requires that the node be in the current task's
Paul Jackson02a0e532006-12-13 00:34:25 -08002565 * mems_allowed or that we're in interrupt. It does not scan up the
2566 * cpuset hierarchy for the nearest enclosing mem_exclusive cpuset.
2567 * It never sleeps.
2568 */
David Rientjesa1bc5a42009-04-02 16:57:54 -07002569int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
Paul Jackson02a0e532006-12-13 00:34:25 -08002570{
Paul Jackson02a0e532006-12-13 00:34:25 -08002571 if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
2572 return 1;
Paul Jackson02a0e532006-12-13 00:34:25 -08002573 if (node_isset(node, current->mems_allowed))
2574 return 1;
Daniel Walkerdedf8b72007-10-18 03:06:04 -07002575 /*
2576 * Allow tasks that have access to memory reserves because they have
2577 * been OOM killed to get memory anywhere.
2578 */
2579 if (unlikely(test_thread_flag(TIF_MEMDIE)))
2580 return 1;
Paul Jackson02a0e532006-12-13 00:34:25 -08002581 return 0;
2582}
2583
Paul Jacksonef08e3b2005-09-06 15:18:13 -07002584/**
Jack Steiner6adef3e2010-05-26 14:42:49 -07002585 * cpuset_mem_spread_node() - On which node to begin search for a file page
2586 * cpuset_slab_spread_node() - On which node to begin search for a slab page
Paul Jackson825a46a2006-03-24 03:16:03 -08002587 *
2588 * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
2589 * tasks in a cpuset with is_spread_page or is_spread_slab set),
2590 * and if the memory allocation used cpuset_mem_spread_node()
2591 * to determine on which node to start looking, as it will for
2592 * certain page cache or slab cache pages such as used for file
2593 * system buffers and inode caches, then instead of starting on the
2594 * local node to look for a free page, rather spread the starting
2595 * node around the tasks mems_allowed nodes.
2596 *
2597 * We don't have to worry about the returned node being offline
2598 * because "it can't happen", and even if it did, it would be ok.
2599 *
2600 * The routines calling guarantee_online_mems() are careful to
2601 * only set nodes in task->mems_allowed that are online. So it
2602 * should not be possible for the following code to return an
2603 * offline node. But if it did, that would be ok, as this routine
2604 * is not returning the node where the allocation must be, only
2605 * the node where the search should start. The zonelist passed to
2606 * __alloc_pages() will include all nodes. If the slab allocator
2607 * is passed an offline node, it will fall back to the local node.
2608 * See kmem_cache_alloc_node().
2609 */
2610
Jack Steiner6adef3e2010-05-26 14:42:49 -07002611static int cpuset_spread_node(int *rotor)
Paul Jackson825a46a2006-03-24 03:16:03 -08002612{
2613 int node;
2614
Jack Steiner6adef3e2010-05-26 14:42:49 -07002615 node = next_node(*rotor, current->mems_allowed);
Paul Jackson825a46a2006-03-24 03:16:03 -08002616 if (node == MAX_NUMNODES)
2617 node = first_node(current->mems_allowed);
Jack Steiner6adef3e2010-05-26 14:42:49 -07002618 *rotor = node;
Paul Jackson825a46a2006-03-24 03:16:03 -08002619 return node;
2620}
Jack Steiner6adef3e2010-05-26 14:42:49 -07002621
2622int cpuset_mem_spread_node(void)
2623{
Michal Hocko778d3b02011-07-26 16:08:30 -07002624 if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
2625 current->cpuset_mem_spread_rotor =
2626 node_random(&current->mems_allowed);
2627
Jack Steiner6adef3e2010-05-26 14:42:49 -07002628 return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
2629}
2630
2631int cpuset_slab_spread_node(void)
2632{
Michal Hocko778d3b02011-07-26 16:08:30 -07002633 if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE)
2634 current->cpuset_slab_spread_rotor =
2635 node_random(&current->mems_allowed);
2636
Jack Steiner6adef3e2010-05-26 14:42:49 -07002637 return cpuset_spread_node(&current->cpuset_slab_spread_rotor);
2638}
2639
Paul Jackson825a46a2006-03-24 03:16:03 -08002640EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);
2641
2642/**
David Rientjesbbe373f2007-10-16 23:25:58 -07002643 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
2644 * @tsk1: pointer to task_struct of some task.
2645 * @tsk2: pointer to task_struct of some other task.
Paul Jacksonef08e3b2005-09-06 15:18:13 -07002646 *
David Rientjesbbe373f2007-10-16 23:25:58 -07002647 * Description: Return true if @tsk1's mems_allowed intersects the
2648 * mems_allowed of @tsk2. Used by the OOM killer to determine if
2649 * one of the task's memory usage might impact the memory available
2650 * to the other.
Paul Jacksonef08e3b2005-09-06 15:18:13 -07002651 **/
2652
David Rientjesbbe373f2007-10-16 23:25:58 -07002653int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
2654 const struct task_struct *tsk2)
Paul Jacksonef08e3b2005-09-06 15:18:13 -07002655{
David Rientjesbbe373f2007-10-16 23:25:58 -07002656 return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
Paul Jacksonef08e3b2005-09-06 15:18:13 -07002657}
2658
Li Zefanf440d982013-03-01 15:02:15 +08002659#define CPUSET_NODELIST_LEN (256)
2660
David Rientjes75aa1992009-01-06 14:39:01 -08002661/**
2662 * cpuset_print_task_mems_allowed - prints task's cpuset and mems_allowed
Fabian Frederickfc34ac12014-05-05 19:46:55 +02002663 * @tsk: pointer to task_struct of some task.
David Rientjes75aa1992009-01-06 14:39:01 -08002664 *
2665 * Description: Prints @task's name, cpuset name, and cached copy of its
Li Zefanb8dadcb2014-03-03 17:28:36 -05002666 * mems_allowed to the kernel log.
David Rientjes75aa1992009-01-06 14:39:01 -08002667 */
2668void cpuset_print_task_mems_allowed(struct task_struct *tsk)
2669{
Li Zefanf440d982013-03-01 15:02:15 +08002670 /* Statically allocated to prevent using excess stack. */
2671 static char cpuset_nodelist[CPUSET_NODELIST_LEN];
2672 static DEFINE_SPINLOCK(cpuset_buffer_lock);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002673 struct cgroup *cgrp;
David Rientjes75aa1992009-01-06 14:39:01 -08002674
David Rientjes75aa1992009-01-06 14:39:01 -08002675 spin_lock(&cpuset_buffer_lock);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002676 rcu_read_lock();
Li Zefan63f43f52013-01-25 16:08:01 +08002677
Li Zefanb8dadcb2014-03-03 17:28:36 -05002678 cgrp = task_cs(tsk)->css.cgroup;
David Rientjes75aa1992009-01-06 14:39:01 -08002679 nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,
2680 tsk->mems_allowed);
Fabian Frederick12d30892014-05-05 19:49:00 +02002681 pr_info("%s cpuset=", tsk->comm);
Tejun Heoe61734c2014-02-12 09:29:50 -05002682 pr_cont_cgroup_name(cgrp);
2683 pr_cont(" mems_allowed=%s\n", cpuset_nodelist);
Li Zefanf440d982013-03-01 15:02:15 +08002684
Li Zefancfb59662013-03-12 10:28:39 +08002685 rcu_read_unlock();
Paul Jackson3e0d98b2006-01-08 01:01:49 -08002686 spin_unlock(&cpuset_buffer_lock);
Paul Jackson3e0d98b2006-01-08 01:01:49 -08002687}
Paul Menage8793d852007-10-18 23:39:39 -07002688
Paul Jackson3e0d98b2006-01-08 01:01:49 -08002689/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002690 * Collection of memory_pressure is suppressed unless
2691 * this flag is enabled by writing "1" to the special
2692 * cpuset file 'memory_pressure_enabled' in the root cpuset.
Paul Jackson053199e2005-10-30 15:02:30 -08002693 */
2694
Paul Jacksonc8d9c902008-02-07 00:14:46 -08002695int cpuset_memory_pressure_enabled __read_mostly;
Paul Menage2df167a2008-02-07 00:14:45 -08002696
Linus Torvalds1da177e2005-04-16 15:20:36 -07002697/**
Paul Jackson029190c2007-10-18 23:40:20 -07002698 * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002699 *
Eric W. Biederman13b41b02006-06-26 00:25:56 -07002700 * Keep a running average of the rate of synchronous (direct)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701 * page reclaim efforts initiated by tasks in each cpuset.
2702 *
Paul Menage8793d852007-10-18 23:39:39 -07002703 * This represents the rate at which some task in the cpuset
Eric W. Biederman99f89552006-06-26 00:25:55 -07002704 * ran low on memory on all nodes it was allowed to use, and
Linus Torvalds1da177e2005-04-16 15:20:36 -07002705 * had to enter the kernels page reclaim code in an effort to
Eric W. Biederman99f89552006-06-26 00:25:55 -07002706 * create more free memory by tossing clean pages or swapping
Linus Torvalds1da177e2005-04-16 15:20:36 -07002707 * or writing dirty pages.
2708 *
Eric W. Biederman99f89552006-06-26 00:25:55 -07002709 * Display to user space in the per-cpuset read-only file
Linus Torvalds1da177e2005-04-16 15:20:36 -07002710 * "memory_pressure". Value displayed is an integer
Eric W. Biederman99f89552006-06-26 00:25:55 -07002711 * representing the recent rate of entry into the synchronous
Eric W. Biederman13b41b02006-06-26 00:25:56 -07002712 * (direct) page reclaim by any task attached to the cpuset.
2713 **/
Eric W. Biederman99f89552006-06-26 00:25:55 -07002714
2715void __cpuset_memory_pressure_bump(void)
2716{
Li Zefanb8dadcb2014-03-03 17:28:36 -05002717 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718 fmeter_markevent(&task_cs(current)->fmeter);
Li Zefanb8dadcb2014-03-03 17:28:36 -05002719 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002720}
2721
2722#ifdef CONFIG_PROC_PID_CPUSET
2723/*
2724 * proc_cpuset_show()
2725 * - Print tasks cpuset path into seq_file.
2726 * - Used for /proc/<pid>/cpuset.
2727 * - No need to task_lock(tsk) on this tsk->cpuset reference, as it
2728 * doesn't really matter if tsk->cpuset changes after we read it,
Tejun Heo5d21cc22013-01-07 08:51:08 -08002729 * and we take cpuset_mutex, keeping cpuset_attach() from changing it
Linus Torvalds1da177e2005-04-16 15:20:36 -07002730 * anyway.
2731 */
Al Viro8d8b97b2013-04-19 23:11:24 -04002732int proc_cpuset_show(struct seq_file *m, void *unused_v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002733{
2734 struct pid *pid;
2735 struct task_struct *tsk;
Tejun Heoe61734c2014-02-12 09:29:50 -05002736 char *buf, *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002737 struct cgroup_subsys_state *css;
2738 int retval;
2739
2740 retval = -ENOMEM;
Tejun Heoe61734c2014-02-12 09:29:50 -05002741 buf = kmalloc(PATH_MAX, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002742 if (!buf)
2743 goto out;
2744
2745 retval = -ESRCH;
2746 pid = m->private;
2747 tsk = get_pid_task(pid, PIDTYPE_PID);
2748 if (!tsk)
2749 goto out_free;
2750
Tejun Heoe61734c2014-02-12 09:29:50 -05002751 retval = -ENAMETOOLONG;
Li Zefan27e89ae2013-01-15 14:10:57 +08002752 rcu_read_lock();
Tejun Heo073219e2014-02-08 10:36:58 -05002753 css = task_css(tsk, cpuset_cgrp_id);
Tejun Heoe61734c2014-02-12 09:29:50 -05002754 p = cgroup_path(css->cgroup, buf, PATH_MAX);
Li Zefan27e89ae2013-01-15 14:10:57 +08002755 rcu_read_unlock();
Tejun Heoe61734c2014-02-12 09:29:50 -05002756 if (!p)
Li Zefan27e89ae2013-01-15 14:10:57 +08002757 goto out_put_task;
Tejun Heoe61734c2014-02-12 09:29:50 -05002758 seq_puts(m, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002759 seq_putc(m, '\n');
Tejun Heoe61734c2014-02-12 09:29:50 -05002760 retval = 0;
Li Zefan27e89ae2013-01-15 14:10:57 +08002761out_put_task:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002762 put_task_struct(tsk);
2763out_free:
2764 kfree(buf);
2765out:
2766 return retval;
2767}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002768#endif /* CONFIG_PROC_PID_CPUSET */
2769
Heiko Carstensd01d4822009-09-21 11:06:27 +02002770/* Display task mems_allowed in /proc/<pid>/status file. */
Eric W. Biedermandf5f8312008-02-08 04:18:33 -08002771void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002772{
Fabian Frederickfc34ac12014-05-05 19:46:55 +02002773 seq_puts(m, "Mems_allowed:\t");
Lai Jiangshan30e8e132008-10-18 20:28:20 -07002774 seq_nodemask(m, &task->mems_allowed);
Fabian Frederickfc34ac12014-05-05 19:46:55 +02002775 seq_puts(m, "\n");
2776 seq_puts(m, "Mems_allowed_list:\t");
Lai Jiangshan30e8e132008-10-18 20:28:20 -07002777 seq_nodemask_list(m, &task->mems_allowed);
Fabian Frederickfc34ac12014-05-05 19:46:55 +02002778 seq_puts(m, "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002779}