blob: 5f627084f2e998b2605016c311411d91f7016918 [file] [log] [blame]
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09001#include <linux/slab.h>
Andrew Mortonccb46002006-03-25 03:08:08 -08002#include <linux/kernel.h>
3#include <linux/bitops.h>
4#include <linux/cpumask.h>
Paul Gortmaker8bc3bcc2011-11-16 21:29:17 -05005#include <linux/export.h>
Rusty Russell2d3854a2008-11-05 13:39:10 +11006#include <linux/bootmem.h>
Andrew Mortonccb46002006-03-25 03:08:08 -08007
Rusty Russell2d3854a2008-11-05 13:39:10 +11008/**
9 * cpumask_next_and - get the next cpu in *src1p & *src2p
10 * @n: the cpu prior to the place to search (ie. return will be > @n)
11 * @src1p: the first cpumask pointer
12 * @src2p: the second cpumask pointer
13 *
14 * Returns >= nr_cpu_ids if no further cpus set in both.
15 */
16int cpumask_next_and(int n, const struct cpumask *src1p,
17 const struct cpumask *src2p)
18{
Sergey Senozhatsky534b4832015-04-16 12:48:04 -070019 struct cpumask tmp;
20
21 if (cpumask_and(&tmp, src1p, src2p))
22 return cpumask_next(n, &tmp);
23 return nr_cpu_ids;
Rusty Russell2d3854a2008-11-05 13:39:10 +110024}
25EXPORT_SYMBOL(cpumask_next_and);
26
27/**
28 * cpumask_any_but - return a "random" in a cpumask, but not this one.
29 * @mask: the cpumask to search
30 * @cpu: the cpu to ignore.
31 *
32 * Often used to find any cpu but smp_processor_id() in a mask.
33 * Returns >= nr_cpu_ids if no cpus set.
34 */
35int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
36{
37 unsigned int i;
38
Rusty Russell984f2f32008-11-08 20:24:19 +110039 cpumask_check(cpu);
Rusty Russell2d3854a2008-11-05 13:39:10 +110040 for_each_cpu(i, mask)
41 if (i != cpu)
42 break;
43 return i;
44}
45
46/* These are not inline because of header tangles. */
47#ifdef CONFIG_CPUMASK_OFFSTACK
Mike Travisec26b802008-12-19 16:56:52 +103048/**
49 * alloc_cpumask_var_node - allocate a struct cpumask on a given node
50 * @mask: pointer to cpumask_var_t where the cpumask is returned
51 * @flags: GFP_ flags
52 *
53 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
54 * a nop returning a constant 1 (in <linux/cpumask.h>)
55 * Returns TRUE if memory allocation succeeded, FALSE otherwise.
56 *
57 * In addition, mask will be NULL if this fails. Note that gcc is
58 * usually smart enough to know that mask can never be NULL if
59 * CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case
60 * too.
61 */
Mike Travis7b4967c2008-12-19 16:56:37 +103062bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
Rusty Russell2d3854a2008-11-05 13:39:10 +110063{
Yinghai Lu38c7fed2009-05-25 15:10:58 +030064 *mask = kmalloc_node(cpumask_size(), flags, node);
65
Rusty Russell2d3854a2008-11-05 13:39:10 +110066#ifdef CONFIG_DEBUG_PER_CPU_MAPS
67 if (!*mask) {
68 printk(KERN_ERR "=> alloc_cpumask_var: failed!\n");
69 dump_stack();
70 }
71#endif
Rusty Russell2a530082009-01-01 10:12:30 +103072
Rusty Russell2d3854a2008-11-05 13:39:10 +110073 return *mask != NULL;
74}
Mike Travis7b4967c2008-12-19 16:56:37 +103075EXPORT_SYMBOL(alloc_cpumask_var_node);
76
Yinghai Lu0281b5d2009-06-06 14:50:36 -070077bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
78{
79 return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
80}
81EXPORT_SYMBOL(zalloc_cpumask_var_node);
82
Mike Travisec26b802008-12-19 16:56:52 +103083/**
84 * alloc_cpumask_var - allocate a struct cpumask
85 * @mask: pointer to cpumask_var_t where the cpumask is returned
86 * @flags: GFP_ flags
87 *
88 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
89 * a nop returning a constant 1 (in <linux/cpumask.h>).
90 *
91 * See alloc_cpumask_var_node.
92 */
Mike Travis7b4967c2008-12-19 16:56:37 +103093bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
94{
KOSAKI Motohiro37e7b5f2011-07-26 16:08:44 -070095 return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
Mike Travis7b4967c2008-12-19 16:56:37 +103096}
Rusty Russell2d3854a2008-11-05 13:39:10 +110097EXPORT_SYMBOL(alloc_cpumask_var);
98
Yinghai Lu0281b5d2009-06-06 14:50:36 -070099bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
100{
101 return alloc_cpumask_var(mask, flags | __GFP_ZERO);
102}
103EXPORT_SYMBOL(zalloc_cpumask_var);
104
Mike Travisec26b802008-12-19 16:56:52 +1030105/**
106 * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena.
107 * @mask: pointer to cpumask_var_t where the cpumask is returned
108 *
109 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
Li Zefane9690a62008-12-31 16:45:50 +0800110 * a nop (in <linux/cpumask.h>).
Mike Travisec26b802008-12-19 16:56:52 +1030111 * Either returns an allocated (zero-filled) cpumask, or causes the
112 * system to panic.
113 */
Rusty Russell2d3854a2008-11-05 13:39:10 +1100114void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask)
115{
Santosh Shilimkarc1529502014-01-21 15:50:32 -0800116 *mask = memblock_virt_alloc(cpumask_size(), 0);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100117}
118
Mike Travisec26b802008-12-19 16:56:52 +1030119/**
120 * free_cpumask_var - frees memory allocated for a struct cpumask.
121 * @mask: cpumask to free
122 *
123 * This is safe on a NULL mask.
124 */
Rusty Russell2d3854a2008-11-05 13:39:10 +1100125void free_cpumask_var(cpumask_var_t mask)
126{
127 kfree(mask);
128}
129EXPORT_SYMBOL(free_cpumask_var);
Rusty Russellcd83e422008-11-07 11:12:29 +1100130
Mike Travisec26b802008-12-19 16:56:52 +1030131/**
132 * free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var
133 * @mask: cpumask to free
134 */
Rusty Russell984f2f32008-11-08 20:24:19 +1100135void __init free_bootmem_cpumask_var(cpumask_var_t mask)
Rusty Russellcd83e422008-11-07 11:12:29 +1100136{
Santosh Shilimkarc1529502014-01-21 15:50:32 -0800137 memblock_free_early(__pa(mask), cpumask_size());
Rusty Russellcd83e422008-11-07 11:12:29 +1100138}
Rusty Russell2d3854a2008-11-05 13:39:10 +1100139#endif
Amir Vadaida913092014-06-09 10:24:38 +0300140
141/**
Rusty Russellf36963c2015-05-09 03:14:13 +0930142 * cpumask_local_spread - select the i'th cpu with local numa cpu's first
Amir Vadaida913092014-06-09 10:24:38 +0300143 * @i: index number
Rusty Russellf36963c2015-05-09 03:14:13 +0930144 * @node: local numa_node
Amir Vadaida913092014-06-09 10:24:38 +0300145 *
Rusty Russellf36963c2015-05-09 03:14:13 +0930146 * This function selects an online CPU according to a numa aware policy;
147 * local cpus are returned first, followed by non-local ones, then it
148 * wraps around.
Amir Vadaida913092014-06-09 10:24:38 +0300149 *
Rusty Russellf36963c2015-05-09 03:14:13 +0930150 * It's not very efficient, but useful for setup.
Amir Vadaida913092014-06-09 10:24:38 +0300151 */
Rusty Russellf36963c2015-05-09 03:14:13 +0930152unsigned int cpumask_local_spread(unsigned int i, int node)
Amir Vadaida913092014-06-09 10:24:38 +0300153{
Amir Vadaida913092014-06-09 10:24:38 +0300154 int cpu;
Amir Vadaida913092014-06-09 10:24:38 +0300155
Rusty Russellf36963c2015-05-09 03:14:13 +0930156 /* Wrap: we always want a cpu. */
Amir Vadaida913092014-06-09 10:24:38 +0300157 i %= num_online_cpus();
158
Rusty Russellf36963c2015-05-09 03:14:13 +0930159 if (node == -1) {
160 for_each_cpu(cpu, cpu_online_mask)
161 if (i-- == 0)
162 return cpu;
Amir Vadaida913092014-06-09 10:24:38 +0300163 } else {
Rusty Russellf36963c2015-05-09 03:14:13 +0930164 /* NUMA first. */
165 for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
166 if (i-- == 0)
167 return cpu;
Amir Vadaida913092014-06-09 10:24:38 +0300168
Rusty Russellf36963c2015-05-09 03:14:13 +0930169 for_each_cpu(cpu, cpu_online_mask) {
170 /* Skip NUMA nodes, done above. */
171 if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
172 continue;
Amir Vadaida913092014-06-09 10:24:38 +0300173
Rusty Russellf36963c2015-05-09 03:14:13 +0930174 if (i-- == 0)
175 return cpu;
Amir Vadaida913092014-06-09 10:24:38 +0300176 }
177 }
Rusty Russellf36963c2015-05-09 03:14:13 +0930178 BUG();
Amir Vadaida913092014-06-09 10:24:38 +0300179}
Rusty Russellf36963c2015-05-09 03:14:13 +0930180EXPORT_SYMBOL(cpumask_local_spread);