blob: d2f53f35d86a5ca5926b9e320c238b13d387e604 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * ACPI 3.0 based NUMA setup
3 * Copyright 2004 Andi Kleen, SuSE Labs.
4 *
5 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
6 *
7 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
8 * Assumes all memory regions belonging to a single proximity domain
9 * are in one chunk. Holes between them will be included in the node.
10 */
11
12#include <linux/kernel.h>
13#include <linux/acpi.h>
14#include <linux/mmzone.h>
15#include <linux/bitmap.h>
16#include <linux/module.h>
17#include <linux/topology.h>
Andi Kleen68a3a7f2006-04-07 19:49:18 +020018#include <linux/bootmem.h>
Yinghai Lua9ce6bc2010-08-25 13:39:17 -070019#include <linux/memblock.h>
Andi Kleen68a3a7f2006-04-07 19:49:18 +020020#include <linux/mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/proto.h>
22#include <asm/numa.h>
Andi Kleen8a6fdd32006-01-11 22:44:39 +010023#include <asm/e820.h>
Ingo Molnar7b6aa332009-02-17 13:58:15 +010024#include <asm/apic.h>
Ingo Molnar4ec71fa2009-01-21 10:24:27 +010025#include <asm/uv/uv.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
Andi Kleenc31fbb12006-09-26 10:52:33 +020027int acpi_numa __initdata;
28
Linus Torvalds1da177e2005-04-16 15:20:36 -070029static struct acpi_table_slit *acpi_slit;
30
Keith Mannthey4942e992006-09-30 23:27:06 -070031static struct bootnode nodes_add[MAX_NUMNODES];
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
33static __init int setup_node(int pxm)
34{
Yasunori Goto762834e2006-06-23 02:03:19 -070035 return acpi_map_pxm_to_node(pxm);
Linus Torvalds1da177e2005-04-16 15:20:36 -070036}
37
Linus Torvalds1da177e2005-04-16 15:20:36 -070038static __init void bad_srat(void)
39{
40 printk(KERN_ERR "SRAT: SRAT not used.\n");
41 acpi_numa = -1;
Tejun Heo91556232011-02-16 17:11:09 +010042 memset(nodes_add, 0, sizeof(nodes_add));
Linus Torvalds1da177e2005-04-16 15:20:36 -070043}
44
45static __init inline int srat_disabled(void)
46{
Tejun Heoffe77a42011-02-16 12:13:06 +010047 return acpi_numa < 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048}
49
50/* Callback for SLIT parsing */
51void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
52{
Tejun Heoac7136b2011-02-16 17:11:09 +010053 int i, j;
Yinghai Luf302a5bb2008-07-10 20:36:37 -070054 unsigned length;
55 unsigned long phys;
56
Tejun Heoac7136b2011-02-16 17:11:09 +010057 for (i = 0; i < slit->locality_count; i++)
58 for (j = 0; j < slit->locality_count; j++)
59 numa_set_distance(pxm_to_node(i), pxm_to_node(j),
60 slit->entry[slit->locality_count * i + j]);
61
62 /* acpi_slit is used only by emulation */
Yinghai Luf302a5bb2008-07-10 20:36:37 -070063 length = slit->header.length;
Yinghai Lua9ce6bc2010-08-25 13:39:17 -070064 phys = memblock_find_in_range(0, max_pfn_mapped<<PAGE_SHIFT, length,
Yinghai Luf302a5bb2008-07-10 20:36:37 -070065 PAGE_SIZE);
66
Yinghai Lua9ce6bc2010-08-25 13:39:17 -070067 if (phys == MEMBLOCK_ERROR)
Yinghai Luf302a5bb2008-07-10 20:36:37 -070068 panic(" Can not save slit!\n");
69
70 acpi_slit = __va(phys);
71 memcpy(acpi_slit, slit, length);
Yinghai Lua9ce6bc2010-08-25 13:39:17 -070072 memblock_x86_reserve_range(phys, phys + length, "ACPI SLIT");
Linus Torvalds1da177e2005-04-16 15:20:36 -070073}
74
Suresh Siddha7237d3d2009-03-30 13:55:30 -080075/* Callback for Proximity Domain -> x2APIC mapping */
76void __init
77acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
78{
79 int pxm, node;
80 int apic_id;
81
82 if (srat_disabled())
83 return;
84 if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
85 bad_srat();
86 return;
87 }
88 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
89 return;
90 pxm = pa->proximity_domain;
91 node = setup_node(pxm);
92 if (node < 0) {
93 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
94 bad_srat();
95 return;
96 }
97
98 apic_id = pa->apic_id;
Yinghai Lud3bd0582010-12-16 19:09:58 -080099 if (apic_id >= MAX_LOCAL_APIC) {
100 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
101 return;
102 }
Tejun Heobbc9e2f2011-01-23 14:37:39 +0100103 set_apicid_to_node(apic_id, node);
Tejun Heo92d4a432011-02-16 17:11:09 +0100104 node_set(node, numa_nodes_parsed);
Suresh Siddha7237d3d2009-03-30 13:55:30 -0800105 acpi_numa = 1;
Yinghai Lu163d3862009-11-21 00:23:37 -0800106 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
Suresh Siddha7237d3d2009-03-30 13:55:30 -0800107 pxm, apic_id, node);
108}
109
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110/* Callback for Proximity Domain -> LAPIC mapping */
111void __init
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300112acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113{
114 int pxm, node;
travis@sgi.comef970012008-01-30 13:33:10 +0100115 int apic_id;
116
Andi Kleend22fe802006-02-03 21:51:26 +0100117 if (srat_disabled())
118 return;
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300119 if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
Andi Kleenfad79062006-05-15 18:19:44 +0200120 bad_srat();
Andi Kleend22fe802006-02-03 21:51:26 +0100121 return;
122 }
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300123 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124 return;
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300125 pxm = pa->proximity_domain_lo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 node = setup_node(pxm);
127 if (node < 0) {
128 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
129 bad_srat();
130 return;
131 }
Yinghai Lubeafe912008-02-16 23:00:22 -0800132
Jack Steiner2e420602008-09-23 15:37:13 -0500133 if (get_uv_system_type() >= UV_X2APIC)
Jack Steinera65d1d62008-03-28 14:12:08 -0500134 apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
135 else
136 apic_id = pa->apic_id;
Yinghai Lud3bd0582010-12-16 19:09:58 -0800137
138 if (apic_id >= MAX_LOCAL_APIC) {
139 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
140 return;
141 }
142
Tejun Heobbc9e2f2011-01-23 14:37:39 +0100143 set_apicid_to_node(apic_id, node);
Tejun Heo92d4a432011-02-16 17:11:09 +0100144 node_set(node, numa_nodes_parsed);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145 acpi_numa = 1;
Yinghai Lu163d3862009-11-21 00:23:37 -0800146 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
travis@sgi.comef970012008-01-30 13:33:10 +0100147 pxm, apic_id, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148}
149
Keith Mannthey71efa8f2006-09-30 23:27:05 -0700150#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
151static inline int save_add_info(void) {return 1;}
152#else
153static inline int save_add_info(void) {return 0;}
154#endif
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200155/*
Yinghai Lu888a5892009-05-15 13:59:37 -0700156 * Update nodes_add[]
157 * This code supports one contiguous hot add area per node
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200158 */
Yinghai Lu888a5892009-05-15 13:59:37 -0700159static void __init
160update_nodes_add(int node, unsigned long start, unsigned long end)
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200161{
162 unsigned long s_pfn = start >> PAGE_SHIFT;
163 unsigned long e_pfn = end >> PAGE_SHIFT;
Yinghai Lu888a5892009-05-15 13:59:37 -0700164 int changed = 0;
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200165 struct bootnode *nd = &nodes_add[node];
166
167 /* I had some trouble with strange memory hotadd regions breaking
168 the boot. Be very strict here and reject anything unexpected.
169 If you want working memory hotadd write correct SRATs.
170
171 The node size check is a basic sanity check to guard against
172 mistakes */
173 if ((signed long)(end - start) < NODE_MIN_SIZE) {
174 printk(KERN_ERR "SRAT: Hotplug area too small\n");
Yinghai Lu888a5892009-05-15 13:59:37 -0700175 return;
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200176 }
177
178 /* This check might be a bit too strict, but I'm keeping it for now. */
Mel Gorman5cb248a2006-09-27 01:49:52 -0700179 if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
Mel Gorman9c7cd682006-09-27 01:49:58 -0700180 printk(KERN_ERR
181 "SRAT: Hotplug area %lu -> %lu has existing memory\n",
182 s_pfn, e_pfn);
Yinghai Lu888a5892009-05-15 13:59:37 -0700183 return;
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200184 }
185
186 /* Looks good */
187
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200188 if (nd->start == nd->end) {
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300189 nd->start = start;
190 nd->end = end;
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200191 changed = 1;
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300192 } else {
193 if (nd->start == end) {
194 nd->start = start;
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200195 changed = 1;
196 }
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300197 if (nd->end == start) {
198 nd->end = end;
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200199 changed = 1;
200 }
201 if (!changed)
202 printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300203 }
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200204
David Rientjes3a5fc0e2010-01-20 12:10:47 -0800205 if (changed) {
Tejun Heo92d4a432011-02-16 17:11:09 +0100206 node_set(node, numa_nodes_parsed);
Yinghai Lu888a5892009-05-15 13:59:37 -0700207 printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
208 nd->start, nd->end);
David Rientjes3a5fc0e2010-01-20 12:10:47 -0800209 }
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200210}
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200211
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
213void __init
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300214acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 unsigned long start, end;
217 int node, pxm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
Andi Kleend22fe802006-02-03 21:51:26 +0100219 if (srat_disabled())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 return;
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300221 if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
Andi Kleend22fe802006-02-03 21:51:26 +0100222 bad_srat();
223 return;
224 }
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300225 if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
Andi Kleend22fe802006-02-03 21:51:26 +0100226 return;
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300227
228 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200229 return;
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300230 start = ma->base_address;
231 end = start + ma->length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 pxm = ma->proximity_domain;
233 node = setup_node(pxm);
234 if (node < 0) {
235 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
236 bad_srat();
237 return;
238 }
Tejun Heoef396ec2011-02-16 17:11:07 +0100239
240 if (numa_add_memblk(node, start, end) < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 bad_srat();
242 return;
243 }
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200244
Suresh Siddha6ec6e0d2008-03-25 10:14:35 -0700245 printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
246 start, end);
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200247
Tejun Heo4697bdc2011-02-16 17:11:09 +0100248 if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE)
Yinghai Lu888a5892009-05-15 13:59:37 -0700249 update_nodes_add(node, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250}
251
252void __init acpi_numa_arch_fixup(void) {}
253
Tejun Heoa9aec562011-02-16 12:13:06 +0100254int __init x86_acpi_numa_init(void)
255{
256 int ret;
257
258 ret = acpi_numa_init();
259 if (ret < 0)
260 return ret;
261 return srat_disabled() ? -EINVAL : 0;
262}
263
David Rientjes3484d792007-07-21 17:10:32 +0200264#ifdef CONFIG_NUMA_EMU
travis@sgi.comef970012008-01-30 13:33:10 +0100265static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
266 [0 ... MAX_NUMNODES-1] = PXM_INVAL
267};
travis@sgi.com602a54a2008-01-30 13:33:21 +0100268static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
travis@sgi.comef970012008-01-30 13:33:10 +0100269 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
270};
David Rientjes3484d792007-07-21 17:10:32 +0200271
272/*
273 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
274 * mappings that respect the real ACPI topology but reflect our emulated
275 * environment. For each emulated node, we find which real node it appears on
276 * and create PXM to NID mappings for those fake nodes which mirror that
277 * locality. SLIT will now represent the correct distances between emulated
278 * nodes as a result of the real topology.
279 */
280void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
281{
David Rientjes08705b82007-07-21 17:10:33 +0200282 int i, j;
David Rientjes3484d792007-07-21 17:10:32 +0200283
David Rientjes3484d792007-07-21 17:10:32 +0200284 for (i = 0; i < num_nodes; i++) {
285 int nid, pxm;
286
287 nid = find_node_by_addr(fake_nodes[i].start);
288 if (nid == NUMA_NO_NODE)
289 continue;
290 pxm = node_to_pxm(nid);
291 if (pxm == PXM_INVAL)
292 continue;
293 fake_node_to_pxm_map[i] = pxm;
David Rientjes08705b82007-07-21 17:10:33 +0200294 /*
295 * For each apicid_to_node mapping that exists for this real
296 * node, it must now point to the fake node ID.
297 */
298 for (j = 0; j < MAX_LOCAL_APIC; j++)
Tejun Heobbc9e2f2011-01-23 14:37:39 +0100299 if (__apicid_to_node[j] == nid &&
David Rientjesb0c4d952010-05-06 02:24:34 -0700300 fake_apicid_to_node[j] == NUMA_NO_NODE)
David Rientjes08705b82007-07-21 17:10:33 +0200301 fake_apicid_to_node[j] = i;
David Rientjes3484d792007-07-21 17:10:32 +0200302 }
David Rientjesa387e952010-12-22 17:23:56 -0800303
304 /*
305 * If there are apicid-to-node mappings for physical nodes that do not
306 * have a corresponding emulated node, it should default to a guaranteed
307 * value.
308 */
309 for (i = 0; i < MAX_LOCAL_APIC; i++)
Tejun Heobbc9e2f2011-01-23 14:37:39 +0100310 if (__apicid_to_node[i] != NUMA_NO_NODE &&
David Rientjesa387e952010-12-22 17:23:56 -0800311 fake_apicid_to_node[i] == NUMA_NO_NODE)
312 fake_apicid_to_node[i] = 0;
313
David Rientjes3484d792007-07-21 17:10:32 +0200314 for (i = 0; i < num_nodes; i++)
315 __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
Tejun Heobbc9e2f2011-01-23 14:37:39 +0100316 memcpy(__apicid_to_node, fake_apicid_to_node, sizeof(__apicid_to_node));
David Rientjes3484d792007-07-21 17:10:32 +0200317
David Rientjes3484d792007-07-21 17:10:32 +0200318 for (i = 0; i < num_nodes; i++)
319 if (fake_nodes[i].start != fake_nodes[i].end)
Tejun Heo4697bdc2011-02-16 17:11:09 +0100320 node_set(i, numa_nodes_parsed);
David Rientjes3484d792007-07-21 17:10:32 +0200321}
322
Tejun Heoac7136b2011-02-16 17:11:09 +0100323int acpi_emu_node_distance(int a, int b)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324{
325 int index;
326
327 if (!acpi_slit)
Tejun Heoac7136b2011-02-16 17:11:09 +0100328 return node_to_pxm(a) == node_to_pxm(b) ?
329 LOCAL_DISTANCE : REMOTE_DISTANCE;
Alexey Starikovskiy15a58ed2007-02-02 19:48:22 +0300330 index = acpi_slit->locality_count * node_to_pxm(a);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 return acpi_slit->entry[index + node_to_pxm(b)];
332}
Tejun Heoac7136b2011-02-16 17:11:09 +0100333#endif /* CONFIG_NUMA_EMU */
Keith Mannthey4942e992006-09-30 23:27:06 -0700334
Thomas Gleixner6a1673a2008-05-12 15:43:38 +0200335#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
Keith Mannthey4942e992006-09-30 23:27:06 -0700336int memory_add_physaddr_to_nid(u64 start)
337{
338 int i, ret = 0;
339
340 for_each_node(i)
341 if (nodes_add[i].start <= start && nodes_add[i].end > start)
342 ret = i;
343
344 return ret;
345}
Keith Mannthey8c2676a2006-09-30 23:27:07 -0700346EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
Thomas Gleixner6a1673a2008-05-12 15:43:38 +0200347#endif