Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * ACPI 3.0 based NUMA setup |
| 3 | * Copyright 2004 Andi Kleen, SuSE Labs. |
| 4 | * |
| 5 | * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs. |
| 6 | * |
| 7 | * Called from acpi_numa_init while reading the SRAT and SLIT tables. |
| 8 | * Assumes all memory regions belonging to a single proximity domain |
| 9 | * are in one chunk. Holes between them will be included in the node. |
| 10 | */ |
| 11 | |
| 12 | #include <linux/kernel.h> |
| 13 | #include <linux/acpi.h> |
| 14 | #include <linux/mmzone.h> |
| 15 | #include <linux/bitmap.h> |
| 16 | #include <linux/module.h> |
| 17 | #include <linux/topology.h> |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 18 | #include <linux/bootmem.h> |
| 19 | #include <linux/mm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <asm/proto.h> |
| 21 | #include <asm/numa.h> |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 22 | #include <asm/e820.h> |
Ingo Molnar | 7b6aa33 | 2009-02-17 13:58:15 +0100 | [diff] [blame] | 23 | #include <asm/apic.h> |
Ingo Molnar | 4ec71fa | 2009-01-21 10:24:27 +0100 | [diff] [blame] | 24 | #include <asm/uv/uv.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
Andi Kleen | c31fbb1 | 2006-09-26 10:52:33 +0200 | [diff] [blame] | 26 | int acpi_numa __initdata; |
| 27 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | static struct acpi_table_slit *acpi_slit; |
| 29 | |
| 30 | static nodemask_t nodes_parsed __initdata; |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 31 | static nodemask_t cpu_nodes_parsed __initdata; |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 32 | static struct bootnode nodes[MAX_NUMNODES] __initdata; |
Keith Mannthey | 4942e99 | 2006-09-30 23:27:06 -0700 | [diff] [blame] | 33 | static struct bootnode nodes_add[MAX_NUMNODES]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 35 | static int num_node_memblks __initdata; |
| 36 | static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata; |
| 37 | static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata; |
| 38 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | static __init int setup_node(int pxm) |
| 40 | { |
Yasunori Goto | 762834e | 2006-06-23 02:03:19 -0700 | [diff] [blame] | 41 | return acpi_map_pxm_to_node(pxm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | } |
| 43 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 44 | static __init int conflicting_memblks(unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | { |
| 46 | int i; |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 47 | for (i = 0; i < num_node_memblks; i++) { |
| 48 | struct bootnode *nd = &node_memblk_range[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | if (nd->start == nd->end) |
| 50 | continue; |
| 51 | if (nd->end > start && nd->start < end) |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 52 | return memblk_nodeid[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | if (nd->end == end && nd->start == start) |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 54 | return memblk_nodeid[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | } |
| 56 | return -1; |
| 57 | } |
| 58 | |
| 59 | static __init void cutoff_node(int i, unsigned long start, unsigned long end) |
| 60 | { |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 61 | struct bootnode *nd = &nodes[i]; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 62 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | if (nd->start < start) { |
| 64 | nd->start = start; |
| 65 | if (nd->end < nd->start) |
| 66 | nd->start = nd->end; |
| 67 | } |
| 68 | if (nd->end > end) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | nd->end = end; |
| 70 | if (nd->start > nd->end) |
| 71 | nd->start = nd->end; |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | static __init void bad_srat(void) |
| 76 | { |
Andi Kleen | 2bce2b5 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 77 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | printk(KERN_ERR "SRAT: SRAT not used.\n"); |
| 79 | acpi_numa = -1; |
Andi Kleen | 2bce2b5 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 80 | for (i = 0; i < MAX_LOCAL_APIC; i++) |
| 81 | apicid_to_node[i] = NUMA_NO_NODE; |
Andi Kleen | 429b2b3 | 2009-07-18 08:56:57 +0200 | [diff] [blame] | 82 | for (i = 0; i < MAX_NUMNODES; i++) { |
| 83 | nodes[i].start = nodes[i].end = 0; |
| 84 | nodes_add[i].start = nodes_add[i].end = 0; |
| 85 | } |
Mel Gorman | 5cb248a | 2006-09-27 01:49:52 -0700 | [diff] [blame] | 86 | remove_all_active_ranges(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 87 | } |
| 88 | |
| 89 | static __init inline int srat_disabled(void) |
| 90 | { |
| 91 | return numa_off || acpi_numa < 0; |
| 92 | } |
| 93 | |
| 94 | /* Callback for SLIT parsing */ |
| 95 | void __init acpi_numa_slit_init(struct acpi_table_slit *slit) |
| 96 | { |
Yinghai Lu | f302a5b | 2008-07-10 20:36:37 -0700 | [diff] [blame] | 97 | unsigned length; |
| 98 | unsigned long phys; |
| 99 | |
| 100 | length = slit->header.length; |
| 101 | phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length, |
| 102 | PAGE_SIZE); |
| 103 | |
| 104 | if (phys == -1L) |
| 105 | panic(" Can not save slit!\n"); |
| 106 | |
| 107 | acpi_slit = __va(phys); |
| 108 | memcpy(acpi_slit, slit, length); |
| 109 | reserve_early(phys, phys + length, "ACPI SLIT"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | } |
| 111 | |
Suresh Siddha | 7237d3d | 2009-03-30 13:55:30 -0800 | [diff] [blame] | 112 | /* Callback for Proximity Domain -> x2APIC mapping */ |
| 113 | void __init |
| 114 | acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) |
| 115 | { |
| 116 | int pxm, node; |
| 117 | int apic_id; |
| 118 | |
| 119 | if (srat_disabled()) |
| 120 | return; |
| 121 | if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) { |
| 122 | bad_srat(); |
| 123 | return; |
| 124 | } |
| 125 | if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) |
| 126 | return; |
| 127 | pxm = pa->proximity_domain; |
| 128 | node = setup_node(pxm); |
| 129 | if (node < 0) { |
| 130 | printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); |
| 131 | bad_srat(); |
| 132 | return; |
| 133 | } |
| 134 | |
| 135 | apic_id = pa->apic_id; |
| 136 | apicid_to_node[apic_id] = node; |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 137 | node_set(node, cpu_nodes_parsed); |
Suresh Siddha | 7237d3d | 2009-03-30 13:55:30 -0800 | [diff] [blame] | 138 | acpi_numa = 1; |
| 139 | printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n", |
| 140 | pxm, apic_id, node); |
| 141 | } |
| 142 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | /* Callback for Proximity Domain -> LAPIC mapping */ |
| 144 | void __init |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 145 | acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | { |
| 147 | int pxm, node; |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 148 | int apic_id; |
| 149 | |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 150 | if (srat_disabled()) |
| 151 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 152 | if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) { |
Andi Kleen | fad7906 | 2006-05-15 18:19:44 +0200 | [diff] [blame] | 153 | bad_srat(); |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 154 | return; |
| 155 | } |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 156 | if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 157 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 158 | pxm = pa->proximity_domain_lo; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 159 | node = setup_node(pxm); |
| 160 | if (node < 0) { |
| 161 | printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); |
| 162 | bad_srat(); |
| 163 | return; |
| 164 | } |
Yinghai Lu | beafe91 | 2008-02-16 23:00:22 -0800 | [diff] [blame] | 165 | |
Jack Steiner | 2e42060 | 2008-09-23 15:37:13 -0500 | [diff] [blame] | 166 | if (get_uv_system_type() >= UV_X2APIC) |
Jack Steiner | a65d1d6 | 2008-03-28 14:12:08 -0500 | [diff] [blame] | 167 | apic_id = (pa->apic_id << 8) | pa->local_sapic_eid; |
| 168 | else |
| 169 | apic_id = pa->apic_id; |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 170 | apicid_to_node[apic_id] = node; |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 171 | node_set(node, cpu_nodes_parsed); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 172 | acpi_numa = 1; |
Andi Kleen | 0b07e98 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 173 | printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n", |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 174 | pxm, apic_id, node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | } |
| 176 | |
Keith Mannthey | 71efa8f | 2006-09-30 23:27:05 -0700 | [diff] [blame] | 177 | #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE |
| 178 | static inline int save_add_info(void) {return 1;} |
| 179 | #else |
| 180 | static inline int save_add_info(void) {return 0;} |
| 181 | #endif |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 182 | /* |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 183 | * Update nodes_add[] |
| 184 | * This code supports one contiguous hot add area per node |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 185 | */ |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 186 | static void __init |
| 187 | update_nodes_add(int node, unsigned long start, unsigned long end) |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 188 | { |
| 189 | unsigned long s_pfn = start >> PAGE_SHIFT; |
| 190 | unsigned long e_pfn = end >> PAGE_SHIFT; |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 191 | int changed = 0; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 192 | struct bootnode *nd = &nodes_add[node]; |
| 193 | |
| 194 | /* I had some trouble with strange memory hotadd regions breaking |
| 195 | the boot. Be very strict here and reject anything unexpected. |
| 196 | If you want working memory hotadd write correct SRATs. |
| 197 | |
| 198 | The node size check is a basic sanity check to guard against |
| 199 | mistakes */ |
| 200 | if ((signed long)(end - start) < NODE_MIN_SIZE) { |
| 201 | printk(KERN_ERR "SRAT: Hotplug area too small\n"); |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 202 | return; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 203 | } |
| 204 | |
| 205 | /* This check might be a bit too strict, but I'm keeping it for now. */ |
Mel Gorman | 5cb248a | 2006-09-27 01:49:52 -0700 | [diff] [blame] | 206 | if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) { |
Mel Gorman | 9c7cd68 | 2006-09-27 01:49:58 -0700 | [diff] [blame] | 207 | printk(KERN_ERR |
| 208 | "SRAT: Hotplug area %lu -> %lu has existing memory\n", |
| 209 | s_pfn, e_pfn); |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 210 | return; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 211 | } |
| 212 | |
| 213 | /* Looks good */ |
| 214 | |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 215 | if (nd->start == nd->end) { |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 216 | nd->start = start; |
| 217 | nd->end = end; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 218 | changed = 1; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 219 | } else { |
| 220 | if (nd->start == end) { |
| 221 | nd->start = start; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 222 | changed = 1; |
| 223 | } |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 224 | if (nd->end == start) { |
| 225 | nd->end = end; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 226 | changed = 1; |
| 227 | } |
| 228 | if (!changed) |
| 229 | printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n"); |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 230 | } |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 231 | |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 232 | if (changed) |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 233 | printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n", |
| 234 | nd->start, nd->end); |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 235 | } |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 236 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 237 | /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */ |
| 238 | void __init |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 239 | acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 240 | { |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 241 | struct bootnode *nd, oldnode; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 242 | unsigned long start, end; |
| 243 | int node, pxm; |
| 244 | int i; |
| 245 | |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 246 | if (srat_disabled()) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 247 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 248 | if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) { |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 249 | bad_srat(); |
| 250 | return; |
| 251 | } |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 252 | if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0) |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 253 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 254 | |
| 255 | if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info()) |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 256 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 257 | start = ma->base_address; |
| 258 | end = start + ma->length; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 259 | pxm = ma->proximity_domain; |
| 260 | node = setup_node(pxm); |
| 261 | if (node < 0) { |
| 262 | printk(KERN_ERR "SRAT: Too many proximity domains.\n"); |
| 263 | bad_srat(); |
| 264 | return; |
| 265 | } |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 266 | i = conflicting_memblks(start, end); |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 267 | if (i == node) { |
| 268 | printk(KERN_WARNING |
| 269 | "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n", |
| 270 | pxm, start, end, nodes[i].start, nodes[i].end); |
| 271 | } else if (i >= 0) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 272 | printk(KERN_ERR |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 273 | "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n", |
| 274 | pxm, start, end, node_to_pxm(i), |
| 275 | nodes[i].start, nodes[i].end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 276 | bad_srat(); |
| 277 | return; |
| 278 | } |
| 279 | nd = &nodes[node]; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 280 | oldnode = *nd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 281 | if (!node_test_and_set(node, nodes_parsed)) { |
| 282 | nd->start = start; |
| 283 | nd->end = end; |
| 284 | } else { |
| 285 | if (start < nd->start) |
| 286 | nd->start = start; |
| 287 | if (nd->end < end) |
| 288 | nd->end = end; |
| 289 | } |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 290 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 291 | printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm, |
| 292 | start, end); |
| 293 | e820_register_active_regions(node, start >> PAGE_SHIFT, |
| 294 | end >> PAGE_SHIFT); |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 295 | |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 296 | if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) { |
| 297 | update_nodes_add(node, start, end); |
| 298 | /* restore nodes[node] */ |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 299 | *nd = oldnode; |
| 300 | if ((nd->start | nd->end) == 0) |
| 301 | node_clear(node, nodes_parsed); |
| 302 | } |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 303 | |
| 304 | node_memblk_range[num_node_memblks].start = start; |
| 305 | node_memblk_range[num_node_memblks].end = end; |
| 306 | memblk_nodeid[num_node_memblks] = node; |
| 307 | num_node_memblks++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 308 | } |
| 309 | |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 310 | /* Sanity check to catch more bad SRATs (they are amazingly common). |
| 311 | Make sure the PXMs cover all memory. */ |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 312 | static int __init nodes_cover_memory(const struct bootnode *nodes) |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 313 | { |
| 314 | int i; |
| 315 | unsigned long pxmram, e820ram; |
| 316 | |
| 317 | pxmram = 0; |
| 318 | for_each_node_mask(i, nodes_parsed) { |
| 319 | unsigned long s = nodes[i].start >> PAGE_SHIFT; |
| 320 | unsigned long e = nodes[i].end >> PAGE_SHIFT; |
| 321 | pxmram += e - s; |
Mel Gorman | 5cb248a | 2006-09-27 01:49:52 -0700 | [diff] [blame] | 322 | pxmram -= absent_pages_in_range(s, e); |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 323 | if ((long)pxmram < 0) |
| 324 | pxmram = 0; |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 325 | } |
| 326 | |
Yinghai Lu | b37ab91 | 2009-05-08 00:36:44 -0700 | [diff] [blame] | 327 | e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT); |
Yinghai Lu | 0964b05 | 2009-05-08 00:37:34 -0700 | [diff] [blame] | 328 | /* We seem to lose 3 pages somewhere. Allow 1M of slack. */ |
| 329 | if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) { |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 330 | printk(KERN_ERR |
| 331 | "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n", |
| 332 | (pxmram << PAGE_SHIFT) >> 20, |
| 333 | (e820ram << PAGE_SHIFT) >> 20); |
| 334 | return 0; |
| 335 | } |
| 336 | return 1; |
| 337 | } |
| 338 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 339 | void __init acpi_numa_arch_fixup(void) {} |
| 340 | |
| 341 | /* Use the information discovered above to actually set up the nodes. */ |
| 342 | int __init acpi_scan_nodes(unsigned long start, unsigned long end) |
| 343 | { |
| 344 | int i; |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 345 | |
David Rientjes | ae2c6dc | 2007-07-21 17:09:56 +0200 | [diff] [blame] | 346 | if (acpi_numa <= 0) |
| 347 | return -1; |
| 348 | |
Andi Kleen | 9391a3f | 2006-02-03 21:51:17 +0100 | [diff] [blame] | 349 | /* First clean up the node list */ |
Yinghai Lu | 7c43769 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 350 | for (i = 0; i < MAX_NUMNODES; i++) |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 351 | cutoff_node(i, start, end); |
Andi Kleen | 9391a3f | 2006-02-03 21:51:17 +0100 | [diff] [blame] | 352 | |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 353 | if (!nodes_cover_memory(nodes)) { |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 354 | bad_srat(); |
| 355 | return -1; |
| 356 | } |
| 357 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 358 | memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks, |
| 359 | memblk_nodeid); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 360 | if (memnode_shift < 0) { |
| 361 | printk(KERN_ERR |
| 362 | "SRAT: No NUMA node hash function found. Contact maintainer\n"); |
| 363 | bad_srat(); |
| 364 | return -1; |
| 365 | } |
Andi Kleen | e58e0d0 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 366 | |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 367 | /* Account for nodes with cpus and no memory */ |
| 368 | nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed); |
Suresh Siddha | e3f1cae | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 369 | |
Andi Kleen | e58e0d0 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 370 | /* Finally register nodes */ |
Suresh Siddha | e3f1cae | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 371 | for_each_node_mask(i, node_possible_map) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 372 | setup_node_bootmem(i, nodes[i].start, nodes[i].end); |
Andi Kleen | a806223 | 2006-04-07 19:49:21 +0200 | [diff] [blame] | 373 | /* Try again in case setup_node_bootmem missed one due |
| 374 | to missing bootmem */ |
Suresh Siddha | e3f1cae | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 375 | for_each_node_mask(i, node_possible_map) |
Andi Kleen | a806223 | 2006-04-07 19:49:21 +0200 | [diff] [blame] | 376 | if (!node_online(i)) |
| 377 | setup_node_bootmem(i, nodes[i].start, nodes[i].end); |
| 378 | |
Mike Travis | 168ef54 | 2008-12-16 17:34:01 -0800 | [diff] [blame] | 379 | for (i = 0; i < nr_cpu_ids; i++) { |
Mike Travis | 0164fe1 | 2008-01-30 13:33:21 +0100 | [diff] [blame] | 380 | int node = early_cpu_to_node(i); |
| 381 | |
travis@sgi.com | 834beda | 2008-01-30 13:33:21 +0100 | [diff] [blame] | 382 | if (node == NUMA_NO_NODE) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 383 | continue; |
Yinghai Lu | 7c43769 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 384 | if (!node_online(node)) |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 385 | numa_clear_node(i); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 386 | } |
| 387 | numa_init_array(); |
| 388 | return 0; |
| 389 | } |
| 390 | |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 391 | #ifdef CONFIG_NUMA_EMU |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 392 | static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = { |
| 393 | [0 ... MAX_NUMNODES-1] = PXM_INVAL |
| 394 | }; |
travis@sgi.com | 602a54a | 2008-01-30 13:33:21 +0100 | [diff] [blame] | 395 | static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = { |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 396 | [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE |
| 397 | }; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 398 | static int __init find_node_by_addr(unsigned long addr) |
| 399 | { |
| 400 | int ret = NUMA_NO_NODE; |
| 401 | int i; |
| 402 | |
| 403 | for_each_node_mask(i, nodes_parsed) { |
| 404 | /* |
| 405 | * Find the real node that this emulated node appears on. For |
| 406 | * the sake of simplicity, we only use a real node's starting |
| 407 | * address to determine which emulated node it appears on. |
| 408 | */ |
| 409 | if (addr >= nodes[i].start && addr < nodes[i].end) { |
| 410 | ret = i; |
| 411 | break; |
| 412 | } |
| 413 | } |
Minoru Usui | 9a1b62f | 2008-01-30 13:33:35 +0100 | [diff] [blame] | 414 | return ret; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 415 | } |
| 416 | |
| 417 | /* |
| 418 | * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID |
| 419 | * mappings that respect the real ACPI topology but reflect our emulated |
| 420 | * environment. For each emulated node, we find which real node it appears on |
| 421 | * and create PXM to NID mappings for those fake nodes which mirror that |
| 422 | * locality. SLIT will now represent the correct distances between emulated |
| 423 | * nodes as a result of the real topology. |
| 424 | */ |
| 425 | void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes) |
| 426 | { |
David Rientjes | 08705b8 | 2007-07-21 17:10:33 +0200 | [diff] [blame] | 427 | int i, j; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 428 | |
| 429 | printk(KERN_INFO "Faking PXM affinity for fake nodes on real " |
| 430 | "topology.\n"); |
| 431 | for (i = 0; i < num_nodes; i++) { |
| 432 | int nid, pxm; |
| 433 | |
| 434 | nid = find_node_by_addr(fake_nodes[i].start); |
| 435 | if (nid == NUMA_NO_NODE) |
| 436 | continue; |
| 437 | pxm = node_to_pxm(nid); |
| 438 | if (pxm == PXM_INVAL) |
| 439 | continue; |
| 440 | fake_node_to_pxm_map[i] = pxm; |
David Rientjes | 08705b8 | 2007-07-21 17:10:33 +0200 | [diff] [blame] | 441 | /* |
| 442 | * For each apicid_to_node mapping that exists for this real |
| 443 | * node, it must now point to the fake node ID. |
| 444 | */ |
| 445 | for (j = 0; j < MAX_LOCAL_APIC; j++) |
| 446 | if (apicid_to_node[j] == nid) |
| 447 | fake_apicid_to_node[j] = i; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 448 | } |
| 449 | for (i = 0; i < num_nodes; i++) |
| 450 | __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i); |
David Rientjes | 08705b8 | 2007-07-21 17:10:33 +0200 | [diff] [blame] | 451 | memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node)); |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 452 | |
| 453 | nodes_clear(nodes_parsed); |
| 454 | for (i = 0; i < num_nodes; i++) |
| 455 | if (fake_nodes[i].start != fake_nodes[i].end) |
| 456 | node_set(i, nodes_parsed); |
| 457 | WARN_ON(!nodes_cover_memory(fake_nodes)); |
| 458 | } |
| 459 | |
| 460 | static int null_slit_node_compare(int a, int b) |
| 461 | { |
| 462 | return node_to_pxm(a) == node_to_pxm(b); |
| 463 | } |
| 464 | #else |
| 465 | static int null_slit_node_compare(int a, int b) |
| 466 | { |
| 467 | return a == b; |
| 468 | } |
| 469 | #endif /* CONFIG_NUMA_EMU */ |
| 470 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | int __node_distance(int a, int b) |
| 472 | { |
| 473 | int index; |
| 474 | |
| 475 | if (!acpi_slit) |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 476 | return null_slit_node_compare(a, b) ? LOCAL_DISTANCE : |
| 477 | REMOTE_DISTANCE; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 478 | index = acpi_slit->locality_count * node_to_pxm(a); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 479 | return acpi_slit->entry[index + node_to_pxm(b)]; |
| 480 | } |
| 481 | |
| 482 | EXPORT_SYMBOL(__node_distance); |
Keith Mannthey | 4942e99 | 2006-09-30 23:27:06 -0700 | [diff] [blame] | 483 | |
Thomas Gleixner | 6a1673a | 2008-05-12 15:43:38 +0200 | [diff] [blame] | 484 | #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY) |
Keith Mannthey | 4942e99 | 2006-09-30 23:27:06 -0700 | [diff] [blame] | 485 | int memory_add_physaddr_to_nid(u64 start) |
| 486 | { |
| 487 | int i, ret = 0; |
| 488 | |
| 489 | for_each_node(i) |
| 490 | if (nodes_add[i].start <= start && nodes_add[i].end > start) |
| 491 | ret = i; |
| 492 | |
| 493 | return ret; |
| 494 | } |
Keith Mannthey | 8c2676a | 2006-09-30 23:27:07 -0700 | [diff] [blame] | 495 | EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); |
Thomas Gleixner | 6a1673a | 2008-05-12 15:43:38 +0200 | [diff] [blame] | 496 | #endif |