Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * ACPI 3.0 based NUMA setup |
| 3 | * Copyright 2004 Andi Kleen, SuSE Labs. |
| 4 | * |
| 5 | * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs. |
| 6 | * |
| 7 | * Called from acpi_numa_init while reading the SRAT and SLIT tables. |
| 8 | * Assumes all memory regions belonging to a single proximity domain |
| 9 | * are in one chunk. Holes between them will be included in the node. |
| 10 | */ |
| 11 | |
| 12 | #include <linux/kernel.h> |
| 13 | #include <linux/acpi.h> |
| 14 | #include <linux/mmzone.h> |
| 15 | #include <linux/bitmap.h> |
| 16 | #include <linux/module.h> |
| 17 | #include <linux/topology.h> |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 18 | #include <linux/bootmem.h> |
| 19 | #include <linux/mm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <asm/proto.h> |
| 21 | #include <asm/numa.h> |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 22 | #include <asm/e820.h> |
Ingo Molnar | 7b6aa33 | 2009-02-17 13:58:15 +0100 | [diff] [blame] | 23 | #include <asm/apic.h> |
Ingo Molnar | 4ec71fa | 2009-01-21 10:24:27 +0100 | [diff] [blame] | 24 | #include <asm/uv/uv.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
Andi Kleen | c31fbb1 | 2006-09-26 10:52:33 +0200 | [diff] [blame] | 26 | int acpi_numa __initdata; |
| 27 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | static struct acpi_table_slit *acpi_slit; |
| 29 | |
| 30 | static nodemask_t nodes_parsed __initdata; |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 31 | static nodemask_t cpu_nodes_parsed __initdata; |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 32 | static struct bootnode nodes[MAX_NUMNODES] __initdata; |
Keith Mannthey | 4942e99 | 2006-09-30 23:27:06 -0700 | [diff] [blame] | 33 | static struct bootnode nodes_add[MAX_NUMNODES]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 35 | static int num_node_memblks __initdata; |
| 36 | static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata; |
| 37 | static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata; |
| 38 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | static __init int setup_node(int pxm) |
| 40 | { |
Yasunori Goto | 762834e | 2006-06-23 02:03:19 -0700 | [diff] [blame] | 41 | return acpi_map_pxm_to_node(pxm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | } |
| 43 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 44 | static __init int conflicting_memblks(unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | { |
| 46 | int i; |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 47 | for (i = 0; i < num_node_memblks; i++) { |
| 48 | struct bootnode *nd = &node_memblk_range[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | if (nd->start == nd->end) |
| 50 | continue; |
| 51 | if (nd->end > start && nd->start < end) |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 52 | return memblk_nodeid[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | if (nd->end == end && nd->start == start) |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 54 | return memblk_nodeid[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | } |
| 56 | return -1; |
| 57 | } |
| 58 | |
| 59 | static __init void cutoff_node(int i, unsigned long start, unsigned long end) |
| 60 | { |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 61 | struct bootnode *nd = &nodes[i]; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 62 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | if (nd->start < start) { |
| 64 | nd->start = start; |
| 65 | if (nd->end < nd->start) |
| 66 | nd->start = nd->end; |
| 67 | } |
| 68 | if (nd->end > end) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | nd->end = end; |
| 70 | if (nd->start > nd->end) |
| 71 | nd->start = nd->end; |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | static __init void bad_srat(void) |
| 76 | { |
Andi Kleen | 2bce2b5 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 77 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | printk(KERN_ERR "SRAT: SRAT not used.\n"); |
| 79 | acpi_numa = -1; |
Andi Kleen | 2bce2b5 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 80 | for (i = 0; i < MAX_LOCAL_APIC; i++) |
| 81 | apicid_to_node[i] = NUMA_NO_NODE; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 82 | for (i = 0; i < MAX_NUMNODES; i++) |
| 83 | nodes_add[i].start = nodes[i].end = 0; |
Mel Gorman | 5cb248a | 2006-09-27 01:49:52 -0700 | [diff] [blame] | 84 | remove_all_active_ranges(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 85 | } |
| 86 | |
| 87 | static __init inline int srat_disabled(void) |
| 88 | { |
| 89 | return numa_off || acpi_numa < 0; |
| 90 | } |
| 91 | |
| 92 | /* Callback for SLIT parsing */ |
| 93 | void __init acpi_numa_slit_init(struct acpi_table_slit *slit) |
| 94 | { |
Yinghai Lu | f302a5b | 2008-07-10 20:36:37 -0700 | [diff] [blame] | 95 | unsigned length; |
| 96 | unsigned long phys; |
| 97 | |
| 98 | length = slit->header.length; |
| 99 | phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length, |
| 100 | PAGE_SIZE); |
| 101 | |
| 102 | if (phys == -1L) |
| 103 | panic(" Can not save slit!\n"); |
| 104 | |
| 105 | acpi_slit = __va(phys); |
| 106 | memcpy(acpi_slit, slit, length); |
| 107 | reserve_early(phys, phys + length, "ACPI SLIT"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | } |
| 109 | |
Suresh Siddha | 7237d3d | 2009-03-30 13:55:30 -0800 | [diff] [blame] | 110 | /* Callback for Proximity Domain -> x2APIC mapping */ |
| 111 | void __init |
| 112 | acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) |
| 113 | { |
| 114 | int pxm, node; |
| 115 | int apic_id; |
| 116 | |
| 117 | if (srat_disabled()) |
| 118 | return; |
| 119 | if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) { |
| 120 | bad_srat(); |
| 121 | return; |
| 122 | } |
| 123 | if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) |
| 124 | return; |
| 125 | pxm = pa->proximity_domain; |
| 126 | node = setup_node(pxm); |
| 127 | if (node < 0) { |
| 128 | printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); |
| 129 | bad_srat(); |
| 130 | return; |
| 131 | } |
| 132 | |
| 133 | apic_id = pa->apic_id; |
| 134 | apicid_to_node[apic_id] = node; |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 135 | node_set(node, cpu_nodes_parsed); |
Suresh Siddha | 7237d3d | 2009-03-30 13:55:30 -0800 | [diff] [blame] | 136 | acpi_numa = 1; |
| 137 | printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n", |
| 138 | pxm, apic_id, node); |
| 139 | } |
| 140 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | /* Callback for Proximity Domain -> LAPIC mapping */ |
| 142 | void __init |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 143 | acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 144 | { |
| 145 | int pxm, node; |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 146 | int apic_id; |
| 147 | |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 148 | if (srat_disabled()) |
| 149 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 150 | if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) { |
Andi Kleen | fad7906 | 2006-05-15 18:19:44 +0200 | [diff] [blame] | 151 | bad_srat(); |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 152 | return; |
| 153 | } |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 154 | if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 156 | pxm = pa->proximity_domain_lo; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 157 | node = setup_node(pxm); |
| 158 | if (node < 0) { |
| 159 | printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); |
| 160 | bad_srat(); |
| 161 | return; |
| 162 | } |
Yinghai Lu | beafe91 | 2008-02-16 23:00:22 -0800 | [diff] [blame] | 163 | |
Jack Steiner | 2e42060 | 2008-09-23 15:37:13 -0500 | [diff] [blame] | 164 | if (get_uv_system_type() >= UV_X2APIC) |
Jack Steiner | a65d1d6 | 2008-03-28 14:12:08 -0500 | [diff] [blame] | 165 | apic_id = (pa->apic_id << 8) | pa->local_sapic_eid; |
| 166 | else |
| 167 | apic_id = pa->apic_id; |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 168 | apicid_to_node[apic_id] = node; |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 169 | node_set(node, cpu_nodes_parsed); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | acpi_numa = 1; |
Andi Kleen | 0b07e98 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 171 | printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n", |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 172 | pxm, apic_id, node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | } |
| 174 | |
Keith Mannthey | 71efa8f | 2006-09-30 23:27:05 -0700 | [diff] [blame] | 175 | #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE |
| 176 | static inline int save_add_info(void) {return 1;} |
| 177 | #else |
| 178 | static inline int save_add_info(void) {return 0;} |
| 179 | #endif |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 180 | /* |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 181 | * Update nodes_add[] |
| 182 | * This code supports one contiguous hot add area per node |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 183 | */ |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 184 | static void __init |
| 185 | update_nodes_add(int node, unsigned long start, unsigned long end) |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 186 | { |
| 187 | unsigned long s_pfn = start >> PAGE_SHIFT; |
| 188 | unsigned long e_pfn = end >> PAGE_SHIFT; |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 189 | int changed = 0; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 190 | struct bootnode *nd = &nodes_add[node]; |
| 191 | |
| 192 | /* I had some trouble with strange memory hotadd regions breaking |
| 193 | the boot. Be very strict here and reject anything unexpected. |
| 194 | If you want working memory hotadd write correct SRATs. |
| 195 | |
| 196 | The node size check is a basic sanity check to guard against |
| 197 | mistakes */ |
| 198 | if ((signed long)(end - start) < NODE_MIN_SIZE) { |
| 199 | printk(KERN_ERR "SRAT: Hotplug area too small\n"); |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 200 | return; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 201 | } |
| 202 | |
| 203 | /* This check might be a bit too strict, but I'm keeping it for now. */ |
Mel Gorman | 5cb248a | 2006-09-27 01:49:52 -0700 | [diff] [blame] | 204 | if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) { |
Mel Gorman | 9c7cd68 | 2006-09-27 01:49:58 -0700 | [diff] [blame] | 205 | printk(KERN_ERR |
| 206 | "SRAT: Hotplug area %lu -> %lu has existing memory\n", |
| 207 | s_pfn, e_pfn); |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 208 | return; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 209 | } |
| 210 | |
| 211 | /* Looks good */ |
| 212 | |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 213 | if (nd->start == nd->end) { |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 214 | nd->start = start; |
| 215 | nd->end = end; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 216 | changed = 1; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 217 | } else { |
| 218 | if (nd->start == end) { |
| 219 | nd->start = start; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 220 | changed = 1; |
| 221 | } |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 222 | if (nd->end == start) { |
| 223 | nd->end = end; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 224 | changed = 1; |
| 225 | } |
| 226 | if (!changed) |
| 227 | printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n"); |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 228 | } |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 229 | |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 230 | if (changed) |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 231 | printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n", |
| 232 | nd->start, nd->end); |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 233 | } |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 234 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 235 | /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */ |
| 236 | void __init |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 237 | acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 238 | { |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 239 | struct bootnode *nd, oldnode; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 240 | unsigned long start, end; |
| 241 | int node, pxm; |
| 242 | int i; |
| 243 | |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 244 | if (srat_disabled()) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 245 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 246 | if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) { |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 247 | bad_srat(); |
| 248 | return; |
| 249 | } |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 250 | if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0) |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 251 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 252 | |
| 253 | if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info()) |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 254 | return; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 255 | start = ma->base_address; |
| 256 | end = start + ma->length; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 257 | pxm = ma->proximity_domain; |
| 258 | node = setup_node(pxm); |
| 259 | if (node < 0) { |
| 260 | printk(KERN_ERR "SRAT: Too many proximity domains.\n"); |
| 261 | bad_srat(); |
| 262 | return; |
| 263 | } |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 264 | i = conflicting_memblks(start, end); |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 265 | if (i == node) { |
| 266 | printk(KERN_WARNING |
| 267 | "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n", |
| 268 | pxm, start, end, nodes[i].start, nodes[i].end); |
| 269 | } else if (i >= 0) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 270 | printk(KERN_ERR |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 271 | "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n", |
| 272 | pxm, start, end, node_to_pxm(i), |
| 273 | nodes[i].start, nodes[i].end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 274 | bad_srat(); |
| 275 | return; |
| 276 | } |
| 277 | nd = &nodes[node]; |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 278 | oldnode = *nd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 279 | if (!node_test_and_set(node, nodes_parsed)) { |
| 280 | nd->start = start; |
| 281 | nd->end = end; |
| 282 | } else { |
| 283 | if (start < nd->start) |
| 284 | nd->start = start; |
| 285 | if (nd->end < end) |
| 286 | nd->end = end; |
| 287 | } |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 288 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 289 | printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm, |
| 290 | start, end); |
| 291 | e820_register_active_regions(node, start >> PAGE_SHIFT, |
| 292 | end >> PAGE_SHIFT); |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 293 | |
Yinghai Lu | 888a589 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 294 | if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) { |
| 295 | update_nodes_add(node, start, end); |
| 296 | /* restore nodes[node] */ |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 297 | *nd = oldnode; |
| 298 | if ((nd->start | nd->end) == 0) |
| 299 | node_clear(node, nodes_parsed); |
| 300 | } |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 301 | |
| 302 | node_memblk_range[num_node_memblks].start = start; |
| 303 | node_memblk_range[num_node_memblks].end = end; |
| 304 | memblk_nodeid[num_node_memblks] = node; |
| 305 | num_node_memblks++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 306 | } |
| 307 | |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 308 | /* Sanity check to catch more bad SRATs (they are amazingly common). |
| 309 | Make sure the PXMs cover all memory. */ |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 310 | static int __init nodes_cover_memory(const struct bootnode *nodes) |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 311 | { |
| 312 | int i; |
| 313 | unsigned long pxmram, e820ram; |
| 314 | |
| 315 | pxmram = 0; |
| 316 | for_each_node_mask(i, nodes_parsed) { |
| 317 | unsigned long s = nodes[i].start >> PAGE_SHIFT; |
| 318 | unsigned long e = nodes[i].end >> PAGE_SHIFT; |
| 319 | pxmram += e - s; |
Mel Gorman | 5cb248a | 2006-09-27 01:49:52 -0700 | [diff] [blame] | 320 | pxmram -= absent_pages_in_range(s, e); |
Andi Kleen | 68a3a7f | 2006-04-07 19:49:18 +0200 | [diff] [blame] | 321 | if ((long)pxmram < 0) |
| 322 | pxmram = 0; |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 323 | } |
| 324 | |
Yinghai Lu | b37ab91 | 2009-05-08 00:36:44 -0700 | [diff] [blame] | 325 | e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT); |
Yinghai Lu | 0964b05 | 2009-05-08 00:37:34 -0700 | [diff] [blame] | 326 | /* We seem to lose 3 pages somewhere. Allow 1M of slack. */ |
| 327 | if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) { |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 328 | printk(KERN_ERR |
| 329 | "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n", |
| 330 | (pxmram << PAGE_SHIFT) >> 20, |
| 331 | (e820ram << PAGE_SHIFT) >> 20); |
| 332 | return 0; |
| 333 | } |
| 334 | return 1; |
| 335 | } |
| 336 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 337 | void __init acpi_numa_arch_fixup(void) {} |
| 338 | |
| 339 | /* Use the information discovered above to actually set up the nodes. */ |
| 340 | int __init acpi_scan_nodes(unsigned long start, unsigned long end) |
| 341 | { |
| 342 | int i; |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 343 | |
David Rientjes | ae2c6dc | 2007-07-21 17:09:56 +0200 | [diff] [blame] | 344 | if (acpi_numa <= 0) |
| 345 | return -1; |
| 346 | |
Andi Kleen | 9391a3f | 2006-02-03 21:51:17 +0100 | [diff] [blame] | 347 | /* First clean up the node list */ |
Yinghai Lu | 7c43769 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 348 | for (i = 0; i < MAX_NUMNODES; i++) |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 349 | cutoff_node(i, start, end); |
Andi Kleen | 9391a3f | 2006-02-03 21:51:17 +0100 | [diff] [blame] | 350 | |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 351 | if (!nodes_cover_memory(nodes)) { |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 352 | bad_srat(); |
| 353 | return -1; |
| 354 | } |
| 355 | |
Suresh Siddha | 6ec6e0d | 2008-03-25 10:14:35 -0700 | [diff] [blame] | 356 | memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks, |
| 357 | memblk_nodeid); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 358 | if (memnode_shift < 0) { |
| 359 | printk(KERN_ERR |
| 360 | "SRAT: No NUMA node hash function found. Contact maintainer\n"); |
| 361 | bad_srat(); |
| 362 | return -1; |
| 363 | } |
Andi Kleen | e58e0d0 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 364 | |
Jack Steiner | dc09855 | 2009-04-17 09:22:42 -0500 | [diff] [blame] | 365 | /* Account for nodes with cpus and no memory */ |
| 366 | nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed); |
Suresh Siddha | e3f1cae | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 367 | |
Andi Kleen | e58e0d0 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 368 | /* Finally register nodes */ |
Suresh Siddha | e3f1cae | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 369 | for_each_node_mask(i, node_possible_map) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | setup_node_bootmem(i, nodes[i].start, nodes[i].end); |
Andi Kleen | a806223 | 2006-04-07 19:49:21 +0200 | [diff] [blame] | 371 | /* Try again in case setup_node_bootmem missed one due |
| 372 | to missing bootmem */ |
Suresh Siddha | e3f1cae | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 373 | for_each_node_mask(i, node_possible_map) |
Andi Kleen | a806223 | 2006-04-07 19:49:21 +0200 | [diff] [blame] | 374 | if (!node_online(i)) |
| 375 | setup_node_bootmem(i, nodes[i].start, nodes[i].end); |
| 376 | |
Mike Travis | 168ef54 | 2008-12-16 17:34:01 -0800 | [diff] [blame] | 377 | for (i = 0; i < nr_cpu_ids; i++) { |
Mike Travis | 0164fe1 | 2008-01-30 13:33:21 +0100 | [diff] [blame] | 378 | int node = early_cpu_to_node(i); |
| 379 | |
travis@sgi.com | 834beda | 2008-01-30 13:33:21 +0100 | [diff] [blame] | 380 | if (node == NUMA_NO_NODE) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 381 | continue; |
Yinghai Lu | 7c43769 | 2009-05-15 13:59:37 -0700 | [diff] [blame] | 382 | if (!node_online(node)) |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 383 | numa_clear_node(i); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 384 | } |
| 385 | numa_init_array(); |
| 386 | return 0; |
| 387 | } |
| 388 | |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 389 | #ifdef CONFIG_NUMA_EMU |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 390 | static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = { |
| 391 | [0 ... MAX_NUMNODES-1] = PXM_INVAL |
| 392 | }; |
travis@sgi.com | 602a54a | 2008-01-30 13:33:21 +0100 | [diff] [blame] | 393 | static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = { |
travis@sgi.com | ef97001 | 2008-01-30 13:33:10 +0100 | [diff] [blame] | 394 | [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE |
| 395 | }; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 396 | static int __init find_node_by_addr(unsigned long addr) |
| 397 | { |
| 398 | int ret = NUMA_NO_NODE; |
| 399 | int i; |
| 400 | |
| 401 | for_each_node_mask(i, nodes_parsed) { |
| 402 | /* |
| 403 | * Find the real node that this emulated node appears on. For |
| 404 | * the sake of simplicity, we only use a real node's starting |
| 405 | * address to determine which emulated node it appears on. |
| 406 | */ |
| 407 | if (addr >= nodes[i].start && addr < nodes[i].end) { |
| 408 | ret = i; |
| 409 | break; |
| 410 | } |
| 411 | } |
Minoru Usui | 9a1b62f | 2008-01-30 13:33:35 +0100 | [diff] [blame] | 412 | return ret; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 413 | } |
| 414 | |
| 415 | /* |
| 416 | * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID |
| 417 | * mappings that respect the real ACPI topology but reflect our emulated |
| 418 | * environment. For each emulated node, we find which real node it appears on |
| 419 | * and create PXM to NID mappings for those fake nodes which mirror that |
| 420 | * locality. SLIT will now represent the correct distances between emulated |
| 421 | * nodes as a result of the real topology. |
| 422 | */ |
| 423 | void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes) |
| 424 | { |
David Rientjes | 08705b8 | 2007-07-21 17:10:33 +0200 | [diff] [blame] | 425 | int i, j; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 426 | |
| 427 | printk(KERN_INFO "Faking PXM affinity for fake nodes on real " |
| 428 | "topology.\n"); |
| 429 | for (i = 0; i < num_nodes; i++) { |
| 430 | int nid, pxm; |
| 431 | |
| 432 | nid = find_node_by_addr(fake_nodes[i].start); |
| 433 | if (nid == NUMA_NO_NODE) |
| 434 | continue; |
| 435 | pxm = node_to_pxm(nid); |
| 436 | if (pxm == PXM_INVAL) |
| 437 | continue; |
| 438 | fake_node_to_pxm_map[i] = pxm; |
David Rientjes | 08705b8 | 2007-07-21 17:10:33 +0200 | [diff] [blame] | 439 | /* |
| 440 | * For each apicid_to_node mapping that exists for this real |
| 441 | * node, it must now point to the fake node ID. |
| 442 | */ |
| 443 | for (j = 0; j < MAX_LOCAL_APIC; j++) |
| 444 | if (apicid_to_node[j] == nid) |
| 445 | fake_apicid_to_node[j] = i; |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 446 | } |
| 447 | for (i = 0; i < num_nodes; i++) |
| 448 | __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i); |
David Rientjes | 08705b8 | 2007-07-21 17:10:33 +0200 | [diff] [blame] | 449 | memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node)); |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 450 | |
| 451 | nodes_clear(nodes_parsed); |
| 452 | for (i = 0; i < num_nodes; i++) |
| 453 | if (fake_nodes[i].start != fake_nodes[i].end) |
| 454 | node_set(i, nodes_parsed); |
| 455 | WARN_ON(!nodes_cover_memory(fake_nodes)); |
| 456 | } |
| 457 | |
| 458 | static int null_slit_node_compare(int a, int b) |
| 459 | { |
| 460 | return node_to_pxm(a) == node_to_pxm(b); |
| 461 | } |
| 462 | #else |
| 463 | static int null_slit_node_compare(int a, int b) |
| 464 | { |
| 465 | return a == b; |
| 466 | } |
| 467 | #endif /* CONFIG_NUMA_EMU */ |
| 468 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 469 | int __node_distance(int a, int b) |
| 470 | { |
| 471 | int index; |
| 472 | |
| 473 | if (!acpi_slit) |
David Rientjes | 3484d79 | 2007-07-21 17:10:32 +0200 | [diff] [blame] | 474 | return null_slit_node_compare(a, b) ? LOCAL_DISTANCE : |
| 475 | REMOTE_DISTANCE; |
Alexey Starikovskiy | 15a58ed | 2007-02-02 19:48:22 +0300 | [diff] [blame] | 476 | index = acpi_slit->locality_count * node_to_pxm(a); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 477 | return acpi_slit->entry[index + node_to_pxm(b)]; |
| 478 | } |
| 479 | |
| 480 | EXPORT_SYMBOL(__node_distance); |
Keith Mannthey | 4942e99 | 2006-09-30 23:27:06 -0700 | [diff] [blame] | 481 | |
Thomas Gleixner | 6a1673a | 2008-05-12 15:43:38 +0200 | [diff] [blame] | 482 | #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY) |
Keith Mannthey | 4942e99 | 2006-09-30 23:27:06 -0700 | [diff] [blame] | 483 | int memory_add_physaddr_to_nid(u64 start) |
| 484 | { |
| 485 | int i, ret = 0; |
| 486 | |
| 487 | for_each_node(i) |
| 488 | if (nodes_add[i].start <= start && nodes_add[i].end > start) |
| 489 | ret = i; |
| 490 | |
| 491 | return ret; |
| 492 | } |
Keith Mannthey | 8c2676a | 2006-09-30 23:27:07 -0700 | [diff] [blame] | 493 | EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); |
Thomas Gleixner | 6a1673a | 2008-05-12 15:43:38 +0200 | [diff] [blame] | 494 | #endif |