Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * ACPI 3.0 based NUMA setup |
| 3 | * Copyright 2004 Andi Kleen, SuSE Labs. |
| 4 | * |
| 5 | * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs. |
| 6 | * |
| 7 | * Called from acpi_numa_init while reading the SRAT and SLIT tables. |
| 8 | * Assumes all memory regions belonging to a single proximity domain |
| 9 | * are in one chunk. Holes between them will be included in the node. |
| 10 | */ |
| 11 | |
| 12 | #include <linux/kernel.h> |
| 13 | #include <linux/acpi.h> |
| 14 | #include <linux/mmzone.h> |
| 15 | #include <linux/bitmap.h> |
| 16 | #include <linux/module.h> |
| 17 | #include <linux/topology.h> |
| 18 | #include <asm/proto.h> |
| 19 | #include <asm/numa.h> |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 20 | #include <asm/e820.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | |
| 22 | static struct acpi_table_slit *acpi_slit; |
| 23 | |
| 24 | static nodemask_t nodes_parsed __initdata; |
| 25 | static nodemask_t nodes_found __initdata; |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 26 | static struct bootnode nodes[MAX_NUMNODES] __initdata; |
Andi Kleen | e4e9407 | 2006-01-11 22:43:48 +0100 | [diff] [blame] | 27 | static u8 pxm2node[256] = { [0 ... 255] = 0xff }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | |
Andi Kleen | 9391a3f | 2006-02-03 21:51:17 +0100 | [diff] [blame] | 29 | /* Too small nodes confuse the VM badly. Usually they result |
| 30 | from BIOS bugs. */ |
| 31 | #define NODE_MIN_SIZE (4*1024*1024) |
| 32 | |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 33 | static int node_to_pxm(int n); |
| 34 | |
Andi Kleen | 69e1a33 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 35 | int pxm_to_node(int pxm) |
| 36 | { |
| 37 | if ((unsigned)pxm >= 256) |
Andi Kleen | e4e9407 | 2006-01-11 22:43:48 +0100 | [diff] [blame] | 38 | return -1; |
| 39 | /* Extend 0xff to (int)-1 */ |
| 40 | return (signed char)pxm2node[pxm]; |
Andi Kleen | 69e1a33 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 41 | } |
| 42 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 43 | static __init int setup_node(int pxm) |
| 44 | { |
| 45 | unsigned node = pxm2node[pxm]; |
| 46 | if (node == 0xff) { |
| 47 | if (nodes_weight(nodes_found) >= MAX_NUMNODES) |
| 48 | return -1; |
| 49 | node = first_unset_node(nodes_found); |
| 50 | node_set(node, nodes_found); |
| 51 | pxm2node[pxm] = node; |
| 52 | } |
| 53 | return pxm2node[pxm]; |
| 54 | } |
| 55 | |
| 56 | static __init int conflicting_nodes(unsigned long start, unsigned long end) |
| 57 | { |
| 58 | int i; |
Andi Kleen | 4b6a455 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 59 | for_each_node_mask(i, nodes_parsed) { |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 60 | struct bootnode *nd = &nodes[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 61 | if (nd->start == nd->end) |
| 62 | continue; |
| 63 | if (nd->end > start && nd->start < end) |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 64 | return i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | if (nd->end == end && nd->start == start) |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 66 | return i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | } |
| 68 | return -1; |
| 69 | } |
| 70 | |
| 71 | static __init void cutoff_node(int i, unsigned long start, unsigned long end) |
| 72 | { |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 73 | struct bootnode *nd = &nodes[i]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | if (nd->start < start) { |
| 75 | nd->start = start; |
| 76 | if (nd->end < nd->start) |
| 77 | nd->start = nd->end; |
| 78 | } |
| 79 | if (nd->end > end) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 80 | nd->end = end; |
| 81 | if (nd->start > nd->end) |
| 82 | nd->start = nd->end; |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | static __init void bad_srat(void) |
| 87 | { |
Andi Kleen | 2bce2b5 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 88 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 89 | printk(KERN_ERR "SRAT: SRAT not used.\n"); |
| 90 | acpi_numa = -1; |
Andi Kleen | 2bce2b5 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 91 | for (i = 0; i < MAX_LOCAL_APIC; i++) |
| 92 | apicid_to_node[i] = NUMA_NO_NODE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | } |
| 94 | |
| 95 | static __init inline int srat_disabled(void) |
| 96 | { |
| 97 | return numa_off || acpi_numa < 0; |
| 98 | } |
| 99 | |
Andi Kleen | 1584b89 | 2006-01-11 22:43:42 +0100 | [diff] [blame] | 100 | /* |
| 101 | * A lot of BIOS fill in 10 (= no distance) everywhere. This messes |
| 102 | * up the NUMA heuristics which wants the local node to have a smaller |
| 103 | * distance than the others. |
| 104 | * Do some quick checks here and only use the SLIT if it passes. |
| 105 | */ |
| 106 | static __init int slit_valid(struct acpi_table_slit *slit) |
| 107 | { |
| 108 | int i, j; |
| 109 | int d = slit->localities; |
| 110 | for (i = 0; i < d; i++) { |
| 111 | for (j = 0; j < d; j++) { |
| 112 | u8 val = slit->entry[d*i + j]; |
| 113 | if (i == j) { |
| 114 | if (val != 10) |
| 115 | return 0; |
| 116 | } else if (val <= 10) |
| 117 | return 0; |
| 118 | } |
| 119 | } |
| 120 | return 1; |
| 121 | } |
| 122 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | /* Callback for SLIT parsing */ |
| 124 | void __init acpi_numa_slit_init(struct acpi_table_slit *slit) |
| 125 | { |
Andi Kleen | 1584b89 | 2006-01-11 22:43:42 +0100 | [diff] [blame] | 126 | if (!slit_valid(slit)) { |
| 127 | printk(KERN_INFO "ACPI: SLIT table looks invalid. Not used.\n"); |
| 128 | return; |
| 129 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | acpi_slit = slit; |
| 131 | } |
| 132 | |
| 133 | /* Callback for Proximity Domain -> LAPIC mapping */ |
| 134 | void __init |
| 135 | acpi_numa_processor_affinity_init(struct acpi_table_processor_affinity *pa) |
| 136 | { |
| 137 | int pxm, node; |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 138 | if (srat_disabled()) |
| 139 | return; |
| 140 | if (pa->header.length != sizeof(struct acpi_table_processor_affinity)) { bad_srat(); |
| 141 | return; |
| 142 | } |
| 143 | if (pa->flags.enabled == 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 144 | return; |
| 145 | pxm = pa->proximity_domain; |
| 146 | node = setup_node(pxm); |
| 147 | if (node < 0) { |
| 148 | printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); |
| 149 | bad_srat(); |
| 150 | return; |
| 151 | } |
Andi Kleen | 0b07e98 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 152 | apicid_to_node[pa->apic_id] = node; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | acpi_numa = 1; |
Andi Kleen | 0b07e98 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 154 | printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n", |
| 155 | pxm, pa->apic_id, node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | } |
| 157 | |
| 158 | /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */ |
| 159 | void __init |
| 160 | acpi_numa_memory_affinity_init(struct acpi_table_memory_affinity *ma) |
| 161 | { |
Andi Kleen | abe059e | 2006-03-25 16:29:12 +0100 | [diff] [blame] | 162 | struct bootnode *nd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 163 | unsigned long start, end; |
| 164 | int node, pxm; |
| 165 | int i; |
| 166 | |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 167 | if (srat_disabled()) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 168 | return; |
Andi Kleen | d22fe80 | 2006-02-03 21:51:26 +0100 | [diff] [blame] | 169 | if (ma->header.length != sizeof(struct acpi_table_memory_affinity)) { |
| 170 | bad_srat(); |
| 171 | return; |
| 172 | } |
| 173 | if (ma->flags.enabled == 0) |
| 174 | return; |
| 175 | start = ma->base_addr_lo | ((u64)ma->base_addr_hi << 32); |
| 176 | end = start + (ma->length_lo | ((u64)ma->length_hi << 32)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | pxm = ma->proximity_domain; |
| 178 | node = setup_node(pxm); |
| 179 | if (node < 0) { |
| 180 | printk(KERN_ERR "SRAT: Too many proximity domains.\n"); |
| 181 | bad_srat(); |
| 182 | return; |
| 183 | } |
Andi Kleen | 69cb62e | 2005-07-28 21:15:39 -0700 | [diff] [blame] | 184 | /* It is fine to add this area to the nodes data it will be used later*/ |
| 185 | if (ma->flags.hot_pluggable == 1) |
| 186 | printk(KERN_INFO "SRAT: hot plug zone found %lx - %lx \n", |
| 187 | start, end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | i = conflicting_nodes(start, end); |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 189 | if (i == node) { |
| 190 | printk(KERN_WARNING |
| 191 | "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n", |
| 192 | pxm, start, end, nodes[i].start, nodes[i].end); |
| 193 | } else if (i >= 0) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | printk(KERN_ERR |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 195 | "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n", |
| 196 | pxm, start, end, node_to_pxm(i), |
| 197 | nodes[i].start, nodes[i].end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 198 | bad_srat(); |
| 199 | return; |
| 200 | } |
| 201 | nd = &nodes[node]; |
| 202 | if (!node_test_and_set(node, nodes_parsed)) { |
| 203 | nd->start = start; |
| 204 | nd->end = end; |
| 205 | } else { |
| 206 | if (start < nd->start) |
| 207 | nd->start = start; |
| 208 | if (nd->end < end) |
| 209 | nd->end = end; |
| 210 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | printk(KERN_INFO "SRAT: Node %u PXM %u %Lx-%Lx\n", node, pxm, |
| 212 | nd->start, nd->end); |
| 213 | } |
| 214 | |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 215 | /* Sanity check to catch more bad SRATs (they are amazingly common). |
| 216 | Make sure the PXMs cover all memory. */ |
| 217 | static int nodes_cover_memory(void) |
| 218 | { |
| 219 | int i; |
| 220 | unsigned long pxmram, e820ram; |
| 221 | |
| 222 | pxmram = 0; |
| 223 | for_each_node_mask(i, nodes_parsed) { |
| 224 | unsigned long s = nodes[i].start >> PAGE_SHIFT; |
| 225 | unsigned long e = nodes[i].end >> PAGE_SHIFT; |
| 226 | pxmram += e - s; |
| 227 | pxmram -= e820_hole_size(s, e); |
| 228 | } |
| 229 | |
| 230 | e820ram = end_pfn - e820_hole_size(0, end_pfn); |
Andi Kleen | fdb9df9 | 2006-02-16 23:42:13 +0100 | [diff] [blame] | 231 | /* We seem to lose 3 pages somewhere. Allow a bit of slack. */ |
| 232 | if ((long)(e820ram - pxmram) >= 1*1024*1024) { |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 233 | printk(KERN_ERR |
| 234 | "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n", |
| 235 | (pxmram << PAGE_SHIFT) >> 20, |
| 236 | (e820ram << PAGE_SHIFT) >> 20); |
| 237 | return 0; |
| 238 | } |
| 239 | return 1; |
| 240 | } |
| 241 | |
Andi Kleen | 9391a3f | 2006-02-03 21:51:17 +0100 | [diff] [blame] | 242 | static void unparse_node(int node) |
| 243 | { |
| 244 | int i; |
| 245 | node_clear(node, nodes_parsed); |
| 246 | for (i = 0; i < MAX_LOCAL_APIC; i++) { |
| 247 | if (apicid_to_node[i] == node) |
| 248 | apicid_to_node[i] = NUMA_NO_NODE; |
| 249 | } |
| 250 | } |
| 251 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 252 | void __init acpi_numa_arch_fixup(void) {} |
| 253 | |
| 254 | /* Use the information discovered above to actually set up the nodes. */ |
| 255 | int __init acpi_scan_nodes(unsigned long start, unsigned long end) |
| 256 | { |
| 257 | int i; |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 258 | |
Andi Kleen | 9391a3f | 2006-02-03 21:51:17 +0100 | [diff] [blame] | 259 | /* First clean up the node list */ |
| 260 | for (i = 0; i < MAX_NUMNODES; i++) { |
| 261 | cutoff_node(i, start, end); |
| 262 | if ((nodes[i].end - nodes[i].start) < NODE_MIN_SIZE) |
| 263 | unparse_node(i); |
| 264 | } |
| 265 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | if (acpi_numa <= 0) |
| 267 | return -1; |
Andi Kleen | e58e0d0 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 268 | |
Andi Kleen | 8a6fdd3 | 2006-01-11 22:44:39 +0100 | [diff] [blame] | 269 | if (!nodes_cover_memory()) { |
| 270 | bad_srat(); |
| 271 | return -1; |
| 272 | } |
| 273 | |
Andi Kleen | 2aed711 | 2006-02-16 23:42:16 +0100 | [diff] [blame] | 274 | memnode_shift = compute_hash_shift(nodes, MAX_NUMNODES); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 275 | if (memnode_shift < 0) { |
| 276 | printk(KERN_ERR |
| 277 | "SRAT: No NUMA node hash function found. Contact maintainer\n"); |
| 278 | bad_srat(); |
| 279 | return -1; |
| 280 | } |
Andi Kleen | e58e0d0 | 2005-09-12 18:49:25 +0200 | [diff] [blame] | 281 | |
| 282 | /* Finally register nodes */ |
| 283 | for_each_node_mask(i, nodes_parsed) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 284 | setup_node_bootmem(i, nodes[i].start, nodes[i].end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 285 | for (i = 0; i < NR_CPUS; i++) { |
| 286 | if (cpu_to_node[i] == NUMA_NO_NODE) |
| 287 | continue; |
| 288 | if (!node_isset(cpu_to_node[i], nodes_parsed)) |
Andi Kleen | 69d81fc | 2005-11-05 17:25:53 +0100 | [diff] [blame] | 289 | numa_set_node(i, NUMA_NO_NODE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 290 | } |
| 291 | numa_init_array(); |
| 292 | return 0; |
| 293 | } |
| 294 | |
Andi Kleen | 05d1fa4 | 2005-09-12 18:49:24 +0200 | [diff] [blame] | 295 | static int node_to_pxm(int n) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | { |
| 297 | int i; |
| 298 | if (pxm2node[n] == n) |
| 299 | return n; |
| 300 | for (i = 0; i < 256; i++) |
| 301 | if (pxm2node[i] == n) |
| 302 | return i; |
| 303 | return 0; |
| 304 | } |
| 305 | |
| 306 | int __node_distance(int a, int b) |
| 307 | { |
| 308 | int index; |
| 309 | |
| 310 | if (!acpi_slit) |
| 311 | return a == b ? 10 : 20; |
| 312 | index = acpi_slit->localities * node_to_pxm(a); |
| 313 | return acpi_slit->entry[index + node_to_pxm(b)]; |
| 314 | } |
| 315 | |
| 316 | EXPORT_SYMBOL(__node_distance); |