Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 1 | /* |
| 2 | * Procedures for maintaining information about logical memory blocks. |
| 3 | * |
| 4 | * Peter Bergner, IBM Corp. June 2001. |
| 5 | * Copyright (C) 2001 Peter Bergner. |
| 6 | * |
| 7 | * This program is free software; you can redistribute it and/or |
| 8 | * modify it under the terms of the GNU General Public License |
| 9 | * as published by the Free Software Foundation; either version |
| 10 | * 2 of the License, or (at your option) any later version. |
| 11 | */ |
| 12 | |
| 13 | #include <linux/kernel.h> |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 14 | #include <linux/slab.h> |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 15 | #include <linux/init.h> |
| 16 | #include <linux/bitops.h> |
Benjamin Herrenschmidt | 449e8df | 2010-07-06 15:39:07 -0700 | [diff] [blame] | 17 | #include <linux/poison.h> |
Benjamin Herrenschmidt | c196f76 | 2010-07-06 15:39:16 -0700 | [diff] [blame] | 18 | #include <linux/pfn.h> |
Benjamin Herrenschmidt | 6d03b88 | 2010-07-06 15:39:19 -0700 | [diff] [blame] | 19 | #include <linux/debugfs.h> |
| 20 | #include <linux/seq_file.h> |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 21 | #include <linux/memblock.h> |
| 22 | |
Tejun Heo | fe091c2 | 2011-12-08 10:22:07 -0800 | [diff] [blame] | 23 | static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock; |
| 24 | static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock; |
| 25 | |
| 26 | struct memblock memblock __initdata_memblock = { |
| 27 | .memory.regions = memblock_memory_init_regions, |
| 28 | .memory.cnt = 1, /* empty dummy entry */ |
| 29 | .memory.max = INIT_MEMBLOCK_REGIONS, |
| 30 | |
| 31 | .reserved.regions = memblock_reserved_init_regions, |
| 32 | .reserved.cnt = 1, /* empty dummy entry */ |
| 33 | .reserved.max = INIT_MEMBLOCK_REGIONS, |
| 34 | |
| 35 | .current_limit = MEMBLOCK_ALLOC_ANYWHERE, |
| 36 | }; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 37 | |
Yinghai Lu | 10d0643 | 2010-07-28 15:43:02 +1000 | [diff] [blame] | 38 | int memblock_debug __initdata_memblock; |
| 39 | int memblock_can_resize __initdata_memblock; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 40 | |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 41 | /* inline so we don't get a warning when pr_debug is compiled out */ |
| 42 | static inline const char *memblock_type_name(struct memblock_type *type) |
| 43 | { |
| 44 | if (type == &memblock.memory) |
| 45 | return "memory"; |
| 46 | else if (type == &memblock.reserved) |
| 47 | return "reserved"; |
| 48 | else |
| 49 | return "unknown"; |
| 50 | } |
| 51 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 52 | /* |
| 53 | * Address comparison utilities |
| 54 | */ |
Yinghai Lu | 10d0643 | 2010-07-28 15:43:02 +1000 | [diff] [blame] | 55 | static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, |
Benjamin Herrenschmidt | 2898cc4 | 2010-08-04 13:34:42 +1000 | [diff] [blame] | 56 | phys_addr_t base2, phys_addr_t size2) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 57 | { |
| 58 | return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); |
| 59 | } |
| 60 | |
H Hartley Sweeten | 2d7d3eb | 2011-10-31 17:09:15 -0700 | [diff] [blame] | 61 | static long __init_memblock memblock_overlaps_region(struct memblock_type *type, |
| 62 | phys_addr_t base, phys_addr_t size) |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 63 | { |
| 64 | unsigned long i; |
| 65 | |
| 66 | for (i = 0; i < type->cnt; i++) { |
| 67 | phys_addr_t rgnbase = type->regions[i].base; |
| 68 | phys_addr_t rgnsize = type->regions[i].size; |
| 69 | if (memblock_addrs_overlap(base, size, rgnbase, rgnsize)) |
| 70 | break; |
| 71 | } |
| 72 | |
| 73 | return (i < type->cnt) ? i : -1; |
| 74 | } |
| 75 | |
| 76 | /* |
| 77 | * Find, allocate, deallocate or reserve unreserved regions. All allocations |
| 78 | * are top-down. |
| 79 | */ |
| 80 | |
Yinghai Lu | cd79481 | 2010-10-11 12:34:09 -0700 | [diff] [blame] | 81 | static phys_addr_t __init_memblock memblock_find_region(phys_addr_t start, phys_addr_t end, |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 82 | phys_addr_t size, phys_addr_t align) |
| 83 | { |
| 84 | phys_addr_t base, res_base; |
| 85 | long j; |
| 86 | |
Yinghai Lu | f1af98c | 2010-10-04 14:57:39 -0700 | [diff] [blame] | 87 | /* In case, huge size is requested */ |
| 88 | if (end < size) |
Tejun Heo | 1f5026a | 2011-07-12 09:58:09 +0200 | [diff] [blame] | 89 | return 0; |
Yinghai Lu | f1af98c | 2010-10-04 14:57:39 -0700 | [diff] [blame] | 90 | |
Tejun Heo | 348968e | 2011-07-12 09:58:08 +0200 | [diff] [blame] | 91 | base = round_down(end - size, align); |
Yinghai Lu | f1af98c | 2010-10-04 14:57:39 -0700 | [diff] [blame] | 92 | |
Benjamin Herrenschmidt | 25818f0 | 2010-07-28 15:25:10 +1000 | [diff] [blame] | 93 | /* Prevent allocations returning 0 as it's also used to |
| 94 | * indicate an allocation failure |
| 95 | */ |
| 96 | if (start == 0) |
| 97 | start = PAGE_SIZE; |
| 98 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 99 | while (start <= base) { |
| 100 | j = memblock_overlaps_region(&memblock.reserved, base, size); |
| 101 | if (j < 0) |
| 102 | return base; |
| 103 | res_base = memblock.reserved.regions[j].base; |
| 104 | if (res_base < size) |
| 105 | break; |
Tejun Heo | 348968e | 2011-07-12 09:58:08 +0200 | [diff] [blame] | 106 | base = round_down(res_base - size, align); |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 107 | } |
| 108 | |
Tejun Heo | 1f5026a | 2011-07-12 09:58:09 +0200 | [diff] [blame] | 109 | return 0; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 110 | } |
| 111 | |
Tejun Heo | fc769a8 | 2011-07-12 09:58:10 +0200 | [diff] [blame] | 112 | /* |
| 113 | * Find a free area with specified alignment in a specific range. |
| 114 | */ |
| 115 | phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, phys_addr_t end, |
| 116 | phys_addr_t size, phys_addr_t align) |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 117 | { |
| 118 | long i; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 119 | |
| 120 | BUG_ON(0 == size); |
| 121 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 122 | /* Pump up max_addr */ |
Benjamin Herrenschmidt | fef501d | 2010-07-12 15:00:34 +1000 | [diff] [blame] | 123 | if (end == MEMBLOCK_ALLOC_ACCESSIBLE) |
| 124 | end = memblock.current_limit; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 125 | |
| 126 | /* We do a top-down search, this tends to limit memory |
| 127 | * fragmentation by keeping early boot allocs near the |
| 128 | * top of memory |
| 129 | */ |
| 130 | for (i = memblock.memory.cnt - 1; i >= 0; i--) { |
| 131 | phys_addr_t memblockbase = memblock.memory.regions[i].base; |
| 132 | phys_addr_t memblocksize = memblock.memory.regions[i].size; |
Benjamin Herrenschmidt | fef501d | 2010-07-12 15:00:34 +1000 | [diff] [blame] | 133 | phys_addr_t bottom, top, found; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 134 | |
| 135 | if (memblocksize < size) |
| 136 | continue; |
Benjamin Herrenschmidt | fef501d | 2010-07-12 15:00:34 +1000 | [diff] [blame] | 137 | if ((memblockbase + memblocksize) <= start) |
| 138 | break; |
| 139 | bottom = max(memblockbase, start); |
| 140 | top = min(memblockbase + memblocksize, end); |
| 141 | if (bottom >= top) |
| 142 | continue; |
| 143 | found = memblock_find_region(bottom, top, size, align); |
Tejun Heo | 1f5026a | 2011-07-12 09:58:09 +0200 | [diff] [blame] | 144 | if (found) |
Benjamin Herrenschmidt | fef501d | 2010-07-12 15:00:34 +1000 | [diff] [blame] | 145 | return found; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 146 | } |
Tejun Heo | 1f5026a | 2011-07-12 09:58:09 +0200 | [diff] [blame] | 147 | return 0; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 148 | } |
| 149 | |
Yinghai Lu | 5303b68 | 2010-07-28 15:38:40 +1000 | [diff] [blame] | 150 | /* |
Yinghai Lu | 7950c40 | 2010-08-25 13:39:14 -0700 | [diff] [blame] | 151 | * Free memblock.reserved.regions |
| 152 | */ |
| 153 | int __init_memblock memblock_free_reserved_regions(void) |
| 154 | { |
| 155 | if (memblock.reserved.regions == memblock_reserved_init_regions) |
| 156 | return 0; |
| 157 | |
| 158 | return memblock_free(__pa(memblock.reserved.regions), |
| 159 | sizeof(struct memblock_region) * memblock.reserved.max); |
| 160 | } |
| 161 | |
| 162 | /* |
| 163 | * Reserve memblock.reserved.regions |
| 164 | */ |
| 165 | int __init_memblock memblock_reserve_reserved_regions(void) |
| 166 | { |
| 167 | if (memblock.reserved.regions == memblock_reserved_init_regions) |
| 168 | return 0; |
| 169 | |
| 170 | return memblock_reserve(__pa(memblock.reserved.regions), |
| 171 | sizeof(struct memblock_region) * memblock.reserved.max); |
| 172 | } |
| 173 | |
Yinghai Lu | 10d0643 | 2010-07-28 15:43:02 +1000 | [diff] [blame] | 174 | static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 175 | { |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 176 | memmove(&type->regions[r], &type->regions[r + 1], |
| 177 | (type->cnt - (r + 1)) * sizeof(type->regions[r])); |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 178 | type->cnt--; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 179 | |
Benjamin Herrenschmidt | 8f7a660 | 2011-03-22 16:33:43 -0700 | [diff] [blame] | 180 | /* Special case for empty arrays */ |
| 181 | if (type->cnt == 0) { |
| 182 | type->cnt = 1; |
| 183 | type->regions[0].base = 0; |
| 184 | type->regions[0].size = 0; |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 185 | memblock_set_region_node(&type->regions[0], MAX_NUMNODES); |
Benjamin Herrenschmidt | 8f7a660 | 2011-03-22 16:33:43 -0700 | [diff] [blame] | 186 | } |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 187 | } |
| 188 | |
Yinghai Lu | 10d0643 | 2010-07-28 15:43:02 +1000 | [diff] [blame] | 189 | static int __init_memblock memblock_double_array(struct memblock_type *type) |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 190 | { |
| 191 | struct memblock_region *new_array, *old_array; |
| 192 | phys_addr_t old_size, new_size, addr; |
| 193 | int use_slab = slab_is_available(); |
| 194 | |
| 195 | /* We don't allow resizing until we know about the reserved regions |
| 196 | * of memory that aren't suitable for allocation |
| 197 | */ |
| 198 | if (!memblock_can_resize) |
| 199 | return -1; |
| 200 | |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 201 | /* Calculate new doubled size */ |
| 202 | old_size = type->max * sizeof(struct memblock_region); |
| 203 | new_size = old_size << 1; |
| 204 | |
| 205 | /* Try to find some space for it. |
| 206 | * |
| 207 | * WARNING: We assume that either slab_is_available() and we use it or |
| 208 | * we use MEMBLOCK for allocations. That means that this is unsafe to use |
| 209 | * when bootmem is currently active (unless bootmem itself is implemented |
| 210 | * on top of MEMBLOCK which isn't the case yet) |
| 211 | * |
| 212 | * This should however not be an issue for now, as we currently only |
| 213 | * call into MEMBLOCK while it's still active, or much later when slab is |
| 214 | * active for memory hotplug operations |
| 215 | */ |
| 216 | if (use_slab) { |
| 217 | new_array = kmalloc(new_size, GFP_KERNEL); |
Tejun Heo | 1f5026a | 2011-07-12 09:58:09 +0200 | [diff] [blame] | 218 | addr = new_array ? __pa(new_array) : 0; |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 219 | } else |
Tejun Heo | fc769a8 | 2011-07-12 09:58:10 +0200 | [diff] [blame] | 220 | addr = memblock_find_in_range(0, MEMBLOCK_ALLOC_ACCESSIBLE, new_size, sizeof(phys_addr_t)); |
Tejun Heo | 1f5026a | 2011-07-12 09:58:09 +0200 | [diff] [blame] | 221 | if (!addr) { |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 222 | pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", |
| 223 | memblock_type_name(type), type->max, type->max * 2); |
| 224 | return -1; |
| 225 | } |
| 226 | new_array = __va(addr); |
| 227 | |
Yinghai Lu | ea9e437 | 2010-07-28 15:13:22 +1000 | [diff] [blame] | 228 | memblock_dbg("memblock: %s array is doubled to %ld at [%#010llx-%#010llx]", |
| 229 | memblock_type_name(type), type->max * 2, (u64)addr, (u64)addr + new_size - 1); |
| 230 | |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 231 | /* Found space, we now need to move the array over before |
| 232 | * we add the reserved region since it may be our reserved |
| 233 | * array itself that is full. |
| 234 | */ |
| 235 | memcpy(new_array, type->regions, old_size); |
| 236 | memset(new_array + type->max, 0, old_size); |
| 237 | old_array = type->regions; |
| 238 | type->regions = new_array; |
| 239 | type->max <<= 1; |
| 240 | |
| 241 | /* If we use SLAB that's it, we are done */ |
| 242 | if (use_slab) |
| 243 | return 0; |
| 244 | |
| 245 | /* Add the new reserved region now. Should not fail ! */ |
Tejun Heo | 9c8c27e | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 246 | BUG_ON(memblock_reserve(addr, new_size)); |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 247 | |
| 248 | /* If the array wasn't our static init one, then free it. We only do |
| 249 | * that before SLAB is available as later on, we don't know whether |
| 250 | * to use kfree or free_bootmem_pages(). Shouldn't be a big deal |
| 251 | * anyways |
| 252 | */ |
| 253 | if (old_array != memblock_memory_init_regions && |
| 254 | old_array != memblock_reserved_init_regions) |
| 255 | memblock_free(__pa(old_array), old_size); |
| 256 | |
| 257 | return 0; |
| 258 | } |
| 259 | |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 260 | /** |
| 261 | * memblock_merge_regions - merge neighboring compatible regions |
| 262 | * @type: memblock type to scan |
| 263 | * |
| 264 | * Scan @type and merge neighboring compatible regions. |
| 265 | */ |
| 266 | static void __init_memblock memblock_merge_regions(struct memblock_type *type) |
| 267 | { |
| 268 | int i = 0; |
| 269 | |
| 270 | /* cnt never goes below 1 */ |
| 271 | while (i < type->cnt - 1) { |
| 272 | struct memblock_region *this = &type->regions[i]; |
| 273 | struct memblock_region *next = &type->regions[i + 1]; |
| 274 | |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 275 | if (this->base + this->size != next->base || |
| 276 | memblock_get_region_node(this) != |
| 277 | memblock_get_region_node(next)) { |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 278 | BUG_ON(this->base + this->size > next->base); |
| 279 | i++; |
| 280 | continue; |
| 281 | } |
| 282 | |
| 283 | this->size += next->size; |
| 284 | memmove(next, next + 1, (type->cnt - (i + 1)) * sizeof(*next)); |
| 285 | type->cnt--; |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | /** |
| 290 | * memblock_insert_region - insert new memblock region |
| 291 | * @type: memblock type to insert into |
| 292 | * @idx: index for the insertion point |
| 293 | * @base: base address of the new region |
| 294 | * @size: size of the new region |
| 295 | * |
| 296 | * Insert new memblock region [@base,@base+@size) into @type at @idx. |
| 297 | * @type must already have extra room to accomodate the new region. |
| 298 | */ |
| 299 | static void __init_memblock memblock_insert_region(struct memblock_type *type, |
| 300 | int idx, phys_addr_t base, |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 301 | phys_addr_t size, int nid) |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 302 | { |
| 303 | struct memblock_region *rgn = &type->regions[idx]; |
| 304 | |
| 305 | BUG_ON(type->cnt >= type->max); |
| 306 | memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); |
| 307 | rgn->base = base; |
| 308 | rgn->size = size; |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 309 | memblock_set_region_node(rgn, nid); |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 310 | type->cnt++; |
| 311 | } |
| 312 | |
| 313 | /** |
| 314 | * memblock_add_region - add new memblock region |
| 315 | * @type: memblock type to add new region into |
| 316 | * @base: base address of the new region |
| 317 | * @size: size of the new region |
| 318 | * |
| 319 | * Add new memblock region [@base,@base+@size) into @type. The new region |
| 320 | * is allowed to overlap with existing ones - overlaps don't affect already |
| 321 | * existing regions. @type is guaranteed to be minimal (all neighbouring |
| 322 | * compatible regions are merged) after the addition. |
| 323 | * |
| 324 | * RETURNS: |
| 325 | * 0 on success, -errno on failure. |
| 326 | */ |
Tejun Heo | 581adcb | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 327 | static int __init_memblock memblock_add_region(struct memblock_type *type, |
| 328 | phys_addr_t base, phys_addr_t size) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 329 | { |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 330 | bool insert = false; |
| 331 | phys_addr_t obase = base, end = base + size; |
| 332 | int i, nr_new; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 333 | |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 334 | /* special case for empty array */ |
| 335 | if (type->regions[0].size == 0) { |
| 336 | WARN_ON(type->cnt != 1); |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 337 | type->regions[0].base = base; |
| 338 | type->regions[0].size = size; |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 339 | memblock_set_region_node(&type->regions[0], MAX_NUMNODES); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 340 | return 0; |
| 341 | } |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 342 | repeat: |
| 343 | /* |
| 344 | * The following is executed twice. Once with %false @insert and |
| 345 | * then with %true. The first counts the number of regions needed |
| 346 | * to accomodate the new area. The second actually inserts them. |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 347 | */ |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 348 | base = obase; |
| 349 | nr_new = 0; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 350 | |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 351 | for (i = 0; i < type->cnt; i++) { |
| 352 | struct memblock_region *rgn = &type->regions[i]; |
| 353 | phys_addr_t rbase = rgn->base; |
| 354 | phys_addr_t rend = rbase + rgn->size; |
| 355 | |
| 356 | if (rbase >= end) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 357 | break; |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 358 | if (rend <= base) |
| 359 | continue; |
| 360 | /* |
| 361 | * @rgn overlaps. If it separates the lower part of new |
| 362 | * area, insert that portion. |
| 363 | */ |
| 364 | if (rbase > base) { |
| 365 | nr_new++; |
| 366 | if (insert) |
| 367 | memblock_insert_region(type, i++, base, |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 368 | rbase - base, MAX_NUMNODES); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 369 | } |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 370 | /* area below @rend is dealt with, forget about it */ |
| 371 | base = min(rend, end); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 372 | } |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 373 | |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 374 | /* insert the remaining portion */ |
| 375 | if (base < end) { |
| 376 | nr_new++; |
| 377 | if (insert) |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 378 | memblock_insert_region(type, i, base, end - base, |
| 379 | MAX_NUMNODES); |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 380 | } |
| 381 | |
| 382 | /* |
| 383 | * If this was the first round, resize array and repeat for actual |
| 384 | * insertions; otherwise, merge and return. |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 385 | */ |
Tejun Heo | 784656f | 2011-07-12 11:15:55 +0200 | [diff] [blame] | 386 | if (!insert) { |
| 387 | while (type->cnt + nr_new > type->max) |
| 388 | if (memblock_double_array(type) < 0) |
| 389 | return -ENOMEM; |
| 390 | insert = true; |
| 391 | goto repeat; |
| 392 | } else { |
| 393 | memblock_merge_regions(type); |
| 394 | return 0; |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 395 | } |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 396 | } |
| 397 | |
Tejun Heo | 581adcb | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 398 | int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 399 | { |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 400 | return memblock_add_region(&memblock.memory, base, size); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 401 | } |
| 402 | |
Tejun Heo | 6a9ceb3 | 2011-12-08 10:22:07 -0800 | [diff] [blame^] | 403 | #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP |
| 404 | /** |
| 405 | * memblock_isolate_range - isolate given range into disjoint memblocks |
| 406 | * @type: memblock type to isolate range for |
| 407 | * @base: base of range to isolate |
| 408 | * @size: size of range to isolate |
| 409 | * @start_rgn: out parameter for the start of isolated region |
| 410 | * @end_rgn: out parameter for the end of isolated region |
| 411 | * |
| 412 | * Walk @type and ensure that regions don't cross the boundaries defined by |
| 413 | * [@base,@base+@size). Crossing regions are split at the boundaries, |
| 414 | * which may create at most two more regions. The index of the first |
| 415 | * region inside the range is returned in *@start_rgn and end in *@end_rgn. |
| 416 | * |
| 417 | * RETURNS: |
| 418 | * 0 on success, -errno on failure. |
| 419 | */ |
| 420 | static int __init_memblock memblock_isolate_range(struct memblock_type *type, |
| 421 | phys_addr_t base, phys_addr_t size, |
| 422 | int *start_rgn, int *end_rgn) |
| 423 | { |
| 424 | phys_addr_t end = base + size; |
| 425 | int i; |
| 426 | |
| 427 | *start_rgn = *end_rgn = 0; |
| 428 | |
| 429 | /* we'll create at most two more regions */ |
| 430 | while (type->cnt + 2 > type->max) |
| 431 | if (memblock_double_array(type) < 0) |
| 432 | return -ENOMEM; |
| 433 | |
| 434 | for (i = 0; i < type->cnt; i++) { |
| 435 | struct memblock_region *rgn = &type->regions[i]; |
| 436 | phys_addr_t rbase = rgn->base; |
| 437 | phys_addr_t rend = rbase + rgn->size; |
| 438 | |
| 439 | if (rbase >= end) |
| 440 | break; |
| 441 | if (rend <= base) |
| 442 | continue; |
| 443 | |
| 444 | if (rbase < base) { |
| 445 | /* |
| 446 | * @rgn intersects from below. Split and continue |
| 447 | * to process the next region - the new top half. |
| 448 | */ |
| 449 | rgn->base = base; |
| 450 | rgn->size = rend - rgn->base; |
| 451 | memblock_insert_region(type, i, rbase, base - rbase, |
| 452 | rgn->nid); |
| 453 | } else if (rend > end) { |
| 454 | /* |
| 455 | * @rgn intersects from above. Split and redo the |
| 456 | * current region - the new bottom half. |
| 457 | */ |
| 458 | rgn->base = end; |
| 459 | rgn->size = rend - rgn->base; |
| 460 | memblock_insert_region(type, i--, rbase, end - rbase, |
| 461 | rgn->nid); |
| 462 | } else { |
| 463 | /* @rgn is fully contained, record it */ |
| 464 | if (!*end_rgn) |
| 465 | *start_rgn = i; |
| 466 | *end_rgn = i + 1; |
| 467 | } |
| 468 | } |
| 469 | |
| 470 | return 0; |
| 471 | } |
| 472 | #endif |
| 473 | |
Tejun Heo | 581adcb | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 474 | static int __init_memblock __memblock_remove(struct memblock_type *type, |
| 475 | phys_addr_t base, phys_addr_t size) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 476 | { |
Benjamin Herrenschmidt | 2898cc4 | 2010-08-04 13:34:42 +1000 | [diff] [blame] | 477 | phys_addr_t end = base + size; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 478 | int i; |
| 479 | |
Benjamin Herrenschmidt | 8f7a660 | 2011-03-22 16:33:43 -0700 | [diff] [blame] | 480 | /* Walk through the array for collisions */ |
| 481 | for (i = 0; i < type->cnt; i++) { |
| 482 | struct memblock_region *rgn = &type->regions[i]; |
| 483 | phys_addr_t rend = rgn->base + rgn->size; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 484 | |
Benjamin Herrenschmidt | 8f7a660 | 2011-03-22 16:33:43 -0700 | [diff] [blame] | 485 | /* Nothing more to do, exit */ |
| 486 | if (rgn->base > end || rgn->size == 0) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 487 | break; |
Benjamin Herrenschmidt | 8f7a660 | 2011-03-22 16:33:43 -0700 | [diff] [blame] | 488 | |
| 489 | /* If we fully enclose the block, drop it */ |
| 490 | if (base <= rgn->base && end >= rend) { |
| 491 | memblock_remove_region(type, i--); |
| 492 | continue; |
| 493 | } |
| 494 | |
| 495 | /* If we are fully enclosed within a block |
| 496 | * then we need to split it and we are done |
| 497 | */ |
| 498 | if (base > rgn->base && end < rend) { |
| 499 | rgn->size = base - rgn->base; |
| 500 | if (!memblock_add_region(type, end, rend - end)) |
| 501 | return 0; |
| 502 | /* Failure to split is bad, we at least |
| 503 | * restore the block before erroring |
| 504 | */ |
| 505 | rgn->size = rend - rgn->base; |
| 506 | WARN_ON(1); |
| 507 | return -1; |
| 508 | } |
| 509 | |
| 510 | /* Check if we need to trim the bottom of a block */ |
| 511 | if (rgn->base < end && rend > end) { |
| 512 | rgn->size -= end - rgn->base; |
| 513 | rgn->base = end; |
| 514 | break; |
| 515 | } |
| 516 | |
| 517 | /* And check if we need to trim the top of a block */ |
| 518 | if (base < rend) |
| 519 | rgn->size -= rend - base; |
| 520 | |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 521 | } |
Benjamin Herrenschmidt | 8f7a660 | 2011-03-22 16:33:43 -0700 | [diff] [blame] | 522 | return 0; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 523 | } |
| 524 | |
Tejun Heo | 581adcb | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 525 | int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 526 | { |
| 527 | return __memblock_remove(&memblock.memory, base, size); |
| 528 | } |
| 529 | |
Tejun Heo | 581adcb | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 530 | int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 531 | { |
Tejun Heo | 24aa078 | 2011-07-12 11:16:06 +0200 | [diff] [blame] | 532 | memblock_dbg(" memblock_free: [%#016llx-%#016llx] %pF\n", |
H. Peter Anvin | a150439 | 2011-07-14 11:57:10 -0700 | [diff] [blame] | 533 | (unsigned long long)base, |
| 534 | (unsigned long long)base + size, |
| 535 | (void *)_RET_IP_); |
Tejun Heo | 24aa078 | 2011-07-12 11:16:06 +0200 | [diff] [blame] | 536 | |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 537 | return __memblock_remove(&memblock.reserved, base, size); |
| 538 | } |
| 539 | |
Tejun Heo | 581adcb | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 540 | int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 541 | { |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 542 | struct memblock_type *_rgn = &memblock.reserved; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 543 | |
Tejun Heo | 24aa078 | 2011-07-12 11:16:06 +0200 | [diff] [blame] | 544 | memblock_dbg("memblock_reserve: [%#016llx-%#016llx] %pF\n", |
H. Peter Anvin | a150439 | 2011-07-14 11:57:10 -0700 | [diff] [blame] | 545 | (unsigned long long)base, |
| 546 | (unsigned long long)base + size, |
| 547 | (void *)_RET_IP_); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 548 | BUG_ON(0 == size); |
| 549 | |
| 550 | return memblock_add_region(_rgn, base, size); |
| 551 | } |
| 552 | |
Tejun Heo | 35fd080 | 2011-07-12 11:15:59 +0200 | [diff] [blame] | 553 | /** |
| 554 | * __next_free_mem_range - next function for for_each_free_mem_range() |
| 555 | * @idx: pointer to u64 loop variable |
| 556 | * @nid: nid: node selector, %MAX_NUMNODES for all nodes |
| 557 | * @p_start: ptr to phys_addr_t for start address of the range, can be %NULL |
| 558 | * @p_end: ptr to phys_addr_t for end address of the range, can be %NULL |
| 559 | * @p_nid: ptr to int for nid of the range, can be %NULL |
| 560 | * |
| 561 | * Find the first free area from *@idx which matches @nid, fill the out |
| 562 | * parameters, and update *@idx for the next iteration. The lower 32bit of |
| 563 | * *@idx contains index into memory region and the upper 32bit indexes the |
| 564 | * areas before each reserved region. For example, if reserved regions |
| 565 | * look like the following, |
| 566 | * |
| 567 | * 0:[0-16), 1:[32-48), 2:[128-130) |
| 568 | * |
| 569 | * The upper 32bit indexes the following regions. |
| 570 | * |
| 571 | * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) |
| 572 | * |
| 573 | * As both region arrays are sorted, the function advances the two indices |
| 574 | * in lockstep and returns each intersection. |
| 575 | */ |
| 576 | void __init_memblock __next_free_mem_range(u64 *idx, int nid, |
| 577 | phys_addr_t *out_start, |
| 578 | phys_addr_t *out_end, int *out_nid) |
| 579 | { |
| 580 | struct memblock_type *mem = &memblock.memory; |
| 581 | struct memblock_type *rsv = &memblock.reserved; |
| 582 | int mi = *idx & 0xffffffff; |
| 583 | int ri = *idx >> 32; |
| 584 | |
| 585 | for ( ; mi < mem->cnt; mi++) { |
| 586 | struct memblock_region *m = &mem->regions[mi]; |
| 587 | phys_addr_t m_start = m->base; |
| 588 | phys_addr_t m_end = m->base + m->size; |
| 589 | |
| 590 | /* only memory regions are associated with nodes, check it */ |
| 591 | if (nid != MAX_NUMNODES && nid != memblock_get_region_node(m)) |
| 592 | continue; |
| 593 | |
| 594 | /* scan areas before each reservation for intersection */ |
| 595 | for ( ; ri < rsv->cnt + 1; ri++) { |
| 596 | struct memblock_region *r = &rsv->regions[ri]; |
| 597 | phys_addr_t r_start = ri ? r[-1].base + r[-1].size : 0; |
| 598 | phys_addr_t r_end = ri < rsv->cnt ? r->base : ULLONG_MAX; |
| 599 | |
| 600 | /* if ri advanced past mi, break out to advance mi */ |
| 601 | if (r_start >= m_end) |
| 602 | break; |
| 603 | /* if the two regions intersect, we're done */ |
| 604 | if (m_start < r_end) { |
| 605 | if (out_start) |
| 606 | *out_start = max(m_start, r_start); |
| 607 | if (out_end) |
| 608 | *out_end = min(m_end, r_end); |
| 609 | if (out_nid) |
| 610 | *out_nid = memblock_get_region_node(m); |
| 611 | /* |
| 612 | * The region which ends first is advanced |
| 613 | * for the next iteration. |
| 614 | */ |
| 615 | if (m_end <= r_end) |
| 616 | mi++; |
| 617 | else |
| 618 | ri++; |
| 619 | *idx = (u32)mi | (u64)ri << 32; |
| 620 | return; |
| 621 | } |
| 622 | } |
| 623 | } |
| 624 | |
| 625 | /* signal end of iteration */ |
| 626 | *idx = ULLONG_MAX; |
| 627 | } |
| 628 | |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 629 | #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP |
| 630 | /* |
| 631 | * Common iterator interface used to define for_each_mem_range(). |
| 632 | */ |
| 633 | void __init_memblock __next_mem_pfn_range(int *idx, int nid, |
| 634 | unsigned long *out_start_pfn, |
| 635 | unsigned long *out_end_pfn, int *out_nid) |
| 636 | { |
| 637 | struct memblock_type *type = &memblock.memory; |
| 638 | struct memblock_region *r; |
| 639 | |
| 640 | while (++*idx < type->cnt) { |
| 641 | r = &type->regions[*idx]; |
| 642 | |
| 643 | if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) |
| 644 | continue; |
| 645 | if (nid == MAX_NUMNODES || nid == r->nid) |
| 646 | break; |
| 647 | } |
| 648 | if (*idx >= type->cnt) { |
| 649 | *idx = -1; |
| 650 | return; |
| 651 | } |
| 652 | |
| 653 | if (out_start_pfn) |
| 654 | *out_start_pfn = PFN_UP(r->base); |
| 655 | if (out_end_pfn) |
| 656 | *out_end_pfn = PFN_DOWN(r->base + r->size); |
| 657 | if (out_nid) |
| 658 | *out_nid = r->nid; |
| 659 | } |
| 660 | |
| 661 | /** |
| 662 | * memblock_set_node - set node ID on memblock regions |
| 663 | * @base: base of area to set node ID for |
| 664 | * @size: size of area to set node ID for |
| 665 | * @nid: node ID to set |
| 666 | * |
| 667 | * Set the nid of memblock memory regions in [@base,@base+@size) to @nid. |
| 668 | * Regions which cross the area boundaries are split as necessary. |
| 669 | * |
| 670 | * RETURNS: |
| 671 | * 0 on success, -errno on failure. |
| 672 | */ |
| 673 | int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, |
| 674 | int nid) |
| 675 | { |
| 676 | struct memblock_type *type = &memblock.memory; |
Tejun Heo | 6a9ceb3 | 2011-12-08 10:22:07 -0800 | [diff] [blame^] | 677 | int start_rgn, end_rgn; |
| 678 | int i, ret; |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 679 | |
Tejun Heo | 6a9ceb3 | 2011-12-08 10:22:07 -0800 | [diff] [blame^] | 680 | ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); |
| 681 | if (ret) |
| 682 | return ret; |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 683 | |
Tejun Heo | 6a9ceb3 | 2011-12-08 10:22:07 -0800 | [diff] [blame^] | 684 | for (i = start_rgn; i < end_rgn; i++) |
| 685 | type->regions[i].nid = nid; |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 686 | |
| 687 | memblock_merge_regions(type); |
| 688 | return 0; |
| 689 | } |
| 690 | #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ |
| 691 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 692 | phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 693 | { |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 694 | phys_addr_t found; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 695 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 696 | /* We align the size to limit fragmentation. Without this, a lot of |
| 697 | * small allocs quickly eat up the whole reserve array on sparc |
| 698 | */ |
Tejun Heo | 348968e | 2011-07-12 09:58:08 +0200 | [diff] [blame] | 699 | size = round_up(size, align); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 700 | |
Tejun Heo | fc769a8 | 2011-07-12 09:58:10 +0200 | [diff] [blame] | 701 | found = memblock_find_in_range(0, max_addr, size, align); |
Tejun Heo | 9c8c27e | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 702 | if (found && !memblock_reserve(found, size)) |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 703 | return found; |
| 704 | |
| 705 | return 0; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 706 | } |
| 707 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 708 | phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 709 | { |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 710 | phys_addr_t alloc; |
| 711 | |
| 712 | alloc = __memblock_alloc_base(size, align, max_addr); |
| 713 | |
| 714 | if (alloc == 0) |
| 715 | panic("ERROR: Failed to allocate 0x%llx bytes below 0x%llx.\n", |
| 716 | (unsigned long long) size, (unsigned long long) max_addr); |
| 717 | |
| 718 | return alloc; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 719 | } |
| 720 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 721 | phys_addr_t __init memblock_alloc(phys_addr_t size, phys_addr_t align) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 722 | { |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 723 | return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 724 | } |
| 725 | |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 726 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 727 | /* |
Tejun Heo | 34e1845 | 2011-07-12 10:46:33 +0200 | [diff] [blame] | 728 | * Additional node-local top-down allocators. |
Benjamin Herrenschmidt | c196f76 | 2010-07-06 15:39:16 -0700 | [diff] [blame] | 729 | * |
| 730 | * WARNING: Only available after early_node_map[] has been populated, |
| 731 | * on some architectures, that is after all the calls to add_active_range() |
| 732 | * have been done to populate it. |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 733 | */ |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 734 | |
Tejun Heo | 34e1845 | 2011-07-12 10:46:33 +0200 | [diff] [blame] | 735 | static phys_addr_t __init memblock_nid_range_rev(phys_addr_t start, |
| 736 | phys_addr_t end, int *nid) |
Benjamin Herrenschmidt | c3f72b5 | 2010-07-06 15:38:59 -0700 | [diff] [blame] | 737 | { |
Benjamin Herrenschmidt | c196f76 | 2010-07-06 15:39:16 -0700 | [diff] [blame] | 738 | #ifdef CONFIG_ARCH_POPULATES_NODE_MAP |
Benjamin Herrenschmidt | c196f76 | 2010-07-06 15:39:16 -0700 | [diff] [blame] | 739 | unsigned long start_pfn, end_pfn; |
| 740 | int i; |
| 741 | |
Tejun Heo | b2fea98 | 2011-07-12 10:46:31 +0200 | [diff] [blame] | 742 | for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, nid) |
Tejun Heo | 34e1845 | 2011-07-12 10:46:33 +0200 | [diff] [blame] | 743 | if (end > PFN_PHYS(start_pfn) && end <= PFN_PHYS(end_pfn)) |
| 744 | return max(start, PFN_PHYS(start_pfn)); |
Benjamin Herrenschmidt | c196f76 | 2010-07-06 15:39:16 -0700 | [diff] [blame] | 745 | #endif |
Benjamin Herrenschmidt | c3f72b5 | 2010-07-06 15:38:59 -0700 | [diff] [blame] | 746 | *nid = 0; |
Tejun Heo | 34e1845 | 2011-07-12 10:46:33 +0200 | [diff] [blame] | 747 | return start; |
Benjamin Herrenschmidt | c3f72b5 | 2010-07-06 15:38:59 -0700 | [diff] [blame] | 748 | } |
| 749 | |
Tejun Heo | e649804 | 2011-07-12 10:46:34 +0200 | [diff] [blame] | 750 | phys_addr_t __init memblock_find_in_range_node(phys_addr_t start, |
| 751 | phys_addr_t end, |
Benjamin Herrenschmidt | 2898cc4 | 2010-08-04 13:34:42 +1000 | [diff] [blame] | 752 | phys_addr_t size, |
| 753 | phys_addr_t align, int nid) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 754 | { |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 755 | struct memblock_type *mem = &memblock.memory; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 756 | int i; |
| 757 | |
| 758 | BUG_ON(0 == size); |
| 759 | |
Tejun Heo | e649804 | 2011-07-12 10:46:34 +0200 | [diff] [blame] | 760 | /* Pump up max_addr */ |
| 761 | if (end == MEMBLOCK_ALLOC_ACCESSIBLE) |
| 762 | end = memblock.current_limit; |
| 763 | |
| 764 | for (i = mem->cnt - 1; i >= 0; i--) { |
| 765 | struct memblock_region *r = &mem->regions[i]; |
| 766 | phys_addr_t base = max(start, r->base); |
| 767 | phys_addr_t top = min(end, r->base + r->size); |
| 768 | |
| 769 | while (base < top) { |
| 770 | phys_addr_t tbase, ret; |
| 771 | int tnid; |
| 772 | |
| 773 | tbase = memblock_nid_range_rev(base, top, &tnid); |
| 774 | if (nid == MAX_NUMNODES || tnid == nid) { |
| 775 | ret = memblock_find_region(tbase, top, size, align); |
| 776 | if (ret) |
| 777 | return ret; |
| 778 | } |
| 779 | top = tbase; |
| 780 | } |
| 781 | } |
| 782 | |
| 783 | return 0; |
| 784 | } |
| 785 | |
| 786 | phys_addr_t __init memblock_alloc_nid(phys_addr_t size, phys_addr_t align, int nid) |
| 787 | { |
| 788 | phys_addr_t found; |
| 789 | |
| 790 | /* |
| 791 | * We align the size to limit fragmentation. Without this, a lot of |
Benjamin Herrenschmidt | 7f219c7 | 2010-07-12 14:24:57 +1000 | [diff] [blame] | 792 | * small allocs quickly eat up the whole reserve array on sparc |
| 793 | */ |
Tejun Heo | 348968e | 2011-07-12 09:58:08 +0200 | [diff] [blame] | 794 | size = round_up(size, align); |
Benjamin Herrenschmidt | 7f219c7 | 2010-07-12 14:24:57 +1000 | [diff] [blame] | 795 | |
Tejun Heo | e649804 | 2011-07-12 10:46:34 +0200 | [diff] [blame] | 796 | found = memblock_find_in_range_node(0, MEMBLOCK_ALLOC_ACCESSIBLE, |
| 797 | size, align, nid); |
Tejun Heo | 9c8c27e | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 798 | if (found && !memblock_reserve(found, size)) |
Tejun Heo | e649804 | 2011-07-12 10:46:34 +0200 | [diff] [blame] | 799 | return found; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 800 | |
Benjamin Herrenschmidt | 9d1e249 | 2010-07-06 15:39:17 -0700 | [diff] [blame] | 801 | return 0; |
| 802 | } |
| 803 | |
| 804 | phys_addr_t __init memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) |
| 805 | { |
| 806 | phys_addr_t res = memblock_alloc_nid(size, align, nid); |
| 807 | |
| 808 | if (res) |
| 809 | return res; |
Tejun Heo | 15fb097 | 2011-07-12 09:58:07 +0200 | [diff] [blame] | 810 | return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 811 | } |
| 812 | |
Benjamin Herrenschmidt | 9d1e249 | 2010-07-06 15:39:17 -0700 | [diff] [blame] | 813 | |
| 814 | /* |
| 815 | * Remaining API functions |
| 816 | */ |
| 817 | |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 818 | /* You must call memblock_analyze() before this. */ |
Benjamin Herrenschmidt | 2898cc4 | 2010-08-04 13:34:42 +1000 | [diff] [blame] | 819 | phys_addr_t __init memblock_phys_mem_size(void) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 820 | { |
Benjamin Herrenschmidt | 4734b59 | 2010-07-28 14:31:29 +1000 | [diff] [blame] | 821 | return memblock.memory_size; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 822 | } |
| 823 | |
Sam Ravnborg | 0a93ebe | 2011-10-31 17:08:16 -0700 | [diff] [blame] | 824 | /* lowest address */ |
| 825 | phys_addr_t __init_memblock memblock_start_of_DRAM(void) |
| 826 | { |
| 827 | return memblock.memory.regions[0].base; |
| 828 | } |
| 829 | |
Yinghai Lu | 10d0643 | 2010-07-28 15:43:02 +1000 | [diff] [blame] | 830 | phys_addr_t __init_memblock memblock_end_of_DRAM(void) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 831 | { |
| 832 | int idx = memblock.memory.cnt - 1; |
| 833 | |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 834 | return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 835 | } |
| 836 | |
| 837 | /* You must call memblock_analyze() after this. */ |
Benjamin Herrenschmidt | 2898cc4 | 2010-08-04 13:34:42 +1000 | [diff] [blame] | 838 | void __init memblock_enforce_memory_limit(phys_addr_t memory_limit) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 839 | { |
| 840 | unsigned long i; |
Benjamin Herrenschmidt | 2898cc4 | 2010-08-04 13:34:42 +1000 | [diff] [blame] | 841 | phys_addr_t limit; |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 842 | struct memblock_region *p; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 843 | |
| 844 | if (!memory_limit) |
| 845 | return; |
| 846 | |
| 847 | /* Truncate the memblock regions to satisfy the memory limit. */ |
| 848 | limit = memory_limit; |
| 849 | for (i = 0; i < memblock.memory.cnt; i++) { |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 850 | if (limit > memblock.memory.regions[i].size) { |
| 851 | limit -= memblock.memory.regions[i].size; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 852 | continue; |
| 853 | } |
| 854 | |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 855 | memblock.memory.regions[i].size = limit; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 856 | memblock.memory.cnt = i + 1; |
| 857 | break; |
| 858 | } |
| 859 | |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 860 | memory_limit = memblock_end_of_DRAM(); |
| 861 | |
| 862 | /* And truncate any reserves above the limit also. */ |
| 863 | for (i = 0; i < memblock.reserved.cnt; i++) { |
Benjamin Herrenschmidt | e3239ff | 2010-08-04 14:06:41 +1000 | [diff] [blame] | 864 | p = &memblock.reserved.regions[i]; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 865 | |
| 866 | if (p->base > memory_limit) |
| 867 | p->size = 0; |
| 868 | else if ((p->base + p->size) > memory_limit) |
| 869 | p->size = memory_limit - p->base; |
| 870 | |
| 871 | if (p->size == 0) { |
| 872 | memblock_remove_region(&memblock.reserved, i); |
| 873 | i--; |
| 874 | } |
| 875 | } |
| 876 | } |
| 877 | |
Yinghai Lu | cd79481 | 2010-10-11 12:34:09 -0700 | [diff] [blame] | 878 | static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) |
Benjamin Herrenschmidt | 72d4b0b | 2010-08-04 14:38:47 +1000 | [diff] [blame] | 879 | { |
| 880 | unsigned int left = 0, right = type->cnt; |
| 881 | |
| 882 | do { |
| 883 | unsigned int mid = (right + left) / 2; |
| 884 | |
| 885 | if (addr < type->regions[mid].base) |
| 886 | right = mid; |
| 887 | else if (addr >= (type->regions[mid].base + |
| 888 | type->regions[mid].size)) |
| 889 | left = mid + 1; |
| 890 | else |
| 891 | return mid; |
| 892 | } while (left < right); |
| 893 | return -1; |
| 894 | } |
| 895 | |
Benjamin Herrenschmidt | 2898cc4 | 2010-08-04 13:34:42 +1000 | [diff] [blame] | 896 | int __init memblock_is_reserved(phys_addr_t addr) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 897 | { |
Benjamin Herrenschmidt | 72d4b0b | 2010-08-04 14:38:47 +1000 | [diff] [blame] | 898 | return memblock_search(&memblock.reserved, addr) != -1; |
| 899 | } |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 900 | |
Yinghai Lu | 3661ca6 | 2010-09-15 13:05:29 -0700 | [diff] [blame] | 901 | int __init_memblock memblock_is_memory(phys_addr_t addr) |
Benjamin Herrenschmidt | 72d4b0b | 2010-08-04 14:38:47 +1000 | [diff] [blame] | 902 | { |
| 903 | return memblock_search(&memblock.memory, addr) != -1; |
| 904 | } |
| 905 | |
Yinghai Lu | 3661ca6 | 2010-09-15 13:05:29 -0700 | [diff] [blame] | 906 | int __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) |
Benjamin Herrenschmidt | 72d4b0b | 2010-08-04 14:38:47 +1000 | [diff] [blame] | 907 | { |
Tomi Valkeinen | abb6527 | 2011-01-20 14:44:20 -0800 | [diff] [blame] | 908 | int idx = memblock_search(&memblock.memory, base); |
Benjamin Herrenschmidt | 72d4b0b | 2010-08-04 14:38:47 +1000 | [diff] [blame] | 909 | |
| 910 | if (idx == -1) |
| 911 | return 0; |
Tomi Valkeinen | abb6527 | 2011-01-20 14:44:20 -0800 | [diff] [blame] | 912 | return memblock.memory.regions[idx].base <= base && |
| 913 | (memblock.memory.regions[idx].base + |
| 914 | memblock.memory.regions[idx].size) >= (base + size); |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 915 | } |
| 916 | |
Yinghai Lu | 10d0643 | 2010-07-28 15:43:02 +1000 | [diff] [blame] | 917 | int __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 918 | { |
Benjamin Herrenschmidt | f1c2c19 | 2010-08-04 14:17:17 +1000 | [diff] [blame] | 919 | return memblock_overlaps_region(&memblock.reserved, base, size) >= 0; |
Yinghai Lu | 95f72d1 | 2010-07-12 14:36:09 +1000 | [diff] [blame] | 920 | } |
| 921 | |
Benjamin Herrenschmidt | e63075a | 2010-07-06 15:39:01 -0700 | [diff] [blame] | 922 | |
Yinghai Lu | 3661ca6 | 2010-09-15 13:05:29 -0700 | [diff] [blame] | 923 | void __init_memblock memblock_set_current_limit(phys_addr_t limit) |
Benjamin Herrenschmidt | e63075a | 2010-07-06 15:39:01 -0700 | [diff] [blame] | 924 | { |
| 925 | memblock.current_limit = limit; |
| 926 | } |
| 927 | |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 928 | static void __init_memblock memblock_dump(struct memblock_type *type, char *name) |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 929 | { |
| 930 | unsigned long long base, size; |
| 931 | int i; |
| 932 | |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 933 | pr_info(" %s.cnt = 0x%lx\n", name, type->cnt); |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 934 | |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 935 | for (i = 0; i < type->cnt; i++) { |
| 936 | struct memblock_region *rgn = &type->regions[i]; |
| 937 | char nid_buf[32] = ""; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 938 | |
Tejun Heo | 7c0caeb | 2011-07-14 11:43:42 +0200 | [diff] [blame] | 939 | base = rgn->base; |
| 940 | size = rgn->size; |
| 941 | #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP |
| 942 | if (memblock_get_region_node(rgn) != MAX_NUMNODES) |
| 943 | snprintf(nid_buf, sizeof(nid_buf), " on node %d", |
| 944 | memblock_get_region_node(rgn)); |
| 945 | #endif |
| 946 | pr_info(" %s[%#x]\t[%#016llx-%#016llx], %#llx bytes%s\n", |
| 947 | name, i, base, base + size - 1, size, nid_buf); |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 948 | } |
| 949 | } |
| 950 | |
Tejun Heo | 4ff7b82 | 2011-12-08 10:22:06 -0800 | [diff] [blame] | 951 | void __init_memblock __memblock_dump_all(void) |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 952 | { |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 953 | pr_info("MEMBLOCK configuration:\n"); |
| 954 | pr_info(" memory size = 0x%llx\n", (unsigned long long)memblock.memory_size); |
| 955 | |
| 956 | memblock_dump(&memblock.memory, "memory"); |
| 957 | memblock_dump(&memblock.reserved, "reserved"); |
| 958 | } |
| 959 | |
| 960 | void __init memblock_analyze(void) |
| 961 | { |
| 962 | int i; |
| 963 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 964 | memblock.memory_size = 0; |
| 965 | |
| 966 | for (i = 0; i < memblock.memory.cnt; i++) |
| 967 | memblock.memory_size += memblock.memory.regions[i].size; |
Benjamin Herrenschmidt | 142b45a | 2010-07-06 15:39:13 -0700 | [diff] [blame] | 968 | |
| 969 | /* We allow resizing from there */ |
| 970 | memblock_can_resize = 1; |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 971 | } |
| 972 | |
Benjamin Herrenschmidt | 6ed311b | 2010-07-12 14:36:48 +1000 | [diff] [blame] | 973 | static int __init early_memblock(char *p) |
| 974 | { |
| 975 | if (p && strstr(p, "debug")) |
| 976 | memblock_debug = 1; |
| 977 | return 0; |
| 978 | } |
| 979 | early_param("memblock", early_memblock); |
| 980 | |
Tejun Heo | c378ddd | 2011-07-14 11:46:03 +0200 | [diff] [blame] | 981 | #if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_ARCH_DISCARD_MEMBLOCK) |
Benjamin Herrenschmidt | 6d03b88 | 2010-07-06 15:39:19 -0700 | [diff] [blame] | 982 | |
| 983 | static int memblock_debug_show(struct seq_file *m, void *private) |
| 984 | { |
| 985 | struct memblock_type *type = m->private; |
| 986 | struct memblock_region *reg; |
| 987 | int i; |
| 988 | |
| 989 | for (i = 0; i < type->cnt; i++) { |
| 990 | reg = &type->regions[i]; |
| 991 | seq_printf(m, "%4d: ", i); |
| 992 | if (sizeof(phys_addr_t) == 4) |
| 993 | seq_printf(m, "0x%08lx..0x%08lx\n", |
| 994 | (unsigned long)reg->base, |
| 995 | (unsigned long)(reg->base + reg->size - 1)); |
| 996 | else |
| 997 | seq_printf(m, "0x%016llx..0x%016llx\n", |
| 998 | (unsigned long long)reg->base, |
| 999 | (unsigned long long)(reg->base + reg->size - 1)); |
| 1000 | |
| 1001 | } |
| 1002 | return 0; |
| 1003 | } |
| 1004 | |
| 1005 | static int memblock_debug_open(struct inode *inode, struct file *file) |
| 1006 | { |
| 1007 | return single_open(file, memblock_debug_show, inode->i_private); |
| 1008 | } |
| 1009 | |
| 1010 | static const struct file_operations memblock_debug_fops = { |
| 1011 | .open = memblock_debug_open, |
| 1012 | .read = seq_read, |
| 1013 | .llseek = seq_lseek, |
| 1014 | .release = single_release, |
| 1015 | }; |
| 1016 | |
| 1017 | static int __init memblock_init_debugfs(void) |
| 1018 | { |
| 1019 | struct dentry *root = debugfs_create_dir("memblock", NULL); |
| 1020 | if (!root) |
| 1021 | return -ENXIO; |
| 1022 | debugfs_create_file("memory", S_IRUGO, root, &memblock.memory, &memblock_debug_fops); |
| 1023 | debugfs_create_file("reserved", S_IRUGO, root, &memblock.reserved, &memblock_debug_fops); |
| 1024 | |
| 1025 | return 0; |
| 1026 | } |
| 1027 | __initcall(memblock_init_debugfs); |
| 1028 | |
| 1029 | #endif /* CONFIG_DEBUG_FS */ |