blob: 237944479d25a81483a7d6969cfab05c6f0a4c25 [file] [log] [blame]
Yinghai Lu95f72d12010-07-12 14:36:09 +10001/*
2 * Procedures for maintaining information about logical memory blocks.
3 *
4 * Peter Bergner, IBM Corp. June 2001.
5 * Copyright (C) 2001 Peter Bergner.
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 */
12
13#include <linux/kernel.h>
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -070014#include <linux/slab.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100015#include <linux/init.h>
16#include <linux/bitops.h>
Benjamin Herrenschmidt449e8df2010-07-06 15:39:07 -070017#include <linux/poison.h>
Benjamin Herrenschmidtc196f762010-07-06 15:39:16 -070018#include <linux/pfn.h>
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -070019#include <linux/debugfs.h>
Randy Dunlap514c6032018-04-05 16:25:34 -070020#include <linux/kmemleak.h>
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -070021#include <linux/seq_file.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100022#include <linux/memblock.h>
Mathieu Malaterre19373672018-07-20 17:53:31 -070023#include <linux/bootmem.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100024
Christoph Hellwigc4c5ad62016-07-28 15:48:06 -070025#include <asm/sections.h>
Santosh Shilimkar26f09e92014-01-21 15:50:19 -080026#include <linux/io.h>
27
28#include "internal.h"
Tang Chen79442ed2013-11-12 15:07:59 -080029
Mike Rapoport3e039c52018-06-30 17:55:05 +030030/**
31 * DOC: memblock overview
32 *
33 * Memblock is a method of managing memory regions during the early
34 * boot period when the usual kernel memory allocators are not up and
35 * running.
36 *
37 * Memblock views the system memory as collections of contiguous
38 * regions. There are several types of these collections:
39 *
40 * * ``memory`` - describes the physical memory available to the
41 * kernel; this may differ from the actual physical memory installed
42 * in the system, for instance when the memory is restricted with
43 * ``mem=`` command line parameter
44 * * ``reserved`` - describes the regions that were allocated
45 * * ``physmap`` - describes the actual physical memory regardless of
46 * the possible restrictions; the ``physmap`` type is only available
47 * on some architectures.
48 *
49 * Each region is represented by :c:type:`struct memblock_region` that
50 * defines the region extents, its attributes and NUMA node id on NUMA
51 * systems. Every memory type is described by the :c:type:`struct
52 * memblock_type` which contains an array of memory regions along with
53 * the allocator metadata. The memory types are nicely wrapped with
54 * :c:type:`struct memblock`. This structure is statically initialzed
55 * at build time. The region arrays for the "memory" and "reserved"
56 * types are initially sized to %INIT_MEMBLOCK_REGIONS and for the
57 * "physmap" type to %INIT_PHYSMEM_REGIONS.
58 * The :c:func:`memblock_allow_resize` enables automatic resizing of
59 * the region arrays during addition of new regions. This feature
60 * should be used with care so that memory allocated for the region
61 * array will not overlap with areas that should be reserved, for
62 * example initrd.
63 *
64 * The early architecture setup should tell memblock what the physical
65 * memory layout is by using :c:func:`memblock_add` or
66 * :c:func:`memblock_add_node` functions. The first function does not
67 * assign the region to a NUMA node and it is appropriate for UMA
68 * systems. Yet, it is possible to use it on NUMA systems as well and
69 * assign the region to a NUMA node later in the setup process using
70 * :c:func:`memblock_set_node`. The :c:func:`memblock_add_node`
71 * performs such an assignment directly.
72 *
73 * Once memblock is setup the memory can be allocated using either
74 * memblock or bootmem APIs.
75 *
76 * As the system boot progresses, the architecture specific
77 * :c:func:`mem_init` function frees all the memory to the buddy page
78 * allocator.
79 *
80 * If an architecure enables %CONFIG_ARCH_DISCARD_MEMBLOCK, the
81 * memblock data structures will be discarded after the system
82 * initialization compltes.
83 */
84
Tejun Heofe091c22011-12-08 10:22:07 -080085static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
86static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
Philipp Hachtmann70210ed2014-01-29 18:16:01 +010087#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
88static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS] __initdata_memblock;
89#endif
Tejun Heofe091c22011-12-08 10:22:07 -080090
91struct memblock memblock __initdata_memblock = {
92 .memory.regions = memblock_memory_init_regions,
93 .memory.cnt = 1, /* empty dummy entry */
94 .memory.max = INIT_MEMBLOCK_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -080095 .memory.name = "memory",
Tejun Heofe091c22011-12-08 10:22:07 -080096
97 .reserved.regions = memblock_reserved_init_regions,
98 .reserved.cnt = 1, /* empty dummy entry */
99 .reserved.max = INIT_MEMBLOCK_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800100 .reserved.name = "reserved",
Tejun Heofe091c22011-12-08 10:22:07 -0800101
Philipp Hachtmann70210ed2014-01-29 18:16:01 +0100102#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
103 .physmem.regions = memblock_physmem_init_regions,
104 .physmem.cnt = 1, /* empty dummy entry */
105 .physmem.max = INIT_PHYSMEM_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800106 .physmem.name = "physmem",
Philipp Hachtmann70210ed2014-01-29 18:16:01 +0100107#endif
108
Tang Chen79442ed2013-11-12 15:07:59 -0800109 .bottom_up = false,
Tejun Heofe091c22011-12-08 10:22:07 -0800110 .current_limit = MEMBLOCK_ALLOC_ANYWHERE,
111};
Yinghai Lu95f72d12010-07-12 14:36:09 +1000112
Yinghai Lu10d06432010-07-28 15:43:02 +1000113int memblock_debug __initdata_memblock;
Tony Lucka3f5baf2015-06-24 16:58:12 -0700114static bool system_has_some_mirror __initdata_memblock = false;
Tejun Heo1aadc052011-12-08 10:22:08 -0800115static int memblock_can_resize __initdata_memblock;
Gavin Shan181eb392012-05-29 15:06:50 -0700116static int memblock_memory_in_slab __initdata_memblock = 0;
117static int memblock_reserved_in_slab __initdata_memblock = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000118
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300119enum memblock_flags __init_memblock choose_memblock_flags(void)
Tony Lucka3f5baf2015-06-24 16:58:12 -0700120{
121 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
122}
123
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800124/* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
125static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
126{
Stefan Agner1c4bc432018-06-07 17:06:15 -0700127 return *size = min(*size, PHYS_ADDR_MAX - base);
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800128}
129
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000130/*
131 * Address comparison utilities
132 */
Yinghai Lu10d06432010-07-28 15:43:02 +1000133static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1,
Benjamin Herrenschmidt2898cc42010-08-04 13:34:42 +1000134 phys_addr_t base2, phys_addr_t size2)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000135{
136 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
137}
138
Tang Chen95cf82e2015-09-08 15:02:03 -0700139bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
H Hartley Sweeten2d7d3eb2011-10-31 17:09:15 -0700140 phys_addr_t base, phys_addr_t size)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000141{
142 unsigned long i;
143
Alexander Kuleshovf14516f2016-01-14 15:20:39 -0800144 for (i = 0; i < type->cnt; i++)
145 if (memblock_addrs_overlap(base, size, type->regions[i].base,
146 type->regions[i].size))
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000147 break;
Tang Chenc5c5c9d2015-09-08 15:02:00 -0700148 return i < type->cnt;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000149}
150
Mike Rapoport47cec442018-06-30 17:55:02 +0300151/**
Tang Chen79442ed2013-11-12 15:07:59 -0800152 * __memblock_find_range_bottom_up - find free area utility in bottom-up
153 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300154 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
155 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tang Chen79442ed2013-11-12 15:07:59 -0800156 * @size: size of free area to find
157 * @align: alignment of free area to find
Grygorii Strashkob1154232014-01-21 15:50:16 -0800158 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700159 * @flags: pick from blocks based on memory attributes
Tang Chen79442ed2013-11-12 15:07:59 -0800160 *
161 * Utility called from memblock_find_in_range_node(), find free area bottom-up.
162 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300163 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800164 * Found address on success, 0 on failure.
165 */
166static phys_addr_t __init_memblock
167__memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700168 phys_addr_t size, phys_addr_t align, int nid,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300169 enum memblock_flags flags)
Tang Chen79442ed2013-11-12 15:07:59 -0800170{
171 phys_addr_t this_start, this_end, cand;
172 u64 i;
173
Tony Luckfc6daaf2015-06-24 16:58:09 -0700174 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
Tang Chen79442ed2013-11-12 15:07:59 -0800175 this_start = clamp(this_start, start, end);
176 this_end = clamp(this_end, start, end);
177
178 cand = round_up(this_start, align);
179 if (cand < this_end && this_end - cand >= size)
180 return cand;
181 }
182
183 return 0;
184}
185
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800186/**
Tang Chen14028992013-11-12 15:07:57 -0800187 * __memblock_find_range_top_down - find free area utility, in top-down
188 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300189 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
190 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tang Chen14028992013-11-12 15:07:57 -0800191 * @size: size of free area to find
192 * @align: alignment of free area to find
Grygorii Strashkob1154232014-01-21 15:50:16 -0800193 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700194 * @flags: pick from blocks based on memory attributes
Tang Chen14028992013-11-12 15:07:57 -0800195 *
196 * Utility called from memblock_find_in_range_node(), find free area top-down.
197 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300198 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800199 * Found address on success, 0 on failure.
Tang Chen14028992013-11-12 15:07:57 -0800200 */
201static phys_addr_t __init_memblock
202__memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700203 phys_addr_t size, phys_addr_t align, int nid,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300204 enum memblock_flags flags)
Tang Chen14028992013-11-12 15:07:57 -0800205{
206 phys_addr_t this_start, this_end, cand;
207 u64 i;
208
Tony Luckfc6daaf2015-06-24 16:58:09 -0700209 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
210 NULL) {
Tang Chen14028992013-11-12 15:07:57 -0800211 this_start = clamp(this_start, start, end);
212 this_end = clamp(this_end, start, end);
213
214 if (this_end < size)
215 continue;
216
217 cand = round_down(this_end - size, align);
218 if (cand >= this_start)
219 return cand;
220 }
221
222 return 0;
223}
224
225/**
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800226 * memblock_find_in_range_node - find free area in given range and node
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800227 * @size: size of free area to find
228 * @align: alignment of free area to find
Grygorii Strashko87029ee2014-01-21 15:50:14 -0800229 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300230 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
231 * %MEMBLOCK_ALLOC_ACCESSIBLE
Grygorii Strashkob1154232014-01-21 15:50:16 -0800232 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700233 * @flags: pick from blocks based on memory attributes
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800234 *
235 * Find @size free area aligned to @align in the specified range and node.
236 *
Tang Chen79442ed2013-11-12 15:07:59 -0800237 * When allocation direction is bottom-up, the @start should be greater
238 * than the end of the kernel image. Otherwise, it will be trimmed. The
239 * reason is that we want the bottom-up allocation just near the kernel
240 * image so it is highly likely that the allocated memory and the kernel
241 * will reside in the same node.
242 *
243 * If bottom-up allocation failed, will try to allocate memory top-down.
244 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300245 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800246 * Found address on success, 0 on failure.
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000247 */
Grygorii Strashko87029ee2014-01-21 15:50:14 -0800248phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
249 phys_addr_t align, phys_addr_t start,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300250 phys_addr_t end, int nid,
251 enum memblock_flags flags)
Tang Chenf7210e62013-02-22 16:33:51 -0800252{
Tang Chen0cfb8f02014-08-29 15:18:31 -0700253 phys_addr_t kernel_end, ret;
Tang Chen79442ed2013-11-12 15:07:59 -0800254
Tang Chenf7210e62013-02-22 16:33:51 -0800255 /* pump up @end */
256 if (end == MEMBLOCK_ALLOC_ACCESSIBLE)
257 end = memblock.current_limit;
258
259 /* avoid allocating the first page */
260 start = max_t(phys_addr_t, start, PAGE_SIZE);
261 end = max(start, end);
Tang Chen79442ed2013-11-12 15:07:59 -0800262 kernel_end = __pa_symbol(_end);
263
264 /*
265 * try bottom-up allocation only when bottom-up mode
266 * is set and @end is above the kernel image.
267 */
268 if (memblock_bottom_up() && end > kernel_end) {
269 phys_addr_t bottom_up_start;
270
271 /* make sure we will allocate above the kernel */
272 bottom_up_start = max(start, kernel_end);
273
274 /* ok, try bottom-up allocation first */
275 ret = __memblock_find_range_bottom_up(bottom_up_start, end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700276 size, align, nid, flags);
Tang Chen79442ed2013-11-12 15:07:59 -0800277 if (ret)
278 return ret;
279
280 /*
281 * we always limit bottom-up allocation above the kernel,
282 * but top-down allocation doesn't have the limit, so
283 * retrying top-down allocation may succeed when bottom-up
284 * allocation failed.
285 *
286 * bottom-up allocation is expected to be fail very rarely,
287 * so we use WARN_ONCE() here to see the stack trace if
288 * fail happens.
289 */
Michal Hockoe3d301c2018-07-13 16:59:16 -0700290 WARN_ONCE(IS_ENABLED(CONFIG_MEMORY_HOTREMOVE),
291 "memblock: bottom-up allocation failed, memory hotremove may be affected\n");
Tang Chen79442ed2013-11-12 15:07:59 -0800292 }
Tang Chenf7210e62013-02-22 16:33:51 -0800293
Tony Luckfc6daaf2015-06-24 16:58:09 -0700294 return __memblock_find_range_top_down(start, end, size, align, nid,
295 flags);
Tang Chenf7210e62013-02-22 16:33:51 -0800296}
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000297
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800298/**
299 * memblock_find_in_range - find free area in given range
300 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300301 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
302 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800303 * @size: size of free area to find
304 * @align: alignment of free area to find
305 *
306 * Find @size free area aligned to @align in the specified range.
307 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300308 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800309 * Found address on success, 0 on failure.
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800310 */
311phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
312 phys_addr_t end, phys_addr_t size,
313 phys_addr_t align)
314{
Tony Lucka3f5baf2015-06-24 16:58:12 -0700315 phys_addr_t ret;
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300316 enum memblock_flags flags = choose_memblock_flags();
Tony Lucka3f5baf2015-06-24 16:58:12 -0700317
318again:
319 ret = memblock_find_in_range_node(size, align, start, end,
320 NUMA_NO_NODE, flags);
321
322 if (!ret && (flags & MEMBLOCK_MIRROR)) {
323 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
324 &size);
325 flags &= ~MEMBLOCK_MIRROR;
326 goto again;
327 }
328
329 return ret;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800330}
331
Yinghai Lu10d06432010-07-28 15:43:02 +1000332static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000333{
Tejun Heo1440c4e2011-12-08 10:22:08 -0800334 type->total_size -= type->regions[r].size;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200335 memmove(&type->regions[r], &type->regions[r + 1],
336 (type->cnt - (r + 1)) * sizeof(type->regions[r]));
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +1000337 type->cnt--;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000338
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700339 /* Special case for empty arrays */
340 if (type->cnt == 0) {
Tejun Heo1440c4e2011-12-08 10:22:08 -0800341 WARN_ON(type->total_size != 0);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700342 type->cnt = 1;
343 type->regions[0].base = 0;
344 type->regions[0].size = 0;
Tang Chen66a20752014-01-21 15:49:20 -0800345 type->regions[0].flags = 0;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200346 memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700347 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000348}
349
Philipp Hachtmann354f17e2014-01-23 15:53:24 -0800350#ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
Pavel Tatashin3010f872017-08-18 15:16:05 -0700351/**
Mike Rapoport47cec442018-06-30 17:55:02 +0300352 * memblock_discard - discard memory and reserved arrays if they were allocated
Pavel Tatashin3010f872017-08-18 15:16:05 -0700353 */
354void __init memblock_discard(void)
Yinghai Lu29f67382012-07-11 14:02:56 -0700355{
Pavel Tatashin3010f872017-08-18 15:16:05 -0700356 phys_addr_t addr, size;
Yinghai Lu29f67382012-07-11 14:02:56 -0700357
Pavel Tatashin3010f872017-08-18 15:16:05 -0700358 if (memblock.reserved.regions != memblock_reserved_init_regions) {
359 addr = __pa(memblock.reserved.regions);
360 size = PAGE_ALIGN(sizeof(struct memblock_region) *
361 memblock.reserved.max);
362 __memblock_free_late(addr, size);
363 }
Yinghai Lu29f67382012-07-11 14:02:56 -0700364
Pavel Tatashin91b540f2017-08-25 15:55:46 -0700365 if (memblock.memory.regions != memblock_memory_init_regions) {
Pavel Tatashin3010f872017-08-18 15:16:05 -0700366 addr = __pa(memblock.memory.regions);
367 size = PAGE_ALIGN(sizeof(struct memblock_region) *
368 memblock.memory.max);
369 __memblock_free_late(addr, size);
370 }
Yinghai Lu29f67382012-07-11 14:02:56 -0700371}
Philipp Hachtmann5e270e22014-01-23 15:53:11 -0800372#endif
373
Greg Pearson48c3b582012-06-20 12:53:05 -0700374/**
375 * memblock_double_array - double the size of the memblock regions array
376 * @type: memblock type of the regions array being doubled
377 * @new_area_start: starting address of memory range to avoid overlap with
378 * @new_area_size: size of memory range to avoid overlap with
379 *
380 * Double the size of the @type regions array. If memblock is being used to
381 * allocate memory for a new reserved regions array and there is a previously
Mike Rapoport47cec442018-06-30 17:55:02 +0300382 * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
Greg Pearson48c3b582012-06-20 12:53:05 -0700383 * waiting to be reserved, ensure the memory used by the new array does
384 * not overlap.
385 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300386 * Return:
Greg Pearson48c3b582012-06-20 12:53:05 -0700387 * 0 on success, -1 on failure.
388 */
389static int __init_memblock memblock_double_array(struct memblock_type *type,
390 phys_addr_t new_area_start,
391 phys_addr_t new_area_size)
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700392{
393 struct memblock_region *new_array, *old_array;
Yinghai Lu29f67382012-07-11 14:02:56 -0700394 phys_addr_t old_alloc_size, new_alloc_size;
Mike Rapoporta36aab82018-08-17 15:47:17 -0700395 phys_addr_t old_size, new_size, addr, new_end;
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700396 int use_slab = slab_is_available();
Gavin Shan181eb392012-05-29 15:06:50 -0700397 int *in_slab;
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700398
399 /* We don't allow resizing until we know about the reserved regions
400 * of memory that aren't suitable for allocation
401 */
402 if (!memblock_can_resize)
403 return -1;
404
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700405 /* Calculate new doubled size */
406 old_size = type->max * sizeof(struct memblock_region);
407 new_size = old_size << 1;
Yinghai Lu29f67382012-07-11 14:02:56 -0700408 /*
409 * We need to allocated new one align to PAGE_SIZE,
410 * so we can free them completely later.
411 */
412 old_alloc_size = PAGE_ALIGN(old_size);
413 new_alloc_size = PAGE_ALIGN(new_size);
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700414
Gavin Shan181eb392012-05-29 15:06:50 -0700415 /* Retrieve the slab flag */
416 if (type == &memblock.memory)
417 in_slab = &memblock_memory_in_slab;
418 else
419 in_slab = &memblock_reserved_in_slab;
420
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700421 /* Try to find some space for it.
422 *
423 * WARNING: We assume that either slab_is_available() and we use it or
Andrew Mortonfd073832012-07-31 16:42:40 -0700424 * we use MEMBLOCK for allocations. That means that this is unsafe to
425 * use when bootmem is currently active (unless bootmem itself is
426 * implemented on top of MEMBLOCK which isn't the case yet)
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700427 *
428 * This should however not be an issue for now, as we currently only
Andrew Mortonfd073832012-07-31 16:42:40 -0700429 * call into MEMBLOCK while it's still active, or much later when slab
430 * is active for memory hotplug operations
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700431 */
432 if (use_slab) {
433 new_array = kmalloc(new_size, GFP_KERNEL);
Tejun Heo1f5026a2011-07-12 09:58:09 +0200434 addr = new_array ? __pa(new_array) : 0;
Gavin Shan4e2f0772012-05-29 15:06:50 -0700435 } else {
Greg Pearson48c3b582012-06-20 12:53:05 -0700436 /* only exclude range when trying to double reserved.regions */
437 if (type != &memblock.reserved)
438 new_area_start = new_area_size = 0;
439
440 addr = memblock_find_in_range(new_area_start + new_area_size,
441 memblock.current_limit,
Yinghai Lu29f67382012-07-11 14:02:56 -0700442 new_alloc_size, PAGE_SIZE);
Greg Pearson48c3b582012-06-20 12:53:05 -0700443 if (!addr && new_area_size)
444 addr = memblock_find_in_range(0,
Andrew Mortonfd073832012-07-31 16:42:40 -0700445 min(new_area_start, memblock.current_limit),
446 new_alloc_size, PAGE_SIZE);
Greg Pearson48c3b582012-06-20 12:53:05 -0700447
Sachin Kamat15674862012-09-04 13:55:05 +0530448 new_array = addr ? __va(addr) : NULL;
Gavin Shan4e2f0772012-05-29 15:06:50 -0700449 }
Tejun Heo1f5026a2011-07-12 09:58:09 +0200450 if (!addr) {
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700451 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800452 type->name, type->max, type->max * 2);
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700453 return -1;
454 }
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700455
Mike Rapoporta36aab82018-08-17 15:47:17 -0700456 new_end = addr + new_size - 1;
457 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
458 type->name, type->max * 2, &addr, &new_end);
Yinghai Luea9e4372010-07-28 15:13:22 +1000459
Andrew Mortonfd073832012-07-31 16:42:40 -0700460 /*
461 * Found space, we now need to move the array over before we add the
462 * reserved region since it may be our reserved array itself that is
463 * full.
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700464 */
465 memcpy(new_array, type->regions, old_size);
466 memset(new_array + type->max, 0, old_size);
467 old_array = type->regions;
468 type->regions = new_array;
469 type->max <<= 1;
470
Andrew Mortonfd073832012-07-31 16:42:40 -0700471 /* Free old array. We needn't free it if the array is the static one */
Gavin Shan181eb392012-05-29 15:06:50 -0700472 if (*in_slab)
473 kfree(old_array);
474 else if (old_array != memblock_memory_init_regions &&
475 old_array != memblock_reserved_init_regions)
Yinghai Lu29f67382012-07-11 14:02:56 -0700476 memblock_free(__pa(old_array), old_alloc_size);
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700477
Andrew Mortonfd073832012-07-31 16:42:40 -0700478 /*
479 * Reserve the new array if that comes from the memblock. Otherwise, we
480 * needn't do it
Gavin Shan181eb392012-05-29 15:06:50 -0700481 */
482 if (!use_slab)
Yinghai Lu29f67382012-07-11 14:02:56 -0700483 BUG_ON(memblock_reserve(addr, new_alloc_size));
Gavin Shan181eb392012-05-29 15:06:50 -0700484
485 /* Update slab flag */
486 *in_slab = use_slab;
487
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700488 return 0;
489}
490
Tejun Heo784656f92011-07-12 11:15:55 +0200491/**
492 * memblock_merge_regions - merge neighboring compatible regions
493 * @type: memblock type to scan
494 *
495 * Scan @type and merge neighboring compatible regions.
496 */
497static void __init_memblock memblock_merge_regions(struct memblock_type *type)
498{
499 int i = 0;
500
501 /* cnt never goes below 1 */
502 while (i < type->cnt - 1) {
503 struct memblock_region *this = &type->regions[i];
504 struct memblock_region *next = &type->regions[i + 1];
505
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200506 if (this->base + this->size != next->base ||
507 memblock_get_region_node(this) !=
Tang Chen66a20752014-01-21 15:49:20 -0800508 memblock_get_region_node(next) ||
509 this->flags != next->flags) {
Tejun Heo784656f92011-07-12 11:15:55 +0200510 BUG_ON(this->base + this->size > next->base);
511 i++;
512 continue;
513 }
514
515 this->size += next->size;
Lin Fengc0232ae2013-01-11 14:31:44 -0800516 /* move forward from next + 1, index of which is i + 2 */
517 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
Tejun Heo784656f92011-07-12 11:15:55 +0200518 type->cnt--;
519 }
520}
521
522/**
523 * memblock_insert_region - insert new memblock region
Tang Chen209ff862013-04-29 15:08:41 -0700524 * @type: memblock type to insert into
525 * @idx: index for the insertion point
526 * @base: base address of the new region
527 * @size: size of the new region
528 * @nid: node id of the new region
Tang Chen66a20752014-01-21 15:49:20 -0800529 * @flags: flags of the new region
Tejun Heo784656f92011-07-12 11:15:55 +0200530 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300531 * Insert new memblock region [@base, @base + @size) into @type at @idx.
Alexander Kuleshov412d0002016-08-04 15:31:46 -0700532 * @type must already have extra room to accommodate the new region.
Tejun Heo784656f92011-07-12 11:15:55 +0200533 */
534static void __init_memblock memblock_insert_region(struct memblock_type *type,
535 int idx, phys_addr_t base,
Tang Chen66a20752014-01-21 15:49:20 -0800536 phys_addr_t size,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300537 int nid,
538 enum memblock_flags flags)
Tejun Heo784656f92011-07-12 11:15:55 +0200539{
540 struct memblock_region *rgn = &type->regions[idx];
541
542 BUG_ON(type->cnt >= type->max);
543 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
544 rgn->base = base;
545 rgn->size = size;
Tang Chen66a20752014-01-21 15:49:20 -0800546 rgn->flags = flags;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200547 memblock_set_region_node(rgn, nid);
Tejun Heo784656f92011-07-12 11:15:55 +0200548 type->cnt++;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800549 type->total_size += size;
Tejun Heo784656f92011-07-12 11:15:55 +0200550}
551
552/**
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100553 * memblock_add_range - add new memblock region
Tejun Heo784656f92011-07-12 11:15:55 +0200554 * @type: memblock type to add new region into
555 * @base: base address of the new region
556 * @size: size of the new region
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800557 * @nid: nid of the new region
Tang Chen66a20752014-01-21 15:49:20 -0800558 * @flags: flags of the new region
Tejun Heo784656f92011-07-12 11:15:55 +0200559 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300560 * Add new memblock region [@base, @base + @size) into @type. The new region
Tejun Heo784656f92011-07-12 11:15:55 +0200561 * is allowed to overlap with existing ones - overlaps don't affect already
562 * existing regions. @type is guaranteed to be minimal (all neighbouring
563 * compatible regions are merged) after the addition.
564 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300565 * Return:
Tejun Heo784656f92011-07-12 11:15:55 +0200566 * 0 on success, -errno on failure.
567 */
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100568int __init_memblock memblock_add_range(struct memblock_type *type,
Tang Chen66a20752014-01-21 15:49:20 -0800569 phys_addr_t base, phys_addr_t size,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300570 int nid, enum memblock_flags flags)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000571{
Tejun Heo784656f92011-07-12 11:15:55 +0200572 bool insert = false;
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800573 phys_addr_t obase = base;
574 phys_addr_t end = base + memblock_cap_size(base, &size);
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800575 int idx, nr_new;
576 struct memblock_region *rgn;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000577
Tejun Heob3dc6272012-04-20 08:31:34 -0700578 if (!size)
579 return 0;
580
Tejun Heo784656f92011-07-12 11:15:55 +0200581 /* special case for empty array */
582 if (type->regions[0].size == 0) {
Tejun Heo1440c4e2011-12-08 10:22:08 -0800583 WARN_ON(type->cnt != 1 || type->total_size);
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +1000584 type->regions[0].base = base;
585 type->regions[0].size = size;
Tang Chen66a20752014-01-21 15:49:20 -0800586 type->regions[0].flags = flags;
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800587 memblock_set_region_node(&type->regions[0], nid);
Tejun Heo1440c4e2011-12-08 10:22:08 -0800588 type->total_size = size;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000589 return 0;
590 }
Tejun Heo784656f92011-07-12 11:15:55 +0200591repeat:
592 /*
593 * The following is executed twice. Once with %false @insert and
594 * then with %true. The first counts the number of regions needed
Alexander Kuleshov412d0002016-08-04 15:31:46 -0700595 * to accommodate the new area. The second actually inserts them.
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700596 */
Tejun Heo784656f92011-07-12 11:15:55 +0200597 base = obase;
598 nr_new = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000599
Gioh Kim66e8b432017-11-15 17:33:42 -0800600 for_each_memblock_type(idx, type, rgn) {
Tejun Heo784656f92011-07-12 11:15:55 +0200601 phys_addr_t rbase = rgn->base;
602 phys_addr_t rend = rbase + rgn->size;
603
604 if (rbase >= end)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000605 break;
Tejun Heo784656f92011-07-12 11:15:55 +0200606 if (rend <= base)
607 continue;
608 /*
609 * @rgn overlaps. If it separates the lower part of new
610 * area, insert that portion.
611 */
612 if (rbase > base) {
Wei Yangc0a29492015-09-04 15:47:38 -0700613#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
614 WARN_ON(nid != memblock_get_region_node(rgn));
615#endif
Wei Yang4fcab5f2015-09-08 14:59:53 -0700616 WARN_ON(flags != rgn->flags);
Tejun Heo784656f92011-07-12 11:15:55 +0200617 nr_new++;
618 if (insert)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800619 memblock_insert_region(type, idx++, base,
Tang Chen66a20752014-01-21 15:49:20 -0800620 rbase - base, nid,
621 flags);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000622 }
Tejun Heo784656f92011-07-12 11:15:55 +0200623 /* area below @rend is dealt with, forget about it */
624 base = min(rend, end);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000625 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000626
Tejun Heo784656f92011-07-12 11:15:55 +0200627 /* insert the remaining portion */
628 if (base < end) {
629 nr_new++;
630 if (insert)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800631 memblock_insert_region(type, idx, base, end - base,
Tang Chen66a20752014-01-21 15:49:20 -0800632 nid, flags);
Tejun Heo784656f92011-07-12 11:15:55 +0200633 }
634
nimisoloef3cc4d2016-07-26 15:24:56 -0700635 if (!nr_new)
636 return 0;
637
Tejun Heo784656f92011-07-12 11:15:55 +0200638 /*
639 * If this was the first round, resize array and repeat for actual
640 * insertions; otherwise, merge and return.
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700641 */
Tejun Heo784656f92011-07-12 11:15:55 +0200642 if (!insert) {
643 while (type->cnt + nr_new > type->max)
Greg Pearson48c3b582012-06-20 12:53:05 -0700644 if (memblock_double_array(type, obase, size) < 0)
Tejun Heo784656f92011-07-12 11:15:55 +0200645 return -ENOMEM;
646 insert = true;
647 goto repeat;
648 } else {
649 memblock_merge_regions(type);
650 return 0;
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700651 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000652}
653
Mike Rapoport48a833c2018-06-30 17:55:03 +0300654/**
655 * memblock_add_node - add new memblock region within a NUMA node
656 * @base: base address of the new region
657 * @size: size of the new region
658 * @nid: nid of the new region
659 *
660 * Add new memblock region [@base, @base + @size) to the "memory"
661 * type. See memblock_add_range() description for mode details
662 *
663 * Return:
664 * 0 on success, -errno on failure.
665 */
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800666int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
667 int nid)
668{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100669 return memblock_add_range(&memblock.memory, base, size, nid, 0);
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800670}
671
Mike Rapoport48a833c2018-06-30 17:55:03 +0300672/**
673 * memblock_add - add new memblock region
674 * @base: base address of the new region
675 * @size: size of the new region
676 *
677 * Add new memblock region [@base, @base + @size) to the "memory"
678 * type. See memblock_add_range() description for mode details
679 *
680 * Return:
681 * 0 on success, -errno on failure.
682 */
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700683int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
Alexander Kuleshov6a4055b2015-04-15 16:14:44 -0700684{
Miles Chen5d63f812017-02-22 15:46:42 -0800685 phys_addr_t end = base + size - 1;
686
687 memblock_dbg("memblock_add: [%pa-%pa] %pF\n",
688 &base, &end, (void *)_RET_IP_);
Alexander Kuleshov6a4055b2015-04-15 16:14:44 -0700689
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700690 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000691}
692
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800693/**
694 * memblock_isolate_range - isolate given range into disjoint memblocks
695 * @type: memblock type to isolate range for
696 * @base: base of range to isolate
697 * @size: size of range to isolate
698 * @start_rgn: out parameter for the start of isolated region
699 * @end_rgn: out parameter for the end of isolated region
700 *
701 * Walk @type and ensure that regions don't cross the boundaries defined by
Mike Rapoport47cec442018-06-30 17:55:02 +0300702 * [@base, @base + @size). Crossing regions are split at the boundaries,
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800703 * which may create at most two more regions. The index of the first
704 * region inside the range is returned in *@start_rgn and end in *@end_rgn.
705 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300706 * Return:
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800707 * 0 on success, -errno on failure.
708 */
709static int __init_memblock memblock_isolate_range(struct memblock_type *type,
710 phys_addr_t base, phys_addr_t size,
711 int *start_rgn, int *end_rgn)
712{
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800713 phys_addr_t end = base + memblock_cap_size(base, &size);
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800714 int idx;
715 struct memblock_region *rgn;
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800716
717 *start_rgn = *end_rgn = 0;
718
Tejun Heob3dc6272012-04-20 08:31:34 -0700719 if (!size)
720 return 0;
721
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800722 /* we'll create at most two more regions */
723 while (type->cnt + 2 > type->max)
Greg Pearson48c3b582012-06-20 12:53:05 -0700724 if (memblock_double_array(type, base, size) < 0)
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800725 return -ENOMEM;
726
Gioh Kim66e8b432017-11-15 17:33:42 -0800727 for_each_memblock_type(idx, type, rgn) {
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800728 phys_addr_t rbase = rgn->base;
729 phys_addr_t rend = rbase + rgn->size;
730
731 if (rbase >= end)
732 break;
733 if (rend <= base)
734 continue;
735
736 if (rbase < base) {
737 /*
738 * @rgn intersects from below. Split and continue
739 * to process the next region - the new top half.
740 */
741 rgn->base = base;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800742 rgn->size -= base - rbase;
743 type->total_size -= base - rbase;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800744 memblock_insert_region(type, idx, rbase, base - rbase,
Tang Chen66a20752014-01-21 15:49:20 -0800745 memblock_get_region_node(rgn),
746 rgn->flags);
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800747 } else if (rend > end) {
748 /*
749 * @rgn intersects from above. Split and redo the
750 * current region - the new bottom half.
751 */
752 rgn->base = end;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800753 rgn->size -= end - rbase;
754 type->total_size -= end - rbase;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800755 memblock_insert_region(type, idx--, rbase, end - rbase,
Tang Chen66a20752014-01-21 15:49:20 -0800756 memblock_get_region_node(rgn),
757 rgn->flags);
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800758 } else {
759 /* @rgn is fully contained, record it */
760 if (!*end_rgn)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800761 *start_rgn = idx;
762 *end_rgn = idx + 1;
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800763 }
764 }
765
766 return 0;
767}
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800768
Alexander Kuleshov35bd16a2015-11-05 18:47:00 -0800769static int __init_memblock memblock_remove_range(struct memblock_type *type,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100770 phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000771{
Tejun Heo71936182011-12-08 10:22:07 -0800772 int start_rgn, end_rgn;
773 int i, ret;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000774
Tejun Heo71936182011-12-08 10:22:07 -0800775 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
776 if (ret)
777 return ret;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000778
Tejun Heo71936182011-12-08 10:22:07 -0800779 for (i = end_rgn - 1; i >= start_rgn; i--)
780 memblock_remove_region(type, i);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700781 return 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000782}
783
Tejun Heo581adcb2011-12-08 10:22:06 -0800784int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000785{
Minchan Kim25cf23d2018-06-07 17:07:35 -0700786 phys_addr_t end = base + size - 1;
787
788 memblock_dbg("memblock_remove: [%pa-%pa] %pS\n",
789 &base, &end, (void *)_RET_IP_);
790
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100791 return memblock_remove_range(&memblock.memory, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000792}
793
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100794
Tejun Heo581adcb2011-12-08 10:22:06 -0800795int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000796{
Miles Chen5d63f812017-02-22 15:46:42 -0800797 phys_addr_t end = base + size - 1;
798
799 memblock_dbg(" memblock_free: [%pa-%pa] %pF\n",
800 &base, &end, (void *)_RET_IP_);
Tejun Heo24aa0782011-07-12 11:16:06 +0200801
Catalin Marinas9099dae2016-10-11 13:55:11 -0700802 kmemleak_free_part_phys(base, size);
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100803 return memblock_remove_range(&memblock.reserved, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000804}
805
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700806int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000807{
Miles Chen5d63f812017-02-22 15:46:42 -0800808 phys_addr_t end = base + size - 1;
809
810 memblock_dbg("memblock_reserve: [%pa-%pa] %pF\n",
811 &base, &end, (void *)_RET_IP_);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000812
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700813 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000814}
815
Tejun Heo35fd0802011-07-12 11:15:59 +0200816/**
Mike Rapoport47cec442018-06-30 17:55:02 +0300817 * memblock_setclr_flag - set or clear flag for a memory region
818 * @base: base address of the region
819 * @size: size of the region
820 * @set: set or clear the flag
821 * @flag: the flag to udpate
Tang Chen66b16ed2014-01-21 15:49:23 -0800822 *
Tony Luck4308ce12014-12-12 16:54:59 -0800823 * This function isolates region [@base, @base + @size), and sets/clears flag
Tang Chen66b16ed2014-01-21 15:49:23 -0800824 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300825 * Return: 0 on success, -errno on failure.
Tang Chen66b16ed2014-01-21 15:49:23 -0800826 */
Tony Luck4308ce12014-12-12 16:54:59 -0800827static int __init_memblock memblock_setclr_flag(phys_addr_t base,
828 phys_addr_t size, int set, int flag)
Tang Chen66b16ed2014-01-21 15:49:23 -0800829{
830 struct memblock_type *type = &memblock.memory;
831 int i, ret, start_rgn, end_rgn;
832
833 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
834 if (ret)
835 return ret;
836
837 for (i = start_rgn; i < end_rgn; i++)
Tony Luck4308ce12014-12-12 16:54:59 -0800838 if (set)
839 memblock_set_region_flags(&type->regions[i], flag);
840 else
841 memblock_clear_region_flags(&type->regions[i], flag);
Tang Chen66b16ed2014-01-21 15:49:23 -0800842
843 memblock_merge_regions(type);
844 return 0;
845}
846
847/**
Tony Luck4308ce12014-12-12 16:54:59 -0800848 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
849 * @base: the base phys addr of the region
850 * @size: the size of the region
851 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300852 * Return: 0 on success, -errno on failure.
Tony Luck4308ce12014-12-12 16:54:59 -0800853 */
854int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
855{
856 return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
857}
858
859/**
Tang Chen66b16ed2014-01-21 15:49:23 -0800860 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
861 * @base: the base phys addr of the region
862 * @size: the size of the region
863 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300864 * Return: 0 on success, -errno on failure.
Tang Chen66b16ed2014-01-21 15:49:23 -0800865 */
866int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
867{
Tony Luck4308ce12014-12-12 16:54:59 -0800868 return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
Tang Chen66b16ed2014-01-21 15:49:23 -0800869}
870
871/**
Tony Lucka3f5baf2015-06-24 16:58:12 -0700872 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
873 * @base: the base phys addr of the region
874 * @size: the size of the region
875 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300876 * Return: 0 on success, -errno on failure.
Tony Lucka3f5baf2015-06-24 16:58:12 -0700877 */
878int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
879{
880 system_has_some_mirror = true;
881
882 return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
883}
884
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +0100885/**
886 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
887 * @base: the base phys addr of the region
888 * @size: the size of the region
889 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300890 * Return: 0 on success, -errno on failure.
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +0100891 */
892int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
893{
894 return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
895}
Tony Lucka3f5baf2015-06-24 16:58:12 -0700896
897/**
AKASHI Takahiro4c546b82017-04-03 11:23:54 +0900898 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
899 * @base: the base phys addr of the region
900 * @size: the size of the region
901 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300902 * Return: 0 on success, -errno on failure.
AKASHI Takahiro4c546b82017-04-03 11:23:54 +0900903 */
904int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
905{
906 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
907}
908
909/**
Robin Holt8e7a7f82015-06-30 14:56:41 -0700910 * __next_reserved_mem_region - next function for for_each_reserved_region()
911 * @idx: pointer to u64 loop variable
912 * @out_start: ptr to phys_addr_t for start address of the region, can be %NULL
913 * @out_end: ptr to phys_addr_t for end address of the region, can be %NULL
914 *
915 * Iterate over all reserved memory regions.
916 */
917void __init_memblock __next_reserved_mem_region(u64 *idx,
918 phys_addr_t *out_start,
919 phys_addr_t *out_end)
920{
Alexander Kuleshov567d1172015-09-08 15:03:33 -0700921 struct memblock_type *type = &memblock.reserved;
Robin Holt8e7a7f82015-06-30 14:56:41 -0700922
Richard Leitnercd33a762016-05-20 16:58:33 -0700923 if (*idx < type->cnt) {
Alexander Kuleshov567d1172015-09-08 15:03:33 -0700924 struct memblock_region *r = &type->regions[*idx];
Robin Holt8e7a7f82015-06-30 14:56:41 -0700925 phys_addr_t base = r->base;
926 phys_addr_t size = r->size;
927
928 if (out_start)
929 *out_start = base;
930 if (out_end)
931 *out_end = base + size - 1;
932
933 *idx += 1;
934 return;
935 }
936
937 /* signal end of iteration */
938 *idx = ULLONG_MAX;
939}
940
941/**
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100942 * __next__mem_range - next function for for_each_free_mem_range() etc.
Tejun Heo35fd0802011-07-12 11:15:59 +0200943 * @idx: pointer to u64 loop variable
Grygorii Strashkob1154232014-01-21 15:50:16 -0800944 * @nid: node selector, %NUMA_NO_NODE for all nodes
Tony Luckfc6daaf2015-06-24 16:58:09 -0700945 * @flags: pick from blocks based on memory attributes
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100946 * @type_a: pointer to memblock_type from where the range is taken
947 * @type_b: pointer to memblock_type which excludes memory from being taken
Wanpeng Lidad75572012-06-20 12:53:01 -0700948 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
949 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
950 * @out_nid: ptr to int for nid of the range, can be %NULL
Tejun Heo35fd0802011-07-12 11:15:59 +0200951 *
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100952 * Find the first area from *@idx which matches @nid, fill the out
Tejun Heo35fd0802011-07-12 11:15:59 +0200953 * parameters, and update *@idx for the next iteration. The lower 32bit of
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100954 * *@idx contains index into type_a and the upper 32bit indexes the
955 * areas before each region in type_b. For example, if type_b regions
Tejun Heo35fd0802011-07-12 11:15:59 +0200956 * look like the following,
957 *
958 * 0:[0-16), 1:[32-48), 2:[128-130)
959 *
960 * The upper 32bit indexes the following regions.
961 *
962 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
963 *
964 * As both region arrays are sorted, the function advances the two indices
965 * in lockstep and returns each intersection.
966 */
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300967void __init_memblock __next_mem_range(u64 *idx, int nid,
968 enum memblock_flags flags,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100969 struct memblock_type *type_a,
970 struct memblock_type *type_b,
971 phys_addr_t *out_start,
972 phys_addr_t *out_end, int *out_nid)
Tejun Heo35fd0802011-07-12 11:15:59 +0200973{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100974 int idx_a = *idx & 0xffffffff;
975 int idx_b = *idx >> 32;
Grygorii Strashkob1154232014-01-21 15:50:16 -0800976
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100977 if (WARN_ONCE(nid == MAX_NUMNODES,
978 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
Grygorii Strashko560dca272014-01-21 15:50:55 -0800979 nid = NUMA_NO_NODE;
Tejun Heo35fd0802011-07-12 11:15:59 +0200980
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100981 for (; idx_a < type_a->cnt; idx_a++) {
982 struct memblock_region *m = &type_a->regions[idx_a];
983
Tejun Heo35fd0802011-07-12 11:15:59 +0200984 phys_addr_t m_start = m->base;
985 phys_addr_t m_end = m->base + m->size;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100986 int m_nid = memblock_get_region_node(m);
Tejun Heo35fd0802011-07-12 11:15:59 +0200987
988 /* only memory regions are associated with nodes, check it */
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100989 if (nid != NUMA_NO_NODE && nid != m_nid)
Tejun Heo35fd0802011-07-12 11:15:59 +0200990 continue;
991
Xishi Qiu0a313a92014-09-09 14:50:46 -0700992 /* skip hotpluggable memory regions if needed */
993 if (movable_node_is_enabled() && memblock_is_hotpluggable(m))
994 continue;
995
Tony Lucka3f5baf2015-06-24 16:58:12 -0700996 /* if we want mirror memory skip non-mirror memory regions */
997 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
998 continue;
999
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001000 /* skip nomap memory unless we were asked for it explicitly */
1001 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
1002 continue;
1003
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001004 if (!type_b) {
1005 if (out_start)
1006 *out_start = m_start;
1007 if (out_end)
1008 *out_end = m_end;
1009 if (out_nid)
1010 *out_nid = m_nid;
1011 idx_a++;
1012 *idx = (u32)idx_a | (u64)idx_b << 32;
1013 return;
1014 }
Tejun Heo35fd0802011-07-12 11:15:59 +02001015
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001016 /* scan areas before each reservation */
1017 for (; idx_b < type_b->cnt + 1; idx_b++) {
1018 struct memblock_region *r;
1019 phys_addr_t r_start;
1020 phys_addr_t r_end;
1021
1022 r = &type_b->regions[idx_b];
1023 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1024 r_end = idx_b < type_b->cnt ?
Stefan Agner1c4bc432018-06-07 17:06:15 -07001025 r->base : PHYS_ADDR_MAX;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001026
1027 /*
1028 * if idx_b advanced past idx_a,
1029 * break out to advance idx_a
1030 */
Tejun Heo35fd0802011-07-12 11:15:59 +02001031 if (r_start >= m_end)
1032 break;
1033 /* if the two regions intersect, we're done */
1034 if (m_start < r_end) {
1035 if (out_start)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001036 *out_start =
1037 max(m_start, r_start);
Tejun Heo35fd0802011-07-12 11:15:59 +02001038 if (out_end)
1039 *out_end = min(m_end, r_end);
1040 if (out_nid)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001041 *out_nid = m_nid;
Tejun Heo35fd0802011-07-12 11:15:59 +02001042 /*
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001043 * The region which ends first is
1044 * advanced for the next iteration.
Tejun Heo35fd0802011-07-12 11:15:59 +02001045 */
1046 if (m_end <= r_end)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001047 idx_a++;
Tejun Heo35fd0802011-07-12 11:15:59 +02001048 else
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001049 idx_b++;
1050 *idx = (u32)idx_a | (u64)idx_b << 32;
Tejun Heo35fd0802011-07-12 11:15:59 +02001051 return;
1052 }
1053 }
1054 }
1055
1056 /* signal end of iteration */
1057 *idx = ULLONG_MAX;
1058}
1059
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001060/**
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001061 * __next_mem_range_rev - generic next function for for_each_*_range_rev()
1062 *
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001063 * @idx: pointer to u64 loop variable
Alexander Kuleshovad5ea8c2015-09-08 15:04:22 -07001064 * @nid: node selector, %NUMA_NO_NODE for all nodes
Tony Luckfc6daaf2015-06-24 16:58:09 -07001065 * @flags: pick from blocks based on memory attributes
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001066 * @type_a: pointer to memblock_type from where the range is taken
1067 * @type_b: pointer to memblock_type which excludes memory from being taken
Wanpeng Lidad75572012-06-20 12:53:01 -07001068 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1069 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1070 * @out_nid: ptr to int for nid of the range, can be %NULL
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001071 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001072 * Finds the next range from type_a which is not marked as unsuitable
1073 * in type_b.
1074 *
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001075 * Reverse of __next_mem_range().
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001076 */
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001077void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
1078 enum memblock_flags flags,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001079 struct memblock_type *type_a,
1080 struct memblock_type *type_b,
1081 phys_addr_t *out_start,
1082 phys_addr_t *out_end, int *out_nid)
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001083{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001084 int idx_a = *idx & 0xffffffff;
1085 int idx_b = *idx >> 32;
Grygorii Strashkob1154232014-01-21 15:50:16 -08001086
Grygorii Strashko560dca272014-01-21 15:50:55 -08001087 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1088 nid = NUMA_NO_NODE;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001089
1090 if (*idx == (u64)ULLONG_MAX) {
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001091 idx_a = type_a->cnt - 1;
zijun_hue47608a2016-08-04 15:32:00 -07001092 if (type_b != NULL)
1093 idx_b = type_b->cnt;
1094 else
1095 idx_b = 0;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001096 }
1097
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001098 for (; idx_a >= 0; idx_a--) {
1099 struct memblock_region *m = &type_a->regions[idx_a];
1100
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001101 phys_addr_t m_start = m->base;
1102 phys_addr_t m_end = m->base + m->size;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001103 int m_nid = memblock_get_region_node(m);
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001104
1105 /* only memory regions are associated with nodes, check it */
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001106 if (nid != NUMA_NO_NODE && nid != m_nid)
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001107 continue;
1108
Tang Chen55ac5902014-01-21 15:49:35 -08001109 /* skip hotpluggable memory regions if needed */
1110 if (movable_node_is_enabled() && memblock_is_hotpluggable(m))
1111 continue;
1112
Tony Lucka3f5baf2015-06-24 16:58:12 -07001113 /* if we want mirror memory skip non-mirror memory regions */
1114 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
1115 continue;
1116
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001117 /* skip nomap memory unless we were asked for it explicitly */
1118 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
1119 continue;
1120
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001121 if (!type_b) {
1122 if (out_start)
1123 *out_start = m_start;
1124 if (out_end)
1125 *out_end = m_end;
1126 if (out_nid)
1127 *out_nid = m_nid;
zijun_hufb399b42016-07-28 15:48:56 -07001128 idx_a--;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001129 *idx = (u32)idx_a | (u64)idx_b << 32;
1130 return;
1131 }
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001132
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001133 /* scan areas before each reservation */
1134 for (; idx_b >= 0; idx_b--) {
1135 struct memblock_region *r;
1136 phys_addr_t r_start;
1137 phys_addr_t r_end;
1138
1139 r = &type_b->regions[idx_b];
1140 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1141 r_end = idx_b < type_b->cnt ?
Stefan Agner1c4bc432018-06-07 17:06:15 -07001142 r->base : PHYS_ADDR_MAX;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001143 /*
1144 * if idx_b advanced past idx_a,
1145 * break out to advance idx_a
1146 */
1147
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001148 if (r_end <= m_start)
1149 break;
1150 /* if the two regions intersect, we're done */
1151 if (m_end > r_start) {
1152 if (out_start)
1153 *out_start = max(m_start, r_start);
1154 if (out_end)
1155 *out_end = min(m_end, r_end);
1156 if (out_nid)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001157 *out_nid = m_nid;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001158 if (m_start >= r_start)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001159 idx_a--;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001160 else
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001161 idx_b--;
1162 *idx = (u32)idx_a | (u64)idx_b << 32;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001163 return;
1164 }
1165 }
1166 }
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001167 /* signal end of iteration */
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001168 *idx = ULLONG_MAX;
1169}
1170
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001171#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1172/*
1173 * Common iterator interface used to define for_each_mem_range().
1174 */
1175void __init_memblock __next_mem_pfn_range(int *idx, int nid,
1176 unsigned long *out_start_pfn,
1177 unsigned long *out_end_pfn, int *out_nid)
1178{
1179 struct memblock_type *type = &memblock.memory;
1180 struct memblock_region *r;
1181
1182 while (++*idx < type->cnt) {
1183 r = &type->regions[*idx];
1184
1185 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
1186 continue;
1187 if (nid == MAX_NUMNODES || nid == r->nid)
1188 break;
1189 }
1190 if (*idx >= type->cnt) {
1191 *idx = -1;
1192 return;
1193 }
1194
1195 if (out_start_pfn)
1196 *out_start_pfn = PFN_UP(r->base);
1197 if (out_end_pfn)
1198 *out_end_pfn = PFN_DOWN(r->base + r->size);
1199 if (out_nid)
1200 *out_nid = r->nid;
1201}
1202
1203/**
1204 * memblock_set_node - set node ID on memblock regions
1205 * @base: base of area to set node ID for
1206 * @size: size of area to set node ID for
Tang Chene7e8de52014-01-21 15:49:26 -08001207 * @type: memblock type to set node ID for
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001208 * @nid: node ID to set
1209 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001210 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001211 * Regions which cross the area boundaries are split as necessary.
1212 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001213 * Return:
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001214 * 0 on success, -errno on failure.
1215 */
1216int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
Tang Chene7e8de52014-01-21 15:49:26 -08001217 struct memblock_type *type, int nid)
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001218{
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001219 int start_rgn, end_rgn;
1220 int i, ret;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001221
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001222 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
1223 if (ret)
1224 return ret;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001225
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001226 for (i = start_rgn; i < end_rgn; i++)
Wanpeng Lie9d24ad2012-10-08 16:32:21 -07001227 memblock_set_region_node(&type->regions[i], nid);
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001228
1229 memblock_merge_regions(type);
1230 return 0;
1231}
1232#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1233
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001234static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
1235 phys_addr_t align, phys_addr_t start,
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001236 phys_addr_t end, int nid,
1237 enum memblock_flags flags)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001238{
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001239 phys_addr_t found;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001240
Grygorii Strashko79f40fa2014-01-21 15:50:12 -08001241 if (!align)
1242 align = SMP_CACHE_BYTES;
Vineet Gupta94f3d3a2013-04-29 15:06:15 -07001243
Tony Luckfc6daaf2015-06-24 16:58:09 -07001244 found = memblock_find_in_range_node(size, align, start, end, nid,
1245 flags);
Catalin Marinasaedf95e2014-06-06 14:38:20 -07001246 if (found && !memblock_reserve(found, size)) {
1247 /*
1248 * The min_count is set to 0 so that memblock allocations are
1249 * never reported as leaks.
1250 */
Catalin Marinas9099dae2016-10-11 13:55:11 -07001251 kmemleak_alloc_phys(found, size, 0, 0);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001252 return found;
Catalin Marinasaedf95e2014-06-06 14:38:20 -07001253 }
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001254 return 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001255}
1256
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001257phys_addr_t __init memblock_alloc_range(phys_addr_t size, phys_addr_t align,
Tony Luckfc6daaf2015-06-24 16:58:09 -07001258 phys_addr_t start, phys_addr_t end,
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001259 enum memblock_flags flags)
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001260{
Tony Luckfc6daaf2015-06-24 16:58:09 -07001261 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1262 flags);
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001263}
1264
Nicholas Pigginb575454f2018-02-14 01:08:15 +10001265phys_addr_t __init memblock_alloc_base_nid(phys_addr_t size,
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001266 phys_addr_t align, phys_addr_t max_addr,
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001267 int nid, enum memblock_flags flags)
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001268{
Tony Luckfc6daaf2015-06-24 16:58:09 -07001269 return memblock_alloc_range_nid(size, align, 0, max_addr, nid, flags);
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001270}
1271
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001272phys_addr_t __init memblock_alloc_nid(phys_addr_t size, phys_addr_t align, int nid)
1273{
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001274 enum memblock_flags flags = choose_memblock_flags();
Tony Lucka3f5baf2015-06-24 16:58:12 -07001275 phys_addr_t ret;
1276
1277again:
1278 ret = memblock_alloc_base_nid(size, align, MEMBLOCK_ALLOC_ACCESSIBLE,
1279 nid, flags);
1280
1281 if (!ret && (flags & MEMBLOCK_MIRROR)) {
1282 flags &= ~MEMBLOCK_MIRROR;
1283 goto again;
1284 }
1285 return ret;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001286}
1287
1288phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr)
1289{
Tony Luckfc6daaf2015-06-24 16:58:09 -07001290 return memblock_alloc_base_nid(size, align, max_addr, NUMA_NO_NODE,
1291 MEMBLOCK_NONE);
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001292}
1293
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001294phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001295{
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001296 phys_addr_t alloc;
1297
1298 alloc = __memblock_alloc_base(size, align, max_addr);
1299
1300 if (alloc == 0)
Miles Chen5d63f812017-02-22 15:46:42 -08001301 panic("ERROR: Failed to allocate %pa bytes below %pa.\n",
1302 &size, &max_addr);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001303
1304 return alloc;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001305}
1306
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001307phys_addr_t __init memblock_alloc(phys_addr_t size, phys_addr_t align)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001308{
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001309 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001310}
1311
Benjamin Herrenschmidt9d1e2492010-07-06 15:39:17 -07001312phys_addr_t __init memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
1313{
1314 phys_addr_t res = memblock_alloc_nid(size, align, nid);
1315
1316 if (res)
1317 return res;
Tejun Heo15fb0972011-07-12 09:58:07 +02001318 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001319}
1320
Mathieu Malaterre19373672018-07-20 17:53:31 -07001321#if defined(CONFIG_NO_BOOTMEM)
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001322/**
1323 * memblock_virt_alloc_internal - allocate boot memory block
1324 * @size: size of memory block to be allocated in bytes
1325 * @align: alignment of the region and block's size
1326 * @min_addr: the lower bound of the memory region to allocate (phys address)
1327 * @max_addr: the upper bound of the memory region to allocate (phys address)
1328 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1329 *
1330 * The @min_addr limit is dropped if it can not be satisfied and the allocation
1331 * will fall back to memory below @min_addr. Also, allocation may fall back
1332 * to any node in the system if the specified node can not
1333 * hold the requested memory.
1334 *
1335 * The allocation is performed from memory region limited by
1336 * memblock.current_limit if @max_addr == %BOOTMEM_ALLOC_ACCESSIBLE.
1337 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001338 * The memory block is aligned on %SMP_CACHE_BYTES if @align == 0.
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001339 *
1340 * The phys address of allocated boot memory block is converted to virtual and
1341 * allocated memory is reset to 0.
1342 *
1343 * In addition, function sets the min_count to 0 using kmemleak_alloc for
1344 * allocated boot memory block, so that it is never reported as leaks.
1345 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001346 * Return:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001347 * Virtual address of allocated memory block on success, NULL on failure.
1348 */
1349static void * __init memblock_virt_alloc_internal(
1350 phys_addr_t size, phys_addr_t align,
1351 phys_addr_t min_addr, phys_addr_t max_addr,
1352 int nid)
1353{
1354 phys_addr_t alloc;
1355 void *ptr;
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001356 enum memblock_flags flags = choose_memblock_flags();
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001357
Grygorii Strashko560dca272014-01-21 15:50:55 -08001358 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1359 nid = NUMA_NO_NODE;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001360
1361 /*
1362 * Detect any accidental use of these APIs after slab is ready, as at
1363 * this moment memblock may be deinitialized already and its
1364 * internal data may be destroyed (after execution of free_all_bootmem)
1365 */
1366 if (WARN_ON_ONCE(slab_is_available()))
1367 return kzalloc_node(size, GFP_NOWAIT, nid);
1368
1369 if (!align)
1370 align = SMP_CACHE_BYTES;
1371
Yinghai Luf544e142014-01-29 14:05:52 -08001372 if (max_addr > memblock.current_limit)
1373 max_addr = memblock.current_limit;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001374again:
1375 alloc = memblock_find_in_range_node(size, align, min_addr, max_addr,
Tony Lucka3f5baf2015-06-24 16:58:12 -07001376 nid, flags);
Wei Yang7d41c032017-02-22 15:45:07 -08001377 if (alloc && !memblock_reserve(alloc, size))
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001378 goto done;
1379
1380 if (nid != NUMA_NO_NODE) {
1381 alloc = memblock_find_in_range_node(size, align, min_addr,
Tony Luckfc6daaf2015-06-24 16:58:09 -07001382 max_addr, NUMA_NO_NODE,
Tony Lucka3f5baf2015-06-24 16:58:12 -07001383 flags);
Wei Yang7d41c032017-02-22 15:45:07 -08001384 if (alloc && !memblock_reserve(alloc, size))
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001385 goto done;
1386 }
1387
1388 if (min_addr) {
1389 min_addr = 0;
1390 goto again;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001391 }
1392
Tony Lucka3f5baf2015-06-24 16:58:12 -07001393 if (flags & MEMBLOCK_MIRROR) {
1394 flags &= ~MEMBLOCK_MIRROR;
1395 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
1396 &size);
1397 goto again;
1398 }
1399
1400 return NULL;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001401done:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001402 ptr = phys_to_virt(alloc);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001403
1404 /*
1405 * The min_count is set to 0 so that bootmem allocated blocks
1406 * are never reported as leaks. This is because many of these blocks
1407 * are only referred via the physical address which is not
1408 * looked up by kmemleak.
1409 */
1410 kmemleak_alloc(ptr, size, 0, 0);
1411
1412 return ptr;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001413}
1414
1415/**
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001416 * memblock_virt_alloc_try_nid_raw - allocate boot memory block without zeroing
1417 * memory and without panicking
1418 * @size: size of memory block to be allocated in bytes
1419 * @align: alignment of the region and block's size
1420 * @min_addr: the lower bound of the memory region from where the allocation
1421 * is preferred (phys address)
1422 * @max_addr: the upper bound of the memory region from where the allocation
1423 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to
1424 * allocate only from memory limited by memblock.current_limit value
1425 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1426 *
1427 * Public function, provides additional debug information (including caller
1428 * info), if enabled. Does not zero allocated memory, does not panic if request
1429 * cannot be satisfied.
1430 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001431 * Return:
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001432 * Virtual address of allocated memory block on success, NULL on failure.
1433 */
1434void * __init memblock_virt_alloc_try_nid_raw(
1435 phys_addr_t size, phys_addr_t align,
1436 phys_addr_t min_addr, phys_addr_t max_addr,
1437 int nid)
1438{
1439 void *ptr;
1440
Mike Rapoporta36aab82018-08-17 15:47:17 -07001441 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pF\n",
1442 __func__, (u64)size, (u64)align, nid, &min_addr,
1443 &max_addr, (void *)_RET_IP_);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001444
1445 ptr = memblock_virt_alloc_internal(size, align,
1446 min_addr, max_addr, nid);
1447#ifdef CONFIG_DEBUG_VM
1448 if (ptr && size > 0)
Pavel Tatashinf165b372018-04-05 16:22:47 -07001449 memset(ptr, PAGE_POISON_PATTERN, size);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001450#endif
1451 return ptr;
1452}
1453
1454/**
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001455 * memblock_virt_alloc_try_nid_nopanic - allocate boot memory block
1456 * @size: size of memory block to be allocated in bytes
1457 * @align: alignment of the region and block's size
1458 * @min_addr: the lower bound of the memory region from where the allocation
1459 * is preferred (phys address)
1460 * @max_addr: the upper bound of the memory region from where the allocation
1461 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to
1462 * allocate only from memory limited by memblock.current_limit value
1463 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1464 *
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001465 * Public function, provides additional debug information (including caller
1466 * info), if enabled. This function zeroes the allocated memory.
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001467 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001468 * Return:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001469 * Virtual address of allocated memory block on success, NULL on failure.
1470 */
1471void * __init memblock_virt_alloc_try_nid_nopanic(
1472 phys_addr_t size, phys_addr_t align,
1473 phys_addr_t min_addr, phys_addr_t max_addr,
1474 int nid)
1475{
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001476 void *ptr;
1477
Mike Rapoporta36aab82018-08-17 15:47:17 -07001478 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pF\n",
1479 __func__, (u64)size, (u64)align, nid, &min_addr,
1480 &max_addr, (void *)_RET_IP_);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001481
1482 ptr = memblock_virt_alloc_internal(size, align,
1483 min_addr, max_addr, nid);
1484 if (ptr)
1485 memset(ptr, 0, size);
1486 return ptr;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001487}
1488
1489/**
1490 * memblock_virt_alloc_try_nid - allocate boot memory block with panicking
1491 * @size: size of memory block to be allocated in bytes
1492 * @align: alignment of the region and block's size
1493 * @min_addr: the lower bound of the memory region from where the allocation
1494 * is preferred (phys address)
1495 * @max_addr: the upper bound of the memory region from where the allocation
1496 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to
1497 * allocate only from memory limited by memblock.current_limit value
1498 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1499 *
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001500 * Public panicking version of memblock_virt_alloc_try_nid_nopanic()
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001501 * which provides debug information (including caller info), if enabled,
1502 * and panics if the request can not be satisfied.
1503 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001504 * Return:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001505 * Virtual address of allocated memory block on success, NULL on failure.
1506 */
1507void * __init memblock_virt_alloc_try_nid(
1508 phys_addr_t size, phys_addr_t align,
1509 phys_addr_t min_addr, phys_addr_t max_addr,
1510 int nid)
1511{
1512 void *ptr;
1513
Mike Rapoporta36aab82018-08-17 15:47:17 -07001514 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pF\n",
1515 __func__, (u64)size, (u64)align, nid, &min_addr,
1516 &max_addr, (void *)_RET_IP_);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001517 ptr = memblock_virt_alloc_internal(size, align,
1518 min_addr, max_addr, nid);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001519 if (ptr) {
1520 memset(ptr, 0, size);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001521 return ptr;
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001522 }
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001523
Mike Rapoporta36aab82018-08-17 15:47:17 -07001524 panic("%s: Failed to allocate %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa\n",
1525 __func__, (u64)size, (u64)align, nid, &min_addr, &max_addr);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001526 return NULL;
1527}
Mathieu Malaterre19373672018-07-20 17:53:31 -07001528#endif
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001529
1530/**
1531 * __memblock_free_early - free boot memory block
1532 * @base: phys starting address of the boot memory block
1533 * @size: size of the boot memory block in bytes
1534 *
1535 * Free boot memory block previously allocated by memblock_virt_alloc_xx() API.
1536 * The freeing memory will not be released to the buddy allocator.
1537 */
1538void __init __memblock_free_early(phys_addr_t base, phys_addr_t size)
1539{
Mike Rapoporta36aab82018-08-17 15:47:17 -07001540 phys_addr_t end = base + size - 1;
1541
1542 memblock_dbg("%s: [%pa-%pa] %pF\n",
1543 __func__, &base, &end, (void *)_RET_IP_);
Catalin Marinas9099dae2016-10-11 13:55:11 -07001544 kmemleak_free_part_phys(base, size);
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001545 memblock_remove_range(&memblock.reserved, base, size);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001546}
1547
Mike Rapoport48a833c2018-06-30 17:55:03 +03001548/**
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001549 * __memblock_free_late - free bootmem block pages directly to buddy allocator
Mike Rapoport48a833c2018-06-30 17:55:03 +03001550 * @base: phys starting address of the boot memory block
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001551 * @size: size of the boot memory block in bytes
1552 *
1553 * This is only useful when the bootmem allocator has already been torn
1554 * down, but we are still initializing the system. Pages are released directly
1555 * to the buddy allocator, no bootmem metadata is updated because it is gone.
1556 */
1557void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)
1558{
Mike Rapoporta36aab82018-08-17 15:47:17 -07001559 phys_addr_t cursor, end;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001560
Mike Rapoporta36aab82018-08-17 15:47:17 -07001561 end = base + size - 1;
1562 memblock_dbg("%s: [%pa-%pa] %pF\n",
1563 __func__, &base, &end, (void *)_RET_IP_);
Catalin Marinas9099dae2016-10-11 13:55:11 -07001564 kmemleak_free_part_phys(base, size);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001565 cursor = PFN_UP(base);
1566 end = PFN_DOWN(base + size);
1567
1568 for (; cursor < end; cursor++) {
Mel Gormand70ddd72015-06-30 14:56:52 -07001569 __free_pages_bootmem(pfn_to_page(cursor), cursor, 0);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001570 totalram_pages++;
1571 }
1572}
Benjamin Herrenschmidt9d1e2492010-07-06 15:39:17 -07001573
1574/*
1575 * Remaining API functions
1576 */
1577
David Gibson1f1ffb8a2016-02-05 15:36:19 -08001578phys_addr_t __init_memblock memblock_phys_mem_size(void)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001579{
Tejun Heo1440c4e2011-12-08 10:22:08 -08001580 return memblock.memory.total_size;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001581}
1582
Srikar Dronamraju8907de52016-10-07 16:59:18 -07001583phys_addr_t __init_memblock memblock_reserved_size(void)
1584{
1585 return memblock.reserved.total_size;
1586}
1587
Yinghai Lu595ad9a2013-01-24 12:20:09 -08001588phys_addr_t __init memblock_mem_size(unsigned long limit_pfn)
1589{
1590 unsigned long pages = 0;
1591 struct memblock_region *r;
1592 unsigned long start_pfn, end_pfn;
1593
1594 for_each_memblock(memory, r) {
1595 start_pfn = memblock_region_memory_base_pfn(r);
1596 end_pfn = memblock_region_memory_end_pfn(r);
1597 start_pfn = min_t(unsigned long, start_pfn, limit_pfn);
1598 end_pfn = min_t(unsigned long, end_pfn, limit_pfn);
1599 pages += end_pfn - start_pfn;
1600 }
1601
Fabian Frederick16763232014-04-07 15:37:53 -07001602 return PFN_PHYS(pages);
Yinghai Lu595ad9a2013-01-24 12:20:09 -08001603}
1604
Sam Ravnborg0a93ebe2011-10-31 17:08:16 -07001605/* lowest address */
1606phys_addr_t __init_memblock memblock_start_of_DRAM(void)
1607{
1608 return memblock.memory.regions[0].base;
1609}
1610
Yinghai Lu10d06432010-07-28 15:43:02 +10001611phys_addr_t __init_memblock memblock_end_of_DRAM(void)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001612{
1613 int idx = memblock.memory.cnt - 1;
1614
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +10001615 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001616}
1617
Dennis Chena571d4e2016-07-28 15:48:26 -07001618static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001619{
Stefan Agner1c4bc432018-06-07 17:06:15 -07001620 phys_addr_t max_addr = PHYS_ADDR_MAX;
Emil Medve136199f2014-04-07 15:37:52 -07001621 struct memblock_region *r;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001622
Dennis Chena571d4e2016-07-28 15:48:26 -07001623 /*
1624 * translate the memory @limit size into the max address within one of
1625 * the memory memblock regions, if the @limit exceeds the total size
Stefan Agner1c4bc432018-06-07 17:06:15 -07001626 * of those regions, max_addr will keep original value PHYS_ADDR_MAX
Dennis Chena571d4e2016-07-28 15:48:26 -07001627 */
Emil Medve136199f2014-04-07 15:37:52 -07001628 for_each_memblock(memory, r) {
Tejun Heoc0ce8fe2011-12-08 10:22:07 -08001629 if (limit <= r->size) {
1630 max_addr = r->base + limit;
1631 break;
1632 }
1633 limit -= r->size;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001634 }
1635
Dennis Chena571d4e2016-07-28 15:48:26 -07001636 return max_addr;
1637}
1638
1639void __init memblock_enforce_memory_limit(phys_addr_t limit)
1640{
Stefan Agner1c4bc432018-06-07 17:06:15 -07001641 phys_addr_t max_addr = PHYS_ADDR_MAX;
Dennis Chena571d4e2016-07-28 15:48:26 -07001642
1643 if (!limit)
1644 return;
1645
1646 max_addr = __find_max_addr(limit);
1647
1648 /* @limit exceeds the total size of the memory, do nothing */
Stefan Agner1c4bc432018-06-07 17:06:15 -07001649 if (max_addr == PHYS_ADDR_MAX)
Dennis Chena571d4e2016-07-28 15:48:26 -07001650 return;
1651
Tejun Heoc0ce8fe2011-12-08 10:22:07 -08001652 /* truncate both memory and reserved regions */
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001653 memblock_remove_range(&memblock.memory, max_addr,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001654 PHYS_ADDR_MAX);
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001655 memblock_remove_range(&memblock.reserved, max_addr,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001656 PHYS_ADDR_MAX);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001657}
1658
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001659void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
1660{
1661 int start_rgn, end_rgn;
1662 int i, ret;
1663
1664 if (!size)
1665 return;
1666
1667 ret = memblock_isolate_range(&memblock.memory, base, size,
1668 &start_rgn, &end_rgn);
1669 if (ret)
1670 return;
1671
1672 /* remove all the MAP regions */
1673 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
1674 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1675 memblock_remove_region(&memblock.memory, i);
1676
1677 for (i = start_rgn - 1; i >= 0; i--)
1678 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1679 memblock_remove_region(&memblock.memory, i);
1680
1681 /* truncate the reserved regions */
1682 memblock_remove_range(&memblock.reserved, 0, base);
1683 memblock_remove_range(&memblock.reserved,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001684 base + size, PHYS_ADDR_MAX);
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001685}
1686
Dennis Chena571d4e2016-07-28 15:48:26 -07001687void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1688{
Dennis Chena571d4e2016-07-28 15:48:26 -07001689 phys_addr_t max_addr;
Dennis Chena571d4e2016-07-28 15:48:26 -07001690
1691 if (!limit)
1692 return;
1693
1694 max_addr = __find_max_addr(limit);
1695
1696 /* @limit exceeds the total size of the memory, do nothing */
Stefan Agner1c4bc432018-06-07 17:06:15 -07001697 if (max_addr == PHYS_ADDR_MAX)
Dennis Chena571d4e2016-07-28 15:48:26 -07001698 return;
1699
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001700 memblock_cap_memory_range(0, max_addr);
Dennis Chena571d4e2016-07-28 15:48:26 -07001701}
1702
Yinghai Lucd794812010-10-11 12:34:09 -07001703static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001704{
1705 unsigned int left = 0, right = type->cnt;
1706
1707 do {
1708 unsigned int mid = (right + left) / 2;
1709
1710 if (addr < type->regions[mid].base)
1711 right = mid;
1712 else if (addr >= (type->regions[mid].base +
1713 type->regions[mid].size))
1714 left = mid + 1;
1715 else
1716 return mid;
1717 } while (left < right);
1718 return -1;
1719}
1720
Yaowei Baib4ad0c72016-01-14 15:18:54 -08001721bool __init memblock_is_reserved(phys_addr_t addr)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001722{
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001723 return memblock_search(&memblock.reserved, addr) != -1;
1724}
Yinghai Lu95f72d12010-07-12 14:36:09 +10001725
Yaowei Baib4ad0c72016-01-14 15:18:54 -08001726bool __init_memblock memblock_is_memory(phys_addr_t addr)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001727{
1728 return memblock_search(&memblock.memory, addr) != -1;
1729}
1730
Yaowei Bai937f0c22018-02-06 15:41:18 -08001731bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001732{
1733 int i = memblock_search(&memblock.memory, addr);
1734
1735 if (i == -1)
1736 return false;
1737 return !memblock_is_nomap(&memblock.memory.regions[i]);
1738}
1739
Yinghai Lue76b63f2013-09-11 14:22:17 -07001740#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1741int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
1742 unsigned long *start_pfn, unsigned long *end_pfn)
1743{
1744 struct memblock_type *type = &memblock.memory;
Fabian Frederick16763232014-04-07 15:37:53 -07001745 int mid = memblock_search(type, PFN_PHYS(pfn));
Yinghai Lue76b63f2013-09-11 14:22:17 -07001746
1747 if (mid == -1)
1748 return -1;
1749
Fabian Frederickf7e2f7e2014-06-04 16:07:51 -07001750 *start_pfn = PFN_DOWN(type->regions[mid].base);
1751 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
Yinghai Lue76b63f2013-09-11 14:22:17 -07001752
1753 return type->regions[mid].nid;
1754}
1755#endif
1756
Stephen Boydeab30942012-05-24 00:45:21 -07001757/**
1758 * memblock_is_region_memory - check if a region is a subset of memory
1759 * @base: base of region to check
1760 * @size: size of region to check
1761 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001762 * Check if the region [@base, @base + @size) is a subset of a memory block.
Stephen Boydeab30942012-05-24 00:45:21 -07001763 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001764 * Return:
Stephen Boydeab30942012-05-24 00:45:21 -07001765 * 0 if false, non-zero if true
1766 */
Yaowei Bai937f0c22018-02-06 15:41:18 -08001767bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001768{
Tomi Valkeinenabb65272011-01-20 14:44:20 -08001769 int idx = memblock_search(&memblock.memory, base);
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001770 phys_addr_t end = base + memblock_cap_size(base, &size);
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001771
1772 if (idx == -1)
Yaowei Bai937f0c22018-02-06 15:41:18 -08001773 return false;
Wei Yangef415ef2017-02-22 15:45:04 -08001774 return (memblock.memory.regions[idx].base +
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001775 memblock.memory.regions[idx].size) >= end;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001776}
1777
Stephen Boydeab30942012-05-24 00:45:21 -07001778/**
1779 * memblock_is_region_reserved - check if a region intersects reserved memory
1780 * @base: base of region to check
1781 * @size: size of region to check
1782 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001783 * Check if the region [@base, @base + @size) intersects a reserved
1784 * memory block.
Stephen Boydeab30942012-05-24 00:45:21 -07001785 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001786 * Return:
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001787 * True if they intersect, false if not.
Stephen Boydeab30942012-05-24 00:45:21 -07001788 */
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001789bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001790{
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001791 memblock_cap_size(base, &size);
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001792 return memblock_overlaps_region(&memblock.reserved, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001793}
1794
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001795void __init_memblock memblock_trim_memory(phys_addr_t align)
1796{
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001797 phys_addr_t start, end, orig_start, orig_end;
Emil Medve136199f2014-04-07 15:37:52 -07001798 struct memblock_region *r;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001799
Emil Medve136199f2014-04-07 15:37:52 -07001800 for_each_memblock(memory, r) {
1801 orig_start = r->base;
1802 orig_end = r->base + r->size;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001803 start = round_up(orig_start, align);
1804 end = round_down(orig_end, align);
1805
1806 if (start == orig_start && end == orig_end)
1807 continue;
1808
1809 if (start < end) {
Emil Medve136199f2014-04-07 15:37:52 -07001810 r->base = start;
1811 r->size = end - start;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001812 } else {
Emil Medve136199f2014-04-07 15:37:52 -07001813 memblock_remove_region(&memblock.memory,
1814 r - memblock.memory.regions);
1815 r--;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001816 }
1817 }
1818}
Benjamin Herrenschmidte63075a2010-07-06 15:39:01 -07001819
Yinghai Lu3661ca62010-09-15 13:05:29 -07001820void __init_memblock memblock_set_current_limit(phys_addr_t limit)
Benjamin Herrenschmidte63075a2010-07-06 15:39:01 -07001821{
1822 memblock.current_limit = limit;
1823}
1824
Laura Abbottfec51012014-02-27 01:23:43 +01001825phys_addr_t __init_memblock memblock_get_current_limit(void)
1826{
1827 return memblock.current_limit;
1828}
1829
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001830static void __init_memblock memblock_dump(struct memblock_type *type)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001831{
Miles Chen5d63f812017-02-22 15:46:42 -08001832 phys_addr_t base, end, size;
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001833 enum memblock_flags flags;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -08001834 int idx;
1835 struct memblock_region *rgn;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001836
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001837 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001838
Gioh Kim66e8b432017-11-15 17:33:42 -08001839 for_each_memblock_type(idx, type, rgn) {
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001840 char nid_buf[32] = "";
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001841
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001842 base = rgn->base;
1843 size = rgn->size;
Miles Chen5d63f812017-02-22 15:46:42 -08001844 end = base + size - 1;
Tang Chen66a20752014-01-21 15:49:20 -08001845 flags = rgn->flags;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001846#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1847 if (memblock_get_region_node(rgn) != MAX_NUMNODES)
1848 snprintf(nid_buf, sizeof(nid_buf), " on node %d",
1849 memblock_get_region_node(rgn));
1850#endif
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001851 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001852 type->name, idx, &base, &end, &size, nid_buf, flags);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001853 }
1854}
1855
Tejun Heo4ff7b822011-12-08 10:22:06 -08001856void __init_memblock __memblock_dump_all(void)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001857{
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001858 pr_info("MEMBLOCK configuration:\n");
Miles Chen5d63f812017-02-22 15:46:42 -08001859 pr_info(" memory size = %pa reserved size = %pa\n",
1860 &memblock.memory.total_size,
1861 &memblock.reserved.total_size);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001862
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001863 memblock_dump(&memblock.memory);
1864 memblock_dump(&memblock.reserved);
Heiko Carstens409efd42017-02-24 14:55:56 -08001865#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001866 memblock_dump(&memblock.physmem);
Heiko Carstens409efd42017-02-24 14:55:56 -08001867#endif
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001868}
1869
Tejun Heo1aadc052011-12-08 10:22:08 -08001870void __init memblock_allow_resize(void)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001871{
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -07001872 memblock_can_resize = 1;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001873}
1874
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001875static int __init early_memblock(char *p)
1876{
1877 if (p && strstr(p, "debug"))
1878 memblock_debug = 1;
1879 return 0;
1880}
1881early_param("memblock", early_memblock);
1882
Tejun Heoc378ddd2011-07-14 11:46:03 +02001883#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_ARCH_DISCARD_MEMBLOCK)
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07001884
1885static int memblock_debug_show(struct seq_file *m, void *private)
1886{
1887 struct memblock_type *type = m->private;
1888 struct memblock_region *reg;
1889 int i;
Miles Chen5d63f812017-02-22 15:46:42 -08001890 phys_addr_t end;
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07001891
1892 for (i = 0; i < type->cnt; i++) {
1893 reg = &type->regions[i];
Miles Chen5d63f812017-02-22 15:46:42 -08001894 end = reg->base + reg->size - 1;
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07001895
Miles Chen5d63f812017-02-22 15:46:42 -08001896 seq_printf(m, "%4d: ", i);
1897 seq_printf(m, "%pa..%pa\n", &reg->base, &end);
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07001898 }
1899 return 0;
1900}
Andy Shevchenko5ad35092018-04-05 16:23:16 -07001901DEFINE_SHOW_ATTRIBUTE(memblock_debug);
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07001902
1903static int __init memblock_init_debugfs(void)
1904{
1905 struct dentry *root = debugfs_create_dir("memblock", NULL);
1906 if (!root)
1907 return -ENXIO;
Joe Perches0825a6f2018-06-14 15:27:58 -07001908 debugfs_create_file("memory", 0444, root,
1909 &memblock.memory, &memblock_debug_fops);
1910 debugfs_create_file("reserved", 0444, root,
1911 &memblock.reserved, &memblock_debug_fops);
Philipp Hachtmann70210ed2014-01-29 18:16:01 +01001912#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
Joe Perches0825a6f2018-06-14 15:27:58 -07001913 debugfs_create_file("physmem", 0444, root,
1914 &memblock.physmem, &memblock_debug_fops);
Philipp Hachtmann70210ed2014-01-29 18:16:01 +01001915#endif
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07001916
1917 return 0;
1918}
1919__initcall(memblock_init_debugfs);
1920
1921#endif /* CONFIG_DEBUG_FS */