blob: ba37d62b684a34f05b8623361aa8ce7980a8d4e0 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Johannes Weiner57cfc292008-07-23 21:28:00 -07002 * bootmem - A boot-time physical memory allocator and configurator
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 1999 Ingo Molnar
Johannes Weiner57cfc292008-07-23 21:28:00 -07005 * 1999 Kanoj Sarcar, SGI
6 * 2008 Johannes Weiner
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Johannes Weiner57cfc292008-07-23 21:28:00 -07008 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070012#include <linux/pfn.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/module.h>
Catalin Marinasec3a3542009-07-07 10:33:01 +010015#include <linux/kmemleak.h>
Yinghai Lu08677212010-02-10 01:20:20 -080016#include <linux/range.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070017
18#include <asm/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <asm/io.h>
Heiko Carstensdfd54cb2006-09-25 23:31:33 -070020#include <asm/processor.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070021
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include "internal.h"
23
Linus Torvalds1da177e2005-04-16 15:20:36 -070024unsigned long max_low_pfn;
25unsigned long min_low_pfn;
26unsigned long max_pfn;
27
Vivek Goyal92aa63a2005-06-25 14:58:18 -070028#ifdef CONFIG_CRASH_DUMP
29/*
30 * If we have booted due to a crash, max_pfn will be a very low value. We need
31 * to know the amount of memory that the previous kernel used.
32 */
33unsigned long saved_max_pfn;
34#endif
35
Yinghai Lu08677212010-02-10 01:20:20 -080036#ifndef CONFIG_NO_BOOTMEM
Johannes Weinerb61bfa32008-07-23 21:26:55 -070037bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
38
Johannes Weiner636cc402008-07-23 21:28:03 -070039static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
40
Johannes Weiner2e5237d2008-07-23 21:28:02 -070041static int bootmem_debug;
42
43static int __init bootmem_debug_setup(char *buf)
44{
45 bootmem_debug = 1;
46 return 0;
47}
48early_param("bootmem_debug", bootmem_debug_setup);
49
50#define bdebug(fmt, args...) ({ \
51 if (unlikely(bootmem_debug)) \
52 printk(KERN_INFO \
53 "bootmem::%s " fmt, \
Harvey Harrison80a914d2008-10-15 22:01:25 -070054 __func__, ## args); \
Johannes Weiner2e5237d2008-07-23 21:28:02 -070055})
56
Johannes Weinerdf049a52008-07-23 21:28:02 -070057static unsigned long __init bootmap_bytes(unsigned long pages)
Johannes Weiner223e8dc2008-07-23 21:28:00 -070058{
Johannes Weinerdf049a52008-07-23 21:28:02 -070059 unsigned long bytes = (pages + 7) / 8;
Johannes Weiner223e8dc2008-07-23 21:28:00 -070060
Johannes Weinerdf049a52008-07-23 21:28:02 -070061 return ALIGN(bytes, sizeof(long));
Johannes Weiner223e8dc2008-07-23 21:28:00 -070062}
63
Johannes Weinera66fd7d2008-07-23 21:28:01 -070064/**
65 * bootmem_bootmap_pages - calculate bitmap size in pages
66 * @pages: number of pages the bitmap has to represent
67 */
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070068unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -070069{
Johannes Weinerdf049a52008-07-23 21:28:02 -070070 unsigned long bytes = bootmap_bytes(pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Johannes Weinerdf049a52008-07-23 21:28:02 -070072 return PAGE_ALIGN(bytes) >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -070073}
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070074
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080075/*
76 * link bdata in order
77 */
Franck Bui-Huu69d49e62006-09-25 23:31:04 -070078static void __init link_bootmem(bootmem_data_t *bdata)
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080079{
Johannes Weiner636cc402008-07-23 21:28:03 -070080 struct list_head *iter;
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070081
Johannes Weiner636cc402008-07-23 21:28:03 -070082 list_for_each(iter, &bdata_list) {
83 bootmem_data_t *ent;
84
85 ent = list_entry(iter, bootmem_data_t, list);
Johannes Weiner3560e242008-07-23 21:28:09 -070086 if (bdata->node_min_pfn < ent->node_min_pfn)
Johannes Weiner636cc402008-07-23 21:28:03 -070087 break;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080088 }
Johannes Weiner636cc402008-07-23 21:28:03 -070089 list_add_tail(&bdata->list, iter);
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080090}
91
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070092/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 * Called once to set up the allocator itself.
94 */
Johannes Weiner8ae04462008-07-23 21:26:59 -070095static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 unsigned long mapstart, unsigned long start, unsigned long end)
97{
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070098 unsigned long mapsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700100 mminit_validate_memmodel_limits(&start, &end);
Franck Bui-Huubbc7b922006-09-25 23:31:07 -0700101 bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
Johannes Weiner3560e242008-07-23 21:28:09 -0700102 bdata->node_min_pfn = start;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 bdata->node_low_pfn = end;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -0800104 link_bootmem(bdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
106 /*
107 * Initially all pages are reserved - setup_arch() has to
108 * register free RAM areas explicitly.
109 */
Johannes Weinerdf049a52008-07-23 21:28:02 -0700110 mapsize = bootmap_bytes(end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 memset(bdata->node_bootmem_map, 0xff, mapsize);
112
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700113 bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n",
114 bdata - bootmem_node_data, start, mapstart, end, mapsize);
115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 return mapsize;
117}
118
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700119/**
120 * init_bootmem_node - register a node as boot memory
121 * @pgdat: node to register
122 * @freepfn: pfn where the bitmap for this node is to be placed
123 * @startpfn: first pfn on the node
124 * @endpfn: first pfn after the node
125 *
126 * Returns the number of bytes needed to hold the bitmap for this node.
127 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700128unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
129 unsigned long startpfn, unsigned long endpfn)
130{
131 return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
132}
133
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700134/**
135 * init_bootmem - register boot memory
136 * @start: pfn where the bitmap is to be placed
137 * @pages: number of available physical pages
138 *
139 * Returns the number of bytes needed to hold the bitmap.
140 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700141unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
142{
143 max_low_pfn = pages;
144 min_low_pfn = start;
145 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
146}
Yinghai Lu08677212010-02-10 01:20:20 -0800147#endif
FUJITA Tomonori9f993ac2009-11-10 19:46:17 +0900148/*
149 * free_bootmem_late - free bootmem pages directly to page allocator
150 * @addr: starting address of the range
151 * @size: size of the range in bytes
152 *
153 * This is only useful when the bootmem allocator has already been torn
154 * down, but we are still initializing the system. Pages are given directly
155 * to the page allocator, no bootmem metadata is updated because it is gone.
156 */
157void __init free_bootmem_late(unsigned long addr, unsigned long size)
158{
159 unsigned long cursor, end;
160
161 kmemleak_free_part(__va(addr), size);
162
163 cursor = PFN_UP(addr);
164 end = PFN_DOWN(addr + size);
165
166 for (; cursor < end; cursor++) {
167 __free_pages_bootmem(pfn_to_page(cursor), 0);
168 totalram_pages++;
169 }
170}
171
Yinghai Lu08677212010-02-10 01:20:20 -0800172#ifdef CONFIG_NO_BOOTMEM
173static void __init __free_pages_memory(unsigned long start, unsigned long end)
174{
175 int i;
176 unsigned long start_aligned, end_aligned;
177 int order = ilog2(BITS_PER_LONG);
178
179 start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
180 end_aligned = end & ~(BITS_PER_LONG - 1);
181
182 if (end_aligned <= start_aligned) {
Yinghai Lu08677212010-02-10 01:20:20 -0800183 for (i = start; i < end; i++)
184 __free_pages_bootmem(pfn_to_page(i), 0);
185
186 return;
187 }
188
Yinghai Lu08677212010-02-10 01:20:20 -0800189 for (i = start; i < start_aligned; i++)
190 __free_pages_bootmem(pfn_to_page(i), 0);
191
192 for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
193 __free_pages_bootmem(pfn_to_page(i), order);
194
195 for (i = end_aligned; i < end; i++)
196 __free_pages_bootmem(pfn_to_page(i), 0);
197}
198
199unsigned long __init free_all_memory_core_early(int nodeid)
200{
201 int i;
202 u64 start, end;
203 unsigned long count = 0;
204 struct range *range = NULL;
205 int nr_range;
206
207 nr_range = get_free_all_memory_range(&range, nodeid);
208
209 for (i = 0; i < nr_range; i++) {
210 start = range[i].start;
211 end = range[i].end;
212 count += end - start;
213 __free_pages_memory(start, end);
214 }
215
216 return count;
217}
218#else
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700219static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
220{
Johannes Weiner41546c12008-07-23 21:28:03 -0700221 int aligned;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700222 struct page *page;
Johannes Weiner41546c12008-07-23 21:28:03 -0700223 unsigned long start, end, pages, count = 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700224
Johannes Weiner41546c12008-07-23 21:28:03 -0700225 if (!bdata->node_bootmem_map)
226 return 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700227
Johannes Weiner3560e242008-07-23 21:28:09 -0700228 start = bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700229 end = bdata->node_low_pfn;
230
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700231 /*
Johannes Weiner41546c12008-07-23 21:28:03 -0700232 * If the start is aligned to the machines wordsize, we might
233 * be able to free pages in bulks of that order.
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700234 */
Johannes Weiner41546c12008-07-23 21:28:03 -0700235 aligned = !(start & (BITS_PER_LONG - 1));
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700236
Johannes Weiner41546c12008-07-23 21:28:03 -0700237 bdebug("nid=%td start=%lx end=%lx aligned=%d\n",
238 bdata - bootmem_node_data, start, end, aligned);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700239
Johannes Weiner41546c12008-07-23 21:28:03 -0700240 while (start < end) {
241 unsigned long *map, idx, vec;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700242
Johannes Weiner41546c12008-07-23 21:28:03 -0700243 map = bdata->node_bootmem_map;
Johannes Weiner3560e242008-07-23 21:28:09 -0700244 idx = start - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700245 vec = ~map[idx / BITS_PER_LONG];
246
247 if (aligned && vec == ~0UL && start + BITS_PER_LONG < end) {
248 int order = ilog2(BITS_PER_LONG);
249
250 __free_pages_bootmem(pfn_to_page(start), order);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700251 count += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700252 } else {
Johannes Weiner41546c12008-07-23 21:28:03 -0700253 unsigned long off = 0;
254
255 while (vec && off < BITS_PER_LONG) {
256 if (vec & 1) {
257 page = pfn_to_page(start + off);
258 __free_pages_bootmem(page, 0);
259 count++;
260 }
261 vec >>= 1;
262 off++;
263 }
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700264 }
Johannes Weiner41546c12008-07-23 21:28:03 -0700265 start += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700266 }
267
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700268 page = virt_to_page(bdata->node_bootmem_map);
Johannes Weiner3560e242008-07-23 21:28:09 -0700269 pages = bdata->node_low_pfn - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700270 pages = bootmem_bootmap_pages(pages);
271 count += pages;
272 while (pages--)
273 __free_pages_bootmem(page++, 0);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700274
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700275 bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);
276
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700277 return count;
278}
Yinghai Lu08677212010-02-10 01:20:20 -0800279#endif
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700280
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700281/**
282 * free_all_bootmem_node - release a node's free pages to the buddy allocator
283 * @pgdat: node to be released
284 *
285 * Returns the number of pages actually released.
286 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700287unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
288{
289 register_page_bootmem_info_node(pgdat);
Yinghai Lu08677212010-02-10 01:20:20 -0800290#ifdef CONFIG_NO_BOOTMEM
291 /* free_all_memory_core_early(MAX_NUMNODES) will be called later */
292 return 0;
293#else
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700294 return free_all_bootmem_core(pgdat->bdata);
Yinghai Lu08677212010-02-10 01:20:20 -0800295#endif
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700296}
297
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700298/**
299 * free_all_bootmem - release free pages to the buddy allocator
300 *
301 * Returns the number of pages actually released.
302 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700303unsigned long __init free_all_bootmem(void)
304{
Yinghai Lu08677212010-02-10 01:20:20 -0800305#ifdef CONFIG_NO_BOOTMEM
Yinghai Lu33799852010-03-31 20:44:09 -0700306 /*
307 * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
308 * because in some case like Node0 doesnt have RAM installed
309 * low ram will be on Node1
310 * Use MAX_NUMNODES will make sure all ranges in early_node_map[]
311 * will be used instead of only Node0 related
312 */
313 return free_all_memory_core_early(MAX_NUMNODES);
Yinghai Lu08677212010-02-10 01:20:20 -0800314#else
Yinghai Luaa235fc2010-03-31 20:45:27 -0700315 unsigned long total_pages = 0;
316 bootmem_data_t *bdata;
317
318 list_for_each_entry(bdata, &bdata_list, list)
319 total_pages += free_all_bootmem_core(bdata);
320
321 return total_pages;
Yinghai Lu08677212010-02-10 01:20:20 -0800322#endif
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700323}
324
Yinghai Lu08677212010-02-10 01:20:20 -0800325#ifndef CONFIG_NO_BOOTMEM
Johannes Weinerd747fa42008-07-23 21:28:05 -0700326static void __init __free(bootmem_data_t *bdata,
327 unsigned long sidx, unsigned long eidx)
328{
329 unsigned long idx;
330
331 bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700332 sidx + bdata->node_min_pfn,
333 eidx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700334
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700335 if (bdata->hint_idx > sidx)
336 bdata->hint_idx = sidx;
337
Johannes Weinerd747fa42008-07-23 21:28:05 -0700338 for (idx = sidx; idx < eidx; idx++)
339 if (!test_and_clear_bit(idx, bdata->node_bootmem_map))
340 BUG();
341}
342
343static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,
344 unsigned long eidx, int flags)
345{
346 unsigned long idx;
347 int exclusive = flags & BOOTMEM_EXCLUSIVE;
348
349 bdebug("nid=%td start=%lx end=%lx flags=%x\n",
350 bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700351 sidx + bdata->node_min_pfn,
352 eidx + bdata->node_min_pfn,
Johannes Weinerd747fa42008-07-23 21:28:05 -0700353 flags);
354
355 for (idx = sidx; idx < eidx; idx++)
356 if (test_and_set_bit(idx, bdata->node_bootmem_map)) {
357 if (exclusive) {
358 __free(bdata, sidx, idx);
359 return -EBUSY;
360 }
361 bdebug("silent double reserve of PFN %lx\n",
Johannes Weiner3560e242008-07-23 21:28:09 -0700362 idx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700363 }
364 return 0;
365}
366
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700367static int __init mark_bootmem_node(bootmem_data_t *bdata,
368 unsigned long start, unsigned long end,
369 int reserve, int flags)
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700370{
371 unsigned long sidx, eidx;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700372
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700373 bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n",
374 bdata - bootmem_node_data, start, end, reserve, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700375
Johannes Weiner3560e242008-07-23 21:28:09 -0700376 BUG_ON(start < bdata->node_min_pfn);
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700377 BUG_ON(end > bdata->node_low_pfn);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700378
Johannes Weiner3560e242008-07-23 21:28:09 -0700379 sidx = start - bdata->node_min_pfn;
380 eidx = end - bdata->node_min_pfn;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700381
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700382 if (reserve)
383 return __reserve(bdata, sidx, eidx, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700384 else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700385 __free(bdata, sidx, eidx);
386 return 0;
387}
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700388
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700389static int __init mark_bootmem(unsigned long start, unsigned long end,
390 int reserve, int flags)
391{
392 unsigned long pos;
393 bootmem_data_t *bdata;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700394
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700395 pos = start;
396 list_for_each_entry(bdata, &bdata_list, list) {
397 int err;
398 unsigned long max;
399
Johannes Weiner3560e242008-07-23 21:28:09 -0700400 if (pos < bdata->node_min_pfn ||
401 pos >= bdata->node_low_pfn) {
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700402 BUG_ON(pos != start);
403 continue;
404 }
405
406 max = min(bdata->node_low_pfn, end);
407
408 err = mark_bootmem_node(bdata, pos, max, reserve, flags);
409 if (reserve && err) {
410 mark_bootmem(start, pos, 0, 0);
411 return err;
412 }
413
414 if (max == end)
415 return 0;
416 pos = bdata->node_low_pfn;
417 }
418 BUG();
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700419}
Yinghai Lu08677212010-02-10 01:20:20 -0800420#endif
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700421
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700422/**
423 * free_bootmem_node - mark a page range as usable
424 * @pgdat: node the range resides on
425 * @physaddr: starting address of the range
426 * @size: size of the range in bytes
427 *
428 * Partial pages will be considered reserved and left as they are.
429 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700430 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700431 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700432void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
433 unsigned long size)
434{
Yinghai Lu08677212010-02-10 01:20:20 -0800435#ifdef CONFIG_NO_BOOTMEM
436 free_early(physaddr, physaddr + size);
Yinghai Lu08677212010-02-10 01:20:20 -0800437#else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700438 unsigned long start, end;
439
Catalin Marinasec3a3542009-07-07 10:33:01 +0100440 kmemleak_free_part(__va(physaddr), size);
441
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700442 start = PFN_UP(physaddr);
443 end = PFN_DOWN(physaddr + size);
444
445 mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
Yinghai Lu08677212010-02-10 01:20:20 -0800446#endif
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700447}
448
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700449/**
450 * free_bootmem - mark a page range as usable
451 * @addr: starting address of the range
452 * @size: size of the range in bytes
453 *
454 * Partial pages will be considered reserved and left as they are.
455 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700456 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700457 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700458void __init free_bootmem(unsigned long addr, unsigned long size)
459{
Yinghai Lu08677212010-02-10 01:20:20 -0800460#ifdef CONFIG_NO_BOOTMEM
461 free_early(addr, addr + size);
Yinghai Lu08677212010-02-10 01:20:20 -0800462#else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700463 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700464
Catalin Marinasec3a3542009-07-07 10:33:01 +0100465 kmemleak_free_part(__va(addr), size);
466
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700467 start = PFN_UP(addr);
468 end = PFN_DOWN(addr + size);
Yinghai Lua5645a62008-03-18 12:49:12 -0700469
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700470 mark_bootmem(start, end, 0, 0);
Yinghai Lu08677212010-02-10 01:20:20 -0800471#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472}
473
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700474/**
475 * reserve_bootmem_node - mark a page range as reserved
476 * @pgdat: node the range resides on
477 * @physaddr: starting address of the range
478 * @size: size of the range in bytes
479 * @flags: reservation flags (see linux/bootmem.h)
480 *
481 * Partial pages will be reserved.
482 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700483 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700484 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700485int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
486 unsigned long size, int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487{
Yinghai Lu08677212010-02-10 01:20:20 -0800488#ifdef CONFIG_NO_BOOTMEM
489 panic("no bootmem");
490 return 0;
491#else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700492 unsigned long start, end;
Franck Bui-Huubbc7b922006-09-25 23:31:07 -0700493
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700494 start = PFN_DOWN(physaddr);
495 end = PFN_UP(physaddr + size);
496
497 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
Yinghai Lu08677212010-02-10 01:20:20 -0800498#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499}
500
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700501/**
502 * reserve_bootmem - mark a page range as usable
503 * @addr: starting address of the range
504 * @size: size of the range in bytes
505 * @flags: reservation flags (see linux/bootmem.h)
506 *
507 * Partial pages will be reserved.
508 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700509 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700510 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700511int __init reserve_bootmem(unsigned long addr, unsigned long size,
512 int flags)
513{
Yinghai Lu08677212010-02-10 01:20:20 -0800514#ifdef CONFIG_NO_BOOTMEM
515 panic("no bootmem");
516 return 0;
517#else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700518 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700519
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700520 start = PFN_DOWN(addr);
521 end = PFN_UP(addr + size);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700522
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700523 return mark_bootmem(start, end, 1, flags);
Yinghai Lu08677212010-02-10 01:20:20 -0800524#endif
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700525}
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700526
Yinghai Lu08677212010-02-10 01:20:20 -0800527#ifndef CONFIG_NO_BOOTMEM
Jan Beulich8aa043d2009-12-14 17:59:46 -0800528static unsigned long __init align_idx(struct bootmem_data *bdata,
529 unsigned long idx, unsigned long step)
Johannes Weiner481ebd02008-08-20 14:09:15 -0700530{
531 unsigned long base = bdata->node_min_pfn;
532
533 /*
534 * Align the index with respect to the node start so that the
535 * combination of both satisfies the requested alignment.
536 */
537
538 return ALIGN(base + idx, step) - base;
539}
540
Jan Beulich8aa043d2009-12-14 17:59:46 -0800541static unsigned long __init align_off(struct bootmem_data *bdata,
542 unsigned long off, unsigned long align)
Johannes Weiner481ebd02008-08-20 14:09:15 -0700543{
544 unsigned long base = PFN_PHYS(bdata->node_min_pfn);
545
546 /* Same as align_idx for byte offsets */
547
548 return ALIGN(base + off, align) - base;
549}
550
Tejun Heod0c4f572009-03-01 16:06:56 +0900551static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
552 unsigned long size, unsigned long align,
553 unsigned long goal, unsigned long limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700555 unsigned long fallback = 0;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700556 unsigned long min, max, start, sidx, midx, step;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557
Johannes Weiner594fe1a2009-01-06 14:40:32 -0800558 bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
559 bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
560 align, goal, limit);
561
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700562 BUG_ON(!size);
563 BUG_ON(align & (align - 1));
564 BUG_ON(limit && goal + size > limit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565
Christian Krafft7c309a62006-12-06 20:32:41 -0800566 if (!bdata->node_bootmem_map)
567 return NULL;
568
Johannes Weiner3560e242008-07-23 21:28:09 -0700569 min = bdata->node_min_pfn;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700570 max = bdata->node_low_pfn;
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700571
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700572 goal >>= PAGE_SHIFT;
573 limit >>= PAGE_SHIFT;
574
575 if (limit && max > limit)
576 max = limit;
577 if (max <= min)
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700578 return NULL;
579
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700580 step = max(align >> PAGE_SHIFT, 1UL);
Yasunori Goto281dd252005-10-19 15:52:18 -0700581
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700582 if (goal && min < goal && goal < max)
583 start = ALIGN(goal, step);
584 else
585 start = ALIGN(min, step);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586
Johannes Weiner481ebd02008-08-20 14:09:15 -0700587 sidx = start - bdata->node_min_pfn;
Johannes Weiner3560e242008-07-23 21:28:09 -0700588 midx = max - bdata->node_min_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700590 if (bdata->hint_idx > sidx) {
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700591 /*
592 * Handle the valid case of sidx being zero and still
593 * catch the fallback below.
594 */
595 fallback = sidx + 1;
Johannes Weiner481ebd02008-08-20 14:09:15 -0700596 sidx = align_idx(bdata, bdata->hint_idx, step);
Yinghai Luad093152008-03-10 23:23:42 -0700597 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700599 while (1) {
600 int merge;
601 void *region;
602 unsigned long eidx, i, start_off, end_off;
603find_block:
604 sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
Johannes Weiner481ebd02008-08-20 14:09:15 -0700605 sidx = align_idx(bdata, sidx, step);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700606 eidx = sidx + PFN_UP(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700608 if (sidx >= midx || eidx > midx)
Haren Myneni66d43e92005-12-12 00:37:39 -0800609 break;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700610
611 for (i = sidx; i < eidx; i++)
612 if (test_bit(i, bdata->node_bootmem_map)) {
Johannes Weiner481ebd02008-08-20 14:09:15 -0700613 sidx = align_idx(bdata, i, step);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700614 if (sidx == i)
615 sidx += step;
616 goto find_block;
617 }
618
Mikulas Patocka627240a2008-08-15 00:40:17 -0700619 if (bdata->last_end_off & (PAGE_SIZE - 1) &&
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700620 PFN_DOWN(bdata->last_end_off) + 1 == sidx)
Johannes Weiner481ebd02008-08-20 14:09:15 -0700621 start_off = align_off(bdata, bdata->last_end_off, align);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700622 else
623 start_off = PFN_PHYS(sidx);
624
625 merge = PFN_DOWN(start_off) < sidx;
626 end_off = start_off + size;
627
628 bdata->last_end_off = end_off;
629 bdata->hint_idx = PFN_UP(end_off);
630
631 /*
632 * Reserve the area now:
633 */
Johannes Weinerd747fa42008-07-23 21:28:05 -0700634 if (__reserve(bdata, PFN_DOWN(start_off) + merge,
635 PFN_UP(end_off), BOOTMEM_EXCLUSIVE))
636 BUG();
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700637
Johannes Weiner3560e242008-07-23 21:28:09 -0700638 region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
639 start_off);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700640 memset(region, 0, size);
Catalin Marinas008139d2009-08-27 14:29:17 +0100641 /*
642 * The min_count is set to 0 so that bootmem allocated blocks
643 * are never reported as leaks.
644 */
645 kmemleak_alloc(region, size, 0, 0);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700646 return region;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 }
648
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700649 if (fallback) {
Johannes Weiner481ebd02008-08-20 14:09:15 -0700650 sidx = align_idx(bdata, fallback - 1, step);
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700651 fallback = 0;
652 goto find_block;
653 }
654
655 return NULL;
656}
657
Tejun Heod0c4f572009-03-01 16:06:56 +0900658static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
659 unsigned long size, unsigned long align,
660 unsigned long goal, unsigned long limit)
661{
Pekka Enberg441c7e02009-06-10 20:05:53 +0300662 if (WARN_ON_ONCE(slab_is_available()))
663 return kzalloc(size, GFP_NOWAIT);
664
Tejun Heod0c4f572009-03-01 16:06:56 +0900665#ifdef CONFIG_HAVE_ARCH_BOOTMEM
Joe Perches433f13a2009-06-18 16:48:58 -0700666 {
667 bootmem_data_t *p_bdata;
Tejun Heod0c4f572009-03-01 16:06:56 +0900668
Joe Perches433f13a2009-06-18 16:48:58 -0700669 p_bdata = bootmem_arch_preferred_node(bdata, size, align,
670 goal, limit);
671 if (p_bdata)
672 return alloc_bootmem_core(p_bdata, size, align,
673 goal, limit);
674 }
Tejun Heod0c4f572009-03-01 16:06:56 +0900675#endif
676 return NULL;
677}
Yinghai Lu08677212010-02-10 01:20:20 -0800678#endif
Tejun Heod0c4f572009-03-01 16:06:56 +0900679
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700680static void * __init ___alloc_bootmem_nopanic(unsigned long size,
681 unsigned long align,
682 unsigned long goal,
683 unsigned long limit)
684{
Yinghai Lu08677212010-02-10 01:20:20 -0800685#ifdef CONFIG_NO_BOOTMEM
686 void *ptr;
687
688 if (WARN_ON_ONCE(slab_is_available()))
689 return kzalloc(size, GFP_NOWAIT);
690
691restart:
692
693 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
694
695 if (ptr)
696 return ptr;
697
698 if (goal != 0) {
699 goal = 0;
700 goto restart;
701 }
702
703 return NULL;
704#else
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700705 bootmem_data_t *bdata;
Tejun Heod0c4f572009-03-01 16:06:56 +0900706 void *region;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700707
708restart:
Tejun Heod0c4f572009-03-01 16:06:56 +0900709 region = alloc_arch_preferred_bootmem(NULL, size, align, goal, limit);
710 if (region)
711 return region;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700712
Tejun Heod0c4f572009-03-01 16:06:56 +0900713 list_for_each_entry(bdata, &bdata_list, list) {
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700714 if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
715 continue;
Johannes Weiner3560e242008-07-23 21:28:09 -0700716 if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700717 break;
718
719 region = alloc_bootmem_core(bdata, size, align, goal, limit);
720 if (region)
721 return region;
722 }
723
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700724 if (goal) {
725 goal = 0;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700726 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 }
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700728
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 return NULL;
Yinghai Lu08677212010-02-10 01:20:20 -0800730#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731}
732
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700733/**
734 * __alloc_bootmem_nopanic - allocate boot memory without panicking
735 * @size: size of the request in bytes
736 * @align: alignment of the region
737 * @goal: preferred starting address of the region
738 *
739 * The goal is dropped if it can not be satisfied and the allocation will
740 * fall back to memory below @goal.
741 *
742 * Allocation may happen on any node in the system.
743 *
744 * Returns NULL on failure.
745 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700746void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700747 unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748{
Yinghai Lu08677212010-02-10 01:20:20 -0800749 unsigned long limit = 0;
750
751#ifdef CONFIG_NO_BOOTMEM
752 limit = -1UL;
753#endif
754
755 return ___alloc_bootmem_nopanic(size, align, goal, limit);
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700756}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700758static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
759 unsigned long goal, unsigned long limit)
760{
761 void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
762
763 if (mem)
764 return mem;
765 /*
766 * Whoops, we cannot satisfy the allocation request.
767 */
768 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
769 panic("Out of memory");
Andi Kleena8062232006-04-07 19:49:21 +0200770 return NULL;
771}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700773/**
774 * __alloc_bootmem - allocate boot memory
775 * @size: size of the request in bytes
776 * @align: alignment of the region
777 * @goal: preferred starting address of the region
778 *
779 * The goal is dropped if it can not be satisfied and the allocation will
780 * fall back to memory below @goal.
781 *
782 * Allocation may happen on any node in the system.
783 *
784 * The function panics if the request can not be satisfied.
785 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700786void * __init __alloc_bootmem(unsigned long size, unsigned long align,
787 unsigned long goal)
Andi Kleena8062232006-04-07 19:49:21 +0200788{
Yinghai Lu08677212010-02-10 01:20:20 -0800789 unsigned long limit = 0;
790
791#ifdef CONFIG_NO_BOOTMEM
792 limit = -1UL;
793#endif
794
795 return ___alloc_bootmem(size, align, goal, limit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796}
797
Yinghai Lu08677212010-02-10 01:20:20 -0800798#ifndef CONFIG_NO_BOOTMEM
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700799static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
800 unsigned long size, unsigned long align,
801 unsigned long goal, unsigned long limit)
802{
803 void *ptr;
804
Tejun Heod0c4f572009-03-01 16:06:56 +0900805 ptr = alloc_arch_preferred_bootmem(bdata, size, align, goal, limit);
806 if (ptr)
807 return ptr;
808
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700809 ptr = alloc_bootmem_core(bdata, size, align, goal, limit);
810 if (ptr)
811 return ptr;
812
813 return ___alloc_bootmem(size, align, goal, limit);
814}
Yinghai Lu08677212010-02-10 01:20:20 -0800815#endif
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700816
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700817/**
818 * __alloc_bootmem_node - allocate boot memory from a specific node
819 * @pgdat: node to allocate from
820 * @size: size of the request in bytes
821 * @align: alignment of the region
822 * @goal: preferred starting address of the region
823 *
824 * The goal is dropped if it can not be satisfied and the allocation will
825 * fall back to memory below @goal.
826 *
827 * Allocation may fall back to any node in the system if the specified node
828 * can not hold the requested memory.
829 *
830 * The function panics if the request can not be satisfied.
831 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700832void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
833 unsigned long align, unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834{
Pekka Enbergc91c4772009-06-11 08:10:28 +0300835 if (WARN_ON_ONCE(slab_is_available()))
836 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
837
Yinghai Lu08677212010-02-10 01:20:20 -0800838#ifdef CONFIG_NO_BOOTMEM
839 return __alloc_memory_core_early(pgdat->node_id, size, align,
840 goal, -1ULL);
841#else
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700842 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
Yinghai Lu08677212010-02-10 01:20:20 -0800843#endif
844}
845
846void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
847 unsigned long align, unsigned long goal)
848{
849#ifdef MAX_DMA32_PFN
850 unsigned long end_pfn;
851
852 if (WARN_ON_ONCE(slab_is_available()))
853 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
854
855 /* update goal according ...MAX_DMA32_PFN */
856 end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages;
857
858 if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) &&
859 (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) {
860 void *ptr;
861 unsigned long new_goal;
862
863 new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
864#ifdef CONFIG_NO_BOOTMEM
865 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
866 new_goal, -1ULL);
867#else
868 ptr = alloc_bootmem_core(pgdat->bdata, size, align,
869 new_goal, 0);
870#endif
871 if (ptr)
872 return ptr;
873 }
874#endif
875
876 return __alloc_bootmem_node(pgdat, size, align, goal);
877
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878}
879
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700880#ifdef CONFIG_SPARSEMEM
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700881/**
882 * alloc_bootmem_section - allocate boot memory from a specific section
883 * @size: size of the request in bytes
884 * @section_nr: sparse map section to allocate from
885 *
886 * Return NULL on failure.
887 */
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700888void * __init alloc_bootmem_section(unsigned long size,
889 unsigned long section_nr)
890{
Yinghai Lu08677212010-02-10 01:20:20 -0800891#ifdef CONFIG_NO_BOOTMEM
892 unsigned long pfn, goal, limit;
893
894 pfn = section_nr_to_pfn(section_nr);
895 goal = pfn << PAGE_SHIFT;
896 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
897
898 return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
899 SMP_CACHE_BYTES, goal, limit);
900#else
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700901 bootmem_data_t *bdata;
902 unsigned long pfn, goal, limit;
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700903
904 pfn = section_nr_to_pfn(section_nr);
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700905 goal = pfn << PAGE_SHIFT;
906 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
907 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700908
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700909 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
Yinghai Lu08677212010-02-10 01:20:20 -0800910#endif
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700911}
912#endif
913
Andi Kleenb54bbf72008-07-23 21:27:45 -0700914void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
915 unsigned long align, unsigned long goal)
916{
917 void *ptr;
918
Pekka Enbergc91c4772009-06-11 08:10:28 +0300919 if (WARN_ON_ONCE(slab_is_available()))
920 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
921
Yinghai Lu08677212010-02-10 01:20:20 -0800922#ifdef CONFIG_NO_BOOTMEM
923 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
924 goal, -1ULL);
925#else
Tejun Heod0c4f572009-03-01 16:06:56 +0900926 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
927 if (ptr)
928 return ptr;
929
Andi Kleenb54bbf72008-07-23 21:27:45 -0700930 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Yinghai Lu08677212010-02-10 01:20:20 -0800931#endif
Andi Kleenb54bbf72008-07-23 21:27:45 -0700932 if (ptr)
933 return ptr;
934
935 return __alloc_bootmem_nopanic(size, align, goal);
936}
937
Heiko Carstensdfd54cb2006-09-25 23:31:33 -0700938#ifndef ARCH_LOW_ADDRESS_LIMIT
939#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
940#endif
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800941
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700942/**
943 * __alloc_bootmem_low - allocate low boot memory
944 * @size: size of the request in bytes
945 * @align: alignment of the region
946 * @goal: preferred starting address of the region
947 *
948 * The goal is dropped if it can not be satisfied and the allocation will
949 * fall back to memory below @goal.
950 *
951 * Allocation may happen on any node in the system.
952 *
953 * The function panics if the request can not be satisfied.
954 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700955void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
956 unsigned long goal)
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800957{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700958 return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800959}
960
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700961/**
962 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
963 * @pgdat: node to allocate from
964 * @size: size of the request in bytes
965 * @align: alignment of the region
966 * @goal: preferred starting address of the region
967 *
968 * The goal is dropped if it can not be satisfied and the allocation will
969 * fall back to memory below @goal.
970 *
971 * Allocation may fall back to any node in the system if the specified node
972 * can not hold the requested memory.
973 *
974 * The function panics if the request can not be satisfied.
975 */
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800976void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
977 unsigned long align, unsigned long goal)
978{
Pekka Enbergc91c4772009-06-11 08:10:28 +0300979 if (WARN_ON_ONCE(slab_is_available()))
980 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
981
Yinghai Lu08677212010-02-10 01:20:20 -0800982#ifdef CONFIG_NO_BOOTMEM
983 return __alloc_memory_core_early(pgdat->node_id, size, align,
984 goal, ARCH_LOW_ADDRESS_LIMIT);
985#else
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700986 return ___alloc_bootmem_node(pgdat->bdata, size, align,
987 goal, ARCH_LOW_ADDRESS_LIMIT);
Yinghai Lu08677212010-02-10 01:20:20 -0800988#endif
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800989}