blob: 282df0a09e6f27ffb5245daf1eac4ae71f1195c5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Johannes Weiner57cfc292008-07-23 21:28:00 -07002 * bootmem - A boot-time physical memory allocator and configurator
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 1999 Ingo Molnar
Johannes Weiner57cfc292008-07-23 21:28:00 -07005 * 1999 Kanoj Sarcar, SGI
6 * 2008 Johannes Weiner
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Johannes Weiner57cfc292008-07-23 21:28:00 -07008 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070012#include <linux/pfn.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/module.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070015
16#include <asm/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <asm/io.h>
Heiko Carstensdfd54cb2006-09-25 23:31:33 -070018#include <asm/processor.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070019
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include "internal.h"
21
Linus Torvalds1da177e2005-04-16 15:20:36 -070022unsigned long max_low_pfn;
23unsigned long min_low_pfn;
24unsigned long max_pfn;
25
Vivek Goyal92aa63a2005-06-25 14:58:18 -070026#ifdef CONFIG_CRASH_DUMP
27/*
28 * If we have booted due to a crash, max_pfn will be a very low value. We need
29 * to know the amount of memory that the previous kernel used.
30 */
31unsigned long saved_max_pfn;
32#endif
33
Johannes Weinerb61bfa32008-07-23 21:26:55 -070034bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
35
Johannes Weiner636cc402008-07-23 21:28:03 -070036static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
37
Johannes Weiner2e5237d2008-07-23 21:28:02 -070038static int bootmem_debug;
39
40static int __init bootmem_debug_setup(char *buf)
41{
42 bootmem_debug = 1;
43 return 0;
44}
45early_param("bootmem_debug", bootmem_debug_setup);
46
47#define bdebug(fmt, args...) ({ \
48 if (unlikely(bootmem_debug)) \
49 printk(KERN_INFO \
50 "bootmem::%s " fmt, \
Harvey Harrison80a914d2008-10-15 22:01:25 -070051 __func__, ## args); \
Johannes Weiner2e5237d2008-07-23 21:28:02 -070052})
53
Johannes Weinerdf049a52008-07-23 21:28:02 -070054static unsigned long __init bootmap_bytes(unsigned long pages)
Johannes Weiner223e8dc2008-07-23 21:28:00 -070055{
Johannes Weinerdf049a52008-07-23 21:28:02 -070056 unsigned long bytes = (pages + 7) / 8;
Johannes Weiner223e8dc2008-07-23 21:28:00 -070057
Johannes Weinerdf049a52008-07-23 21:28:02 -070058 return ALIGN(bytes, sizeof(long));
Johannes Weiner223e8dc2008-07-23 21:28:00 -070059}
60
Johannes Weinera66fd7d2008-07-23 21:28:01 -070061/**
62 * bootmem_bootmap_pages - calculate bitmap size in pages
63 * @pages: number of pages the bitmap has to represent
64 */
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070065unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066{
Johannes Weinerdf049a52008-07-23 21:28:02 -070067 unsigned long bytes = bootmap_bytes(pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
Johannes Weinerdf049a52008-07-23 21:28:02 -070069 return PAGE_ALIGN(bytes) >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070}
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070071
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080072/*
73 * link bdata in order
74 */
Franck Bui-Huu69d49e62006-09-25 23:31:04 -070075static void __init link_bootmem(bootmem_data_t *bdata)
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080076{
Johannes Weiner636cc402008-07-23 21:28:03 -070077 struct list_head *iter;
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070078
Johannes Weiner636cc402008-07-23 21:28:03 -070079 list_for_each(iter, &bdata_list) {
80 bootmem_data_t *ent;
81
82 ent = list_entry(iter, bootmem_data_t, list);
Johannes Weiner3560e242008-07-23 21:28:09 -070083 if (bdata->node_min_pfn < ent->node_min_pfn)
Johannes Weiner636cc402008-07-23 21:28:03 -070084 break;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080085 }
Johannes Weiner636cc402008-07-23 21:28:03 -070086 list_add_tail(&bdata->list, iter);
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080087}
88
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070089/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 * Called once to set up the allocator itself.
91 */
Johannes Weiner8ae04462008-07-23 21:26:59 -070092static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 unsigned long mapstart, unsigned long start, unsigned long end)
94{
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070095 unsigned long mapsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Mel Gorman2dbb51c2008-07-23 21:26:52 -070097 mminit_validate_memmodel_limits(&start, &end);
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070098 bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
Johannes Weiner3560e242008-07-23 21:28:09 -070099 bdata->node_min_pfn = start;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 bdata->node_low_pfn = end;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -0800101 link_bootmem(bdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
103 /*
104 * Initially all pages are reserved - setup_arch() has to
105 * register free RAM areas explicitly.
106 */
Johannes Weinerdf049a52008-07-23 21:28:02 -0700107 mapsize = bootmap_bytes(end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 memset(bdata->node_bootmem_map, 0xff, mapsize);
109
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700110 bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n",
111 bdata - bootmem_node_data, start, mapstart, end, mapsize);
112
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 return mapsize;
114}
115
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700116/**
117 * init_bootmem_node - register a node as boot memory
118 * @pgdat: node to register
119 * @freepfn: pfn where the bitmap for this node is to be placed
120 * @startpfn: first pfn on the node
121 * @endpfn: first pfn after the node
122 *
123 * Returns the number of bytes needed to hold the bitmap for this node.
124 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700125unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
126 unsigned long startpfn, unsigned long endpfn)
127{
128 return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
129}
130
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700131/**
132 * init_bootmem - register boot memory
133 * @start: pfn where the bitmap is to be placed
134 * @pages: number of available physical pages
135 *
136 * Returns the number of bytes needed to hold the bitmap.
137 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700138unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
139{
140 max_low_pfn = pages;
141 min_low_pfn = start;
142 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
143}
144
145static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
146{
Johannes Weiner41546c12008-07-23 21:28:03 -0700147 int aligned;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700148 struct page *page;
Johannes Weiner41546c12008-07-23 21:28:03 -0700149 unsigned long start, end, pages, count = 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700150
Johannes Weiner41546c12008-07-23 21:28:03 -0700151 if (!bdata->node_bootmem_map)
152 return 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700153
Johannes Weiner3560e242008-07-23 21:28:09 -0700154 start = bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700155 end = bdata->node_low_pfn;
156
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700157 /*
Johannes Weiner41546c12008-07-23 21:28:03 -0700158 * If the start is aligned to the machines wordsize, we might
159 * be able to free pages in bulks of that order.
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700160 */
Johannes Weiner41546c12008-07-23 21:28:03 -0700161 aligned = !(start & (BITS_PER_LONG - 1));
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700162
Johannes Weiner41546c12008-07-23 21:28:03 -0700163 bdebug("nid=%td start=%lx end=%lx aligned=%d\n",
164 bdata - bootmem_node_data, start, end, aligned);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700165
Johannes Weiner41546c12008-07-23 21:28:03 -0700166 while (start < end) {
167 unsigned long *map, idx, vec;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700168
Johannes Weiner41546c12008-07-23 21:28:03 -0700169 map = bdata->node_bootmem_map;
Johannes Weiner3560e242008-07-23 21:28:09 -0700170 idx = start - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700171 vec = ~map[idx / BITS_PER_LONG];
172
173 if (aligned && vec == ~0UL && start + BITS_PER_LONG < end) {
174 int order = ilog2(BITS_PER_LONG);
175
176 __free_pages_bootmem(pfn_to_page(start), order);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700177 count += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700178 } else {
Johannes Weiner41546c12008-07-23 21:28:03 -0700179 unsigned long off = 0;
180
181 while (vec && off < BITS_PER_LONG) {
182 if (vec & 1) {
183 page = pfn_to_page(start + off);
184 __free_pages_bootmem(page, 0);
185 count++;
186 }
187 vec >>= 1;
188 off++;
189 }
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700190 }
Johannes Weiner41546c12008-07-23 21:28:03 -0700191 start += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700192 }
193
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700194 page = virt_to_page(bdata->node_bootmem_map);
Johannes Weiner3560e242008-07-23 21:28:09 -0700195 pages = bdata->node_low_pfn - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700196 pages = bootmem_bootmap_pages(pages);
197 count += pages;
198 while (pages--)
199 __free_pages_bootmem(page++, 0);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700200
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700201 bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);
202
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700203 return count;
204}
205
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700206/**
207 * free_all_bootmem_node - release a node's free pages to the buddy allocator
208 * @pgdat: node to be released
209 *
210 * Returns the number of pages actually released.
211 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700212unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
213{
214 register_page_bootmem_info_node(pgdat);
215 return free_all_bootmem_core(pgdat->bdata);
216}
217
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700218/**
219 * free_all_bootmem - release free pages to the buddy allocator
220 *
221 * Returns the number of pages actually released.
222 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700223unsigned long __init free_all_bootmem(void)
224{
225 return free_all_bootmem_core(NODE_DATA(0)->bdata);
226}
227
Johannes Weinerd747fa42008-07-23 21:28:05 -0700228static void __init __free(bootmem_data_t *bdata,
229 unsigned long sidx, unsigned long eidx)
230{
231 unsigned long idx;
232
233 bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700234 sidx + bdata->node_min_pfn,
235 eidx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700236
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700237 if (bdata->hint_idx > sidx)
238 bdata->hint_idx = sidx;
239
Johannes Weinerd747fa42008-07-23 21:28:05 -0700240 for (idx = sidx; idx < eidx; idx++)
241 if (!test_and_clear_bit(idx, bdata->node_bootmem_map))
242 BUG();
243}
244
245static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,
246 unsigned long eidx, int flags)
247{
248 unsigned long idx;
249 int exclusive = flags & BOOTMEM_EXCLUSIVE;
250
251 bdebug("nid=%td start=%lx end=%lx flags=%x\n",
252 bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700253 sidx + bdata->node_min_pfn,
254 eidx + bdata->node_min_pfn,
Johannes Weinerd747fa42008-07-23 21:28:05 -0700255 flags);
256
257 for (idx = sidx; idx < eidx; idx++)
258 if (test_and_set_bit(idx, bdata->node_bootmem_map)) {
259 if (exclusive) {
260 __free(bdata, sidx, idx);
261 return -EBUSY;
262 }
263 bdebug("silent double reserve of PFN %lx\n",
Johannes Weiner3560e242008-07-23 21:28:09 -0700264 idx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700265 }
266 return 0;
267}
268
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700269static int __init mark_bootmem_node(bootmem_data_t *bdata,
270 unsigned long start, unsigned long end,
271 int reserve, int flags)
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700272{
273 unsigned long sidx, eidx;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700274
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700275 bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n",
276 bdata - bootmem_node_data, start, end, reserve, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700277
Johannes Weiner3560e242008-07-23 21:28:09 -0700278 BUG_ON(start < bdata->node_min_pfn);
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700279 BUG_ON(end > bdata->node_low_pfn);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700280
Johannes Weiner3560e242008-07-23 21:28:09 -0700281 sidx = start - bdata->node_min_pfn;
282 eidx = end - bdata->node_min_pfn;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700283
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700284 if (reserve)
285 return __reserve(bdata, sidx, eidx, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700286 else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700287 __free(bdata, sidx, eidx);
288 return 0;
289}
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700290
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700291static int __init mark_bootmem(unsigned long start, unsigned long end,
292 int reserve, int flags)
293{
294 unsigned long pos;
295 bootmem_data_t *bdata;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700296
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700297 pos = start;
298 list_for_each_entry(bdata, &bdata_list, list) {
299 int err;
300 unsigned long max;
301
Johannes Weiner3560e242008-07-23 21:28:09 -0700302 if (pos < bdata->node_min_pfn ||
303 pos >= bdata->node_low_pfn) {
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700304 BUG_ON(pos != start);
305 continue;
306 }
307
308 max = min(bdata->node_low_pfn, end);
309
310 err = mark_bootmem_node(bdata, pos, max, reserve, flags);
311 if (reserve && err) {
312 mark_bootmem(start, pos, 0, 0);
313 return err;
314 }
315
316 if (max == end)
317 return 0;
318 pos = bdata->node_low_pfn;
319 }
320 BUG();
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700321}
322
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700323/**
324 * free_bootmem_node - mark a page range as usable
325 * @pgdat: node the range resides on
326 * @physaddr: starting address of the range
327 * @size: size of the range in bytes
328 *
329 * Partial pages will be considered reserved and left as they are.
330 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700331 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700332 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700333void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
334 unsigned long size)
335{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700336 unsigned long start, end;
337
338 start = PFN_UP(physaddr);
339 end = PFN_DOWN(physaddr + size);
340
341 mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700342}
343
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700344/**
345 * free_bootmem - mark a page range as usable
346 * @addr: starting address of the range
347 * @size: size of the range in bytes
348 *
349 * Partial pages will be considered reserved and left as they are.
350 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700351 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700352 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700353void __init free_bootmem(unsigned long addr, unsigned long size)
354{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700355 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700356
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700357 start = PFN_UP(addr);
358 end = PFN_DOWN(addr + size);
Yinghai Lua5645a62008-03-18 12:49:12 -0700359
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700360 mark_bootmem(start, end, 0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361}
362
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700363/**
364 * reserve_bootmem_node - mark a page range as reserved
365 * @pgdat: node the range resides on
366 * @physaddr: starting address of the range
367 * @size: size of the range in bytes
368 * @flags: reservation flags (see linux/bootmem.h)
369 *
370 * Partial pages will be reserved.
371 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700372 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700373 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700374int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
375 unsigned long size, int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700377 unsigned long start, end;
Franck Bui-Huubbc7b922006-09-25 23:31:07 -0700378
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700379 start = PFN_DOWN(physaddr);
380 end = PFN_UP(physaddr + size);
381
382 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700385/**
386 * reserve_bootmem - mark a page range as usable
387 * @addr: starting address of the range
388 * @size: size of the range in bytes
389 * @flags: reservation flags (see linux/bootmem.h)
390 *
391 * Partial pages will be reserved.
392 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700393 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700394 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700395int __init reserve_bootmem(unsigned long addr, unsigned long size,
396 int flags)
397{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700398 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700399
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700400 start = PFN_DOWN(addr);
401 end = PFN_UP(addr + size);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700402
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700403 return mark_bootmem(start, end, 1, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700404}
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700405
Johannes Weiner481ebd02008-08-20 14:09:15 -0700406static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx,
407 unsigned long step)
408{
409 unsigned long base = bdata->node_min_pfn;
410
411 /*
412 * Align the index with respect to the node start so that the
413 * combination of both satisfies the requested alignment.
414 */
415
416 return ALIGN(base + idx, step) - base;
417}
418
419static unsigned long align_off(struct bootmem_data *bdata, unsigned long off,
420 unsigned long align)
421{
422 unsigned long base = PFN_PHYS(bdata->node_min_pfn);
423
424 /* Same as align_idx for byte offsets */
425
426 return ALIGN(base + off, align) - base;
427}
428
Tejun Heod0c4f572009-03-01 16:06:56 +0900429static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
430 unsigned long size, unsigned long align,
431 unsigned long goal, unsigned long limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700433 unsigned long fallback = 0;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700434 unsigned long min, max, start, sidx, midx, step;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
Johannes Weiner594fe1a2009-01-06 14:40:32 -0800436 bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
437 bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
438 align, goal, limit);
439
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700440 BUG_ON(!size);
441 BUG_ON(align & (align - 1));
442 BUG_ON(limit && goal + size > limit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443
Christian Krafft7c309a62006-12-06 20:32:41 -0800444 if (!bdata->node_bootmem_map)
445 return NULL;
446
Johannes Weiner3560e242008-07-23 21:28:09 -0700447 min = bdata->node_min_pfn;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700448 max = bdata->node_low_pfn;
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700449
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700450 goal >>= PAGE_SHIFT;
451 limit >>= PAGE_SHIFT;
452
453 if (limit && max > limit)
454 max = limit;
455 if (max <= min)
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700456 return NULL;
457
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700458 step = max(align >> PAGE_SHIFT, 1UL);
Yasunori Goto281dd252005-10-19 15:52:18 -0700459
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700460 if (goal && min < goal && goal < max)
461 start = ALIGN(goal, step);
462 else
463 start = ALIGN(min, step);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
Johannes Weiner481ebd02008-08-20 14:09:15 -0700465 sidx = start - bdata->node_min_pfn;
Johannes Weiner3560e242008-07-23 21:28:09 -0700466 midx = max - bdata->node_min_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700468 if (bdata->hint_idx > sidx) {
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700469 /*
470 * Handle the valid case of sidx being zero and still
471 * catch the fallback below.
472 */
473 fallback = sidx + 1;
Johannes Weiner481ebd02008-08-20 14:09:15 -0700474 sidx = align_idx(bdata, bdata->hint_idx, step);
Yinghai Luad093152008-03-10 23:23:42 -0700475 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700477 while (1) {
478 int merge;
479 void *region;
480 unsigned long eidx, i, start_off, end_off;
481find_block:
482 sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
Johannes Weiner481ebd02008-08-20 14:09:15 -0700483 sidx = align_idx(bdata, sidx, step);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700484 eidx = sidx + PFN_UP(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700486 if (sidx >= midx || eidx > midx)
Haren Myneni66d43e92005-12-12 00:37:39 -0800487 break;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700488
489 for (i = sidx; i < eidx; i++)
490 if (test_bit(i, bdata->node_bootmem_map)) {
Johannes Weiner481ebd02008-08-20 14:09:15 -0700491 sidx = align_idx(bdata, i, step);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700492 if (sidx == i)
493 sidx += step;
494 goto find_block;
495 }
496
Mikulas Patocka627240a2008-08-15 00:40:17 -0700497 if (bdata->last_end_off & (PAGE_SIZE - 1) &&
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700498 PFN_DOWN(bdata->last_end_off) + 1 == sidx)
Johannes Weiner481ebd02008-08-20 14:09:15 -0700499 start_off = align_off(bdata, bdata->last_end_off, align);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700500 else
501 start_off = PFN_PHYS(sidx);
502
503 merge = PFN_DOWN(start_off) < sidx;
504 end_off = start_off + size;
505
506 bdata->last_end_off = end_off;
507 bdata->hint_idx = PFN_UP(end_off);
508
509 /*
510 * Reserve the area now:
511 */
Johannes Weinerd747fa42008-07-23 21:28:05 -0700512 if (__reserve(bdata, PFN_DOWN(start_off) + merge,
513 PFN_UP(end_off), BOOTMEM_EXCLUSIVE))
514 BUG();
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700515
Johannes Weiner3560e242008-07-23 21:28:09 -0700516 region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
517 start_off);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700518 memset(region, 0, size);
519 return region;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 }
521
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700522 if (fallback) {
Johannes Weiner481ebd02008-08-20 14:09:15 -0700523 sidx = align_idx(bdata, fallback - 1, step);
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700524 fallback = 0;
525 goto find_block;
526 }
527
528 return NULL;
529}
530
Tejun Heod0c4f572009-03-01 16:06:56 +0900531static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
532 unsigned long size, unsigned long align,
533 unsigned long goal, unsigned long limit)
534{
Pekka Enberg441c7e02009-06-10 20:05:53 +0300535 if (WARN_ON_ONCE(slab_is_available()))
536 return kzalloc(size, GFP_NOWAIT);
537
Tejun Heod0c4f572009-03-01 16:06:56 +0900538#ifdef CONFIG_HAVE_ARCH_BOOTMEM
539 bootmem_data_t *p_bdata;
540
541 p_bdata = bootmem_arch_preferred_node(bdata, size, align, goal, limit);
542 if (p_bdata)
543 return alloc_bootmem_core(p_bdata, size, align, goal, limit);
544#endif
545 return NULL;
546}
547
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700548static void * __init ___alloc_bootmem_nopanic(unsigned long size,
549 unsigned long align,
550 unsigned long goal,
551 unsigned long limit)
552{
553 bootmem_data_t *bdata;
Tejun Heod0c4f572009-03-01 16:06:56 +0900554 void *region;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700555
556restart:
Tejun Heod0c4f572009-03-01 16:06:56 +0900557 region = alloc_arch_preferred_bootmem(NULL, size, align, goal, limit);
558 if (region)
559 return region;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700560
Tejun Heod0c4f572009-03-01 16:06:56 +0900561 list_for_each_entry(bdata, &bdata_list, list) {
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700562 if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
563 continue;
Johannes Weiner3560e242008-07-23 21:28:09 -0700564 if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700565 break;
566
567 region = alloc_bootmem_core(bdata, size, align, goal, limit);
568 if (region)
569 return region;
570 }
571
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700572 if (goal) {
573 goal = 0;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700574 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 }
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700576
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578}
579
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700580/**
581 * __alloc_bootmem_nopanic - allocate boot memory without panicking
582 * @size: size of the request in bytes
583 * @align: alignment of the region
584 * @goal: preferred starting address of the region
585 *
586 * The goal is dropped if it can not be satisfied and the allocation will
587 * fall back to memory below @goal.
588 *
589 * Allocation may happen on any node in the system.
590 *
591 * Returns NULL on failure.
592 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700593void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700594 unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700596 return ___alloc_bootmem_nopanic(size, align, goal, 0);
597}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700599static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
600 unsigned long goal, unsigned long limit)
601{
602 void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
603
604 if (mem)
605 return mem;
606 /*
607 * Whoops, we cannot satisfy the allocation request.
608 */
609 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
610 panic("Out of memory");
Andi Kleena8062232006-04-07 19:49:21 +0200611 return NULL;
612}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700614/**
615 * __alloc_bootmem - allocate boot memory
616 * @size: size of the request in bytes
617 * @align: alignment of the region
618 * @goal: preferred starting address of the region
619 *
620 * The goal is dropped if it can not be satisfied and the allocation will
621 * fall back to memory below @goal.
622 *
623 * Allocation may happen on any node in the system.
624 *
625 * The function panics if the request can not be satisfied.
626 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700627void * __init __alloc_bootmem(unsigned long size, unsigned long align,
628 unsigned long goal)
Andi Kleena8062232006-04-07 19:49:21 +0200629{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700630 return ___alloc_bootmem(size, align, goal, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631}
632
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700633static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
634 unsigned long size, unsigned long align,
635 unsigned long goal, unsigned long limit)
636{
637 void *ptr;
638
Tejun Heod0c4f572009-03-01 16:06:56 +0900639 ptr = alloc_arch_preferred_bootmem(bdata, size, align, goal, limit);
640 if (ptr)
641 return ptr;
642
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700643 ptr = alloc_bootmem_core(bdata, size, align, goal, limit);
644 if (ptr)
645 return ptr;
646
647 return ___alloc_bootmem(size, align, goal, limit);
648}
649
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700650/**
651 * __alloc_bootmem_node - allocate boot memory from a specific node
652 * @pgdat: node to allocate from
653 * @size: size of the request in bytes
654 * @align: alignment of the region
655 * @goal: preferred starting address of the region
656 *
657 * The goal is dropped if it can not be satisfied and the allocation will
658 * fall back to memory below @goal.
659 *
660 * Allocation may fall back to any node in the system if the specified node
661 * can not hold the requested memory.
662 *
663 * The function panics if the request can not be satisfied.
664 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700665void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
666 unsigned long align, unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667{
Pekka Enbergc91c4772009-06-11 08:10:28 +0300668 if (WARN_ON_ONCE(slab_is_available()))
669 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
670
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700671 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672}
673
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700674#ifdef CONFIG_SPARSEMEM
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700675/**
676 * alloc_bootmem_section - allocate boot memory from a specific section
677 * @size: size of the request in bytes
678 * @section_nr: sparse map section to allocate from
679 *
680 * Return NULL on failure.
681 */
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700682void * __init alloc_bootmem_section(unsigned long size,
683 unsigned long section_nr)
684{
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700685 bootmem_data_t *bdata;
686 unsigned long pfn, goal, limit;
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700687
688 pfn = section_nr_to_pfn(section_nr);
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700689 goal = pfn << PAGE_SHIFT;
690 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
691 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700692
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700693 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700694}
695#endif
696
Andi Kleenb54bbf72008-07-23 21:27:45 -0700697void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
698 unsigned long align, unsigned long goal)
699{
700 void *ptr;
701
Pekka Enbergc91c4772009-06-11 08:10:28 +0300702 if (WARN_ON_ONCE(slab_is_available()))
703 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
704
Tejun Heod0c4f572009-03-01 16:06:56 +0900705 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
706 if (ptr)
707 return ptr;
708
Andi Kleenb54bbf72008-07-23 21:27:45 -0700709 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
710 if (ptr)
711 return ptr;
712
713 return __alloc_bootmem_nopanic(size, align, goal);
714}
715
Heiko Carstensdfd54cb2006-09-25 23:31:33 -0700716#ifndef ARCH_LOW_ADDRESS_LIMIT
717#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
718#endif
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800719
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700720/**
721 * __alloc_bootmem_low - allocate low boot memory
722 * @size: size of the request in bytes
723 * @align: alignment of the region
724 * @goal: preferred starting address of the region
725 *
726 * The goal is dropped if it can not be satisfied and the allocation will
727 * fall back to memory below @goal.
728 *
729 * Allocation may happen on any node in the system.
730 *
731 * The function panics if the request can not be satisfied.
732 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700733void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
734 unsigned long goal)
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800735{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700736 return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800737}
738
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700739/**
740 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
741 * @pgdat: node to allocate from
742 * @size: size of the request in bytes
743 * @align: alignment of the region
744 * @goal: preferred starting address of the region
745 *
746 * The goal is dropped if it can not be satisfied and the allocation will
747 * fall back to memory below @goal.
748 *
749 * Allocation may fall back to any node in the system if the specified node
750 * can not hold the requested memory.
751 *
752 * The function panics if the request can not be satisfied.
753 */
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800754void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
755 unsigned long align, unsigned long goal)
756{
Pekka Enbergc91c4772009-06-11 08:10:28 +0300757 if (WARN_ON_ONCE(slab_is_available()))
758 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
759
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700760 return ___alloc_bootmem_node(pgdat->bdata, size, align,
761 goal, ARCH_LOW_ADDRESS_LIMIT);
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800762}