blob: ad8eec6e44a85a9593c781ce252b127cb96992cd [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Johannes Weiner57cfc292008-07-23 21:28:00 -07002 * bootmem - A boot-time physical memory allocator and configurator
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 1999 Ingo Molnar
Johannes Weiner57cfc292008-07-23 21:28:00 -07005 * 1999 Kanoj Sarcar, SGI
6 * 2008 Johannes Weiner
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Johannes Weiner57cfc292008-07-23 21:28:00 -07008 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070012#include <linux/pfn.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/module.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070015
16#include <asm/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <asm/io.h>
Heiko Carstensdfd54cb2006-09-25 23:31:33 -070018#include <asm/processor.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070019
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include "internal.h"
21
Linus Torvalds1da177e2005-04-16 15:20:36 -070022unsigned long max_low_pfn;
23unsigned long min_low_pfn;
24unsigned long max_pfn;
25
Vivek Goyal92aa63a2005-06-25 14:58:18 -070026#ifdef CONFIG_CRASH_DUMP
27/*
28 * If we have booted due to a crash, max_pfn will be a very low value. We need
29 * to know the amount of memory that the previous kernel used.
30 */
31unsigned long saved_max_pfn;
32#endif
33
Johannes Weinerb61bfa32008-07-23 21:26:55 -070034bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
35
Johannes Weiner636cc402008-07-23 21:28:03 -070036static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
37
Johannes Weiner2e5237d2008-07-23 21:28:02 -070038static int bootmem_debug;
39
40static int __init bootmem_debug_setup(char *buf)
41{
42 bootmem_debug = 1;
43 return 0;
44}
45early_param("bootmem_debug", bootmem_debug_setup);
46
47#define bdebug(fmt, args...) ({ \
48 if (unlikely(bootmem_debug)) \
49 printk(KERN_INFO \
50 "bootmem::%s " fmt, \
51 __FUNCTION__, ## args); \
52})
53
Johannes Weinerdf049a52008-07-23 21:28:02 -070054static unsigned long __init bootmap_bytes(unsigned long pages)
Johannes Weiner223e8dc2008-07-23 21:28:00 -070055{
Johannes Weinerdf049a52008-07-23 21:28:02 -070056 unsigned long bytes = (pages + 7) / 8;
Johannes Weiner223e8dc2008-07-23 21:28:00 -070057
Johannes Weinerdf049a52008-07-23 21:28:02 -070058 return ALIGN(bytes, sizeof(long));
Johannes Weiner223e8dc2008-07-23 21:28:00 -070059}
60
Johannes Weinera66fd7d2008-07-23 21:28:01 -070061/**
62 * bootmem_bootmap_pages - calculate bitmap size in pages
63 * @pages: number of pages the bitmap has to represent
64 */
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070065unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066{
Johannes Weinerdf049a52008-07-23 21:28:02 -070067 unsigned long bytes = bootmap_bytes(pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
Johannes Weinerdf049a52008-07-23 21:28:02 -070069 return PAGE_ALIGN(bytes) >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070}
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070071
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080072/*
73 * link bdata in order
74 */
Franck Bui-Huu69d49e62006-09-25 23:31:04 -070075static void __init link_bootmem(bootmem_data_t *bdata)
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080076{
Johannes Weiner636cc402008-07-23 21:28:03 -070077 struct list_head *iter;
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070078
Johannes Weiner636cc402008-07-23 21:28:03 -070079 list_for_each(iter, &bdata_list) {
80 bootmem_data_t *ent;
81
82 ent = list_entry(iter, bootmem_data_t, list);
Johannes Weiner3560e242008-07-23 21:28:09 -070083 if (bdata->node_min_pfn < ent->node_min_pfn)
Johannes Weiner636cc402008-07-23 21:28:03 -070084 break;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080085 }
Johannes Weiner636cc402008-07-23 21:28:03 -070086 list_add_tail(&bdata->list, iter);
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080087}
88
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070089/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 * Called once to set up the allocator itself.
91 */
Johannes Weiner8ae04462008-07-23 21:26:59 -070092static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 unsigned long mapstart, unsigned long start, unsigned long end)
94{
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070095 unsigned long mapsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Mel Gorman2dbb51c2008-07-23 21:26:52 -070097 mminit_validate_memmodel_limits(&start, &end);
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070098 bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
Johannes Weiner3560e242008-07-23 21:28:09 -070099 bdata->node_min_pfn = start;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 bdata->node_low_pfn = end;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -0800101 link_bootmem(bdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
103 /*
104 * Initially all pages are reserved - setup_arch() has to
105 * register free RAM areas explicitly.
106 */
Johannes Weinerdf049a52008-07-23 21:28:02 -0700107 mapsize = bootmap_bytes(end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 memset(bdata->node_bootmem_map, 0xff, mapsize);
109
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700110 bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n",
111 bdata - bootmem_node_data, start, mapstart, end, mapsize);
112
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 return mapsize;
114}
115
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700116/**
117 * init_bootmem_node - register a node as boot memory
118 * @pgdat: node to register
119 * @freepfn: pfn where the bitmap for this node is to be placed
120 * @startpfn: first pfn on the node
121 * @endpfn: first pfn after the node
122 *
123 * Returns the number of bytes needed to hold the bitmap for this node.
124 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700125unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
126 unsigned long startpfn, unsigned long endpfn)
127{
128 return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
129}
130
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700131/**
132 * init_bootmem - register boot memory
133 * @start: pfn where the bitmap is to be placed
134 * @pages: number of available physical pages
135 *
136 * Returns the number of bytes needed to hold the bitmap.
137 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700138unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
139{
140 max_low_pfn = pages;
141 min_low_pfn = start;
142 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
143}
144
145static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
146{
Johannes Weiner41546c12008-07-23 21:28:03 -0700147 int aligned;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700148 struct page *page;
Johannes Weiner41546c12008-07-23 21:28:03 -0700149 unsigned long start, end, pages, count = 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700150
Johannes Weiner41546c12008-07-23 21:28:03 -0700151 if (!bdata->node_bootmem_map)
152 return 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700153
Johannes Weiner3560e242008-07-23 21:28:09 -0700154 start = bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700155 end = bdata->node_low_pfn;
156
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700157 /*
Johannes Weiner41546c12008-07-23 21:28:03 -0700158 * If the start is aligned to the machines wordsize, we might
159 * be able to free pages in bulks of that order.
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700160 */
Johannes Weiner41546c12008-07-23 21:28:03 -0700161 aligned = !(start & (BITS_PER_LONG - 1));
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700162
Johannes Weiner41546c12008-07-23 21:28:03 -0700163 bdebug("nid=%td start=%lx end=%lx aligned=%d\n",
164 bdata - bootmem_node_data, start, end, aligned);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700165
Johannes Weiner41546c12008-07-23 21:28:03 -0700166 while (start < end) {
167 unsigned long *map, idx, vec;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700168
Johannes Weiner41546c12008-07-23 21:28:03 -0700169 map = bdata->node_bootmem_map;
Johannes Weiner3560e242008-07-23 21:28:09 -0700170 idx = start - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700171 vec = ~map[idx / BITS_PER_LONG];
172
173 if (aligned && vec == ~0UL && start + BITS_PER_LONG < end) {
174 int order = ilog2(BITS_PER_LONG);
175
176 __free_pages_bootmem(pfn_to_page(start), order);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700177 count += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700178 } else {
Johannes Weiner41546c12008-07-23 21:28:03 -0700179 unsigned long off = 0;
180
181 while (vec && off < BITS_PER_LONG) {
182 if (vec & 1) {
183 page = pfn_to_page(start + off);
184 __free_pages_bootmem(page, 0);
185 count++;
186 }
187 vec >>= 1;
188 off++;
189 }
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700190 }
Johannes Weiner41546c12008-07-23 21:28:03 -0700191 start += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700192 }
193
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700194 page = virt_to_page(bdata->node_bootmem_map);
Johannes Weiner3560e242008-07-23 21:28:09 -0700195 pages = bdata->node_low_pfn - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700196 pages = bootmem_bootmap_pages(pages);
197 count += pages;
198 while (pages--)
199 __free_pages_bootmem(page++, 0);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700200
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700201 bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);
202
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700203 return count;
204}
205
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700206/**
207 * free_all_bootmem_node - release a node's free pages to the buddy allocator
208 * @pgdat: node to be released
209 *
210 * Returns the number of pages actually released.
211 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700212unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
213{
214 register_page_bootmem_info_node(pgdat);
215 return free_all_bootmem_core(pgdat->bdata);
216}
217
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700218/**
219 * free_all_bootmem - release free pages to the buddy allocator
220 *
221 * Returns the number of pages actually released.
222 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700223unsigned long __init free_all_bootmem(void)
224{
225 return free_all_bootmem_core(NODE_DATA(0)->bdata);
226}
227
Johannes Weinerd747fa42008-07-23 21:28:05 -0700228static void __init __free(bootmem_data_t *bdata,
229 unsigned long sidx, unsigned long eidx)
230{
231 unsigned long idx;
232
233 bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700234 sidx + bdata->node_min_pfn,
235 eidx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700236
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700237 if (bdata->hint_idx > sidx)
238 bdata->hint_idx = sidx;
239
Johannes Weinerd747fa42008-07-23 21:28:05 -0700240 for (idx = sidx; idx < eidx; idx++)
241 if (!test_and_clear_bit(idx, bdata->node_bootmem_map))
242 BUG();
243}
244
245static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,
246 unsigned long eidx, int flags)
247{
248 unsigned long idx;
249 int exclusive = flags & BOOTMEM_EXCLUSIVE;
250
251 bdebug("nid=%td start=%lx end=%lx flags=%x\n",
252 bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700253 sidx + bdata->node_min_pfn,
254 eidx + bdata->node_min_pfn,
Johannes Weinerd747fa42008-07-23 21:28:05 -0700255 flags);
256
257 for (idx = sidx; idx < eidx; idx++)
258 if (test_and_set_bit(idx, bdata->node_bootmem_map)) {
259 if (exclusive) {
260 __free(bdata, sidx, idx);
261 return -EBUSY;
262 }
263 bdebug("silent double reserve of PFN %lx\n",
Johannes Weiner3560e242008-07-23 21:28:09 -0700264 idx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700265 }
266 return 0;
267}
268
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700269static int __init mark_bootmem_node(bootmem_data_t *bdata,
270 unsigned long start, unsigned long end,
271 int reserve, int flags)
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700272{
273 unsigned long sidx, eidx;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700274
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700275 bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n",
276 bdata - bootmem_node_data, start, end, reserve, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700277
Johannes Weiner3560e242008-07-23 21:28:09 -0700278 BUG_ON(start < bdata->node_min_pfn);
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700279 BUG_ON(end > bdata->node_low_pfn);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700280
Johannes Weiner3560e242008-07-23 21:28:09 -0700281 sidx = start - bdata->node_min_pfn;
282 eidx = end - bdata->node_min_pfn;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700283
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700284 if (reserve)
285 return __reserve(bdata, sidx, eidx, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700286 else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700287 __free(bdata, sidx, eidx);
288 return 0;
289}
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700290
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700291static int __init mark_bootmem(unsigned long start, unsigned long end,
292 int reserve, int flags)
293{
294 unsigned long pos;
295 bootmem_data_t *bdata;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700296
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700297 pos = start;
298 list_for_each_entry(bdata, &bdata_list, list) {
299 int err;
300 unsigned long max;
301
Johannes Weiner3560e242008-07-23 21:28:09 -0700302 if (pos < bdata->node_min_pfn ||
303 pos >= bdata->node_low_pfn) {
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700304 BUG_ON(pos != start);
305 continue;
306 }
307
308 max = min(bdata->node_low_pfn, end);
309
310 err = mark_bootmem_node(bdata, pos, max, reserve, flags);
311 if (reserve && err) {
312 mark_bootmem(start, pos, 0, 0);
313 return err;
314 }
315
316 if (max == end)
317 return 0;
318 pos = bdata->node_low_pfn;
319 }
320 BUG();
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700321}
322
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700323/**
324 * free_bootmem_node - mark a page range as usable
325 * @pgdat: node the range resides on
326 * @physaddr: starting address of the range
327 * @size: size of the range in bytes
328 *
329 * Partial pages will be considered reserved and left as they are.
330 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700331 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700332 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700333void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
334 unsigned long size)
335{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700336 unsigned long start, end;
337
338 start = PFN_UP(physaddr);
339 end = PFN_DOWN(physaddr + size);
340
341 mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700342}
343
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700344/**
345 * free_bootmem - mark a page range as usable
346 * @addr: starting address of the range
347 * @size: size of the range in bytes
348 *
349 * Partial pages will be considered reserved and left as they are.
350 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700351 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700352 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700353void __init free_bootmem(unsigned long addr, unsigned long size)
354{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700355 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700356
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700357 start = PFN_UP(addr);
358 end = PFN_DOWN(addr + size);
Yinghai Lua5645a62008-03-18 12:49:12 -0700359
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700360 mark_bootmem(start, end, 0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361}
362
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700363/**
364 * reserve_bootmem_node - mark a page range as reserved
365 * @pgdat: node the range resides on
366 * @physaddr: starting address of the range
367 * @size: size of the range in bytes
368 * @flags: reservation flags (see linux/bootmem.h)
369 *
370 * Partial pages will be reserved.
371 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700372 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700373 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700374int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
375 unsigned long size, int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700377 unsigned long start, end;
Franck Bui-Huubbc7b922006-09-25 23:31:07 -0700378
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700379 start = PFN_DOWN(physaddr);
380 end = PFN_UP(physaddr + size);
381
382 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700385#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700386/**
387 * reserve_bootmem - mark a page range as usable
388 * @addr: starting address of the range
389 * @size: size of the range in bytes
390 * @flags: reservation flags (see linux/bootmem.h)
391 *
392 * Partial pages will be reserved.
393 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700394 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700395 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700396int __init reserve_bootmem(unsigned long addr, unsigned long size,
397 int flags)
398{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700399 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700400
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700401 start = PFN_DOWN(addr);
402 end = PFN_UP(addr + size);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700403
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700404 return mark_bootmem(start, end, 1, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700405}
406#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
407
Johannes Weiner481ebd02008-08-20 14:09:15 -0700408static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx,
409 unsigned long step)
410{
411 unsigned long base = bdata->node_min_pfn;
412
413 /*
414 * Align the index with respect to the node start so that the
415 * combination of both satisfies the requested alignment.
416 */
417
418 return ALIGN(base + idx, step) - base;
419}
420
421static unsigned long align_off(struct bootmem_data *bdata, unsigned long off,
422 unsigned long align)
423{
424 unsigned long base = PFN_PHYS(bdata->node_min_pfn);
425
426 /* Same as align_idx for byte offsets */
427
428 return ALIGN(base + off, align) - base;
429}
430
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700431static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
432 unsigned long size, unsigned long align,
433 unsigned long goal, unsigned long limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700435 unsigned long fallback = 0;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700436 unsigned long min, max, start, sidx, midx, step;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700438 BUG_ON(!size);
439 BUG_ON(align & (align - 1));
440 BUG_ON(limit && goal + size > limit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441
Christian Krafft7c309a62006-12-06 20:32:41 -0800442 if (!bdata->node_bootmem_map)
443 return NULL;
444
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700445 bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
446 bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
447 align, goal, limit);
448
Johannes Weiner3560e242008-07-23 21:28:09 -0700449 min = bdata->node_min_pfn;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700450 max = bdata->node_low_pfn;
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700451
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700452 goal >>= PAGE_SHIFT;
453 limit >>= PAGE_SHIFT;
454
455 if (limit && max > limit)
456 max = limit;
457 if (max <= min)
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700458 return NULL;
459
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700460 step = max(align >> PAGE_SHIFT, 1UL);
Yasunori Goto281dd252005-10-19 15:52:18 -0700461
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700462 if (goal && min < goal && goal < max)
463 start = ALIGN(goal, step);
464 else
465 start = ALIGN(min, step);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466
Johannes Weiner481ebd02008-08-20 14:09:15 -0700467 sidx = start - bdata->node_min_pfn;
Johannes Weiner3560e242008-07-23 21:28:09 -0700468 midx = max - bdata->node_min_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700470 if (bdata->hint_idx > sidx) {
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700471 /*
472 * Handle the valid case of sidx being zero and still
473 * catch the fallback below.
474 */
475 fallback = sidx + 1;
Johannes Weiner481ebd02008-08-20 14:09:15 -0700476 sidx = align_idx(bdata, bdata->hint_idx, step);
Yinghai Luad093152008-03-10 23:23:42 -0700477 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700479 while (1) {
480 int merge;
481 void *region;
482 unsigned long eidx, i, start_off, end_off;
483find_block:
484 sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
Johannes Weiner481ebd02008-08-20 14:09:15 -0700485 sidx = align_idx(bdata, sidx, step);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700486 eidx = sidx + PFN_UP(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700488 if (sidx >= midx || eidx > midx)
Haren Myneni66d43e92005-12-12 00:37:39 -0800489 break;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700490
491 for (i = sidx; i < eidx; i++)
492 if (test_bit(i, bdata->node_bootmem_map)) {
Johannes Weiner481ebd02008-08-20 14:09:15 -0700493 sidx = align_idx(bdata, i, step);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700494 if (sidx == i)
495 sidx += step;
496 goto find_block;
497 }
498
Mikulas Patocka627240a2008-08-15 00:40:17 -0700499 if (bdata->last_end_off & (PAGE_SIZE - 1) &&
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700500 PFN_DOWN(bdata->last_end_off) + 1 == sidx)
Johannes Weiner481ebd02008-08-20 14:09:15 -0700501 start_off = align_off(bdata, bdata->last_end_off, align);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700502 else
503 start_off = PFN_PHYS(sidx);
504
505 merge = PFN_DOWN(start_off) < sidx;
506 end_off = start_off + size;
507
508 bdata->last_end_off = end_off;
509 bdata->hint_idx = PFN_UP(end_off);
510
511 /*
512 * Reserve the area now:
513 */
Johannes Weinerd747fa42008-07-23 21:28:05 -0700514 if (__reserve(bdata, PFN_DOWN(start_off) + merge,
515 PFN_UP(end_off), BOOTMEM_EXCLUSIVE))
516 BUG();
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700517
Johannes Weiner3560e242008-07-23 21:28:09 -0700518 region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
519 start_off);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700520 memset(region, 0, size);
521 return region;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 }
523
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700524 if (fallback) {
Johannes Weiner481ebd02008-08-20 14:09:15 -0700525 sidx = align_idx(bdata, fallback - 1, step);
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700526 fallback = 0;
527 goto find_block;
528 }
529
530 return NULL;
531}
532
533static void * __init ___alloc_bootmem_nopanic(unsigned long size,
534 unsigned long align,
535 unsigned long goal,
536 unsigned long limit)
537{
538 bootmem_data_t *bdata;
539
540restart:
541 list_for_each_entry(bdata, &bdata_list, list) {
542 void *region;
543
544 if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
545 continue;
Johannes Weiner3560e242008-07-23 21:28:09 -0700546 if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700547 break;
548
549 region = alloc_bootmem_core(bdata, size, align, goal, limit);
550 if (region)
551 return region;
552 }
553
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700554 if (goal) {
555 goal = 0;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700556 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 }
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700558
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560}
561
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700562/**
563 * __alloc_bootmem_nopanic - allocate boot memory without panicking
564 * @size: size of the request in bytes
565 * @align: alignment of the region
566 * @goal: preferred starting address of the region
567 *
568 * The goal is dropped if it can not be satisfied and the allocation will
569 * fall back to memory below @goal.
570 *
571 * Allocation may happen on any node in the system.
572 *
573 * Returns NULL on failure.
574 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700575void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700576 unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700578 return ___alloc_bootmem_nopanic(size, align, goal, 0);
579}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700581static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
582 unsigned long goal, unsigned long limit)
583{
584 void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
585
586 if (mem)
587 return mem;
588 /*
589 * Whoops, we cannot satisfy the allocation request.
590 */
591 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
592 panic("Out of memory");
Andi Kleena8062232006-04-07 19:49:21 +0200593 return NULL;
594}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700596/**
597 * __alloc_bootmem - allocate boot memory
598 * @size: size of the request in bytes
599 * @align: alignment of the region
600 * @goal: preferred starting address of the region
601 *
602 * The goal is dropped if it can not be satisfied and the allocation will
603 * fall back to memory below @goal.
604 *
605 * Allocation may happen on any node in the system.
606 *
607 * The function panics if the request can not be satisfied.
608 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700609void * __init __alloc_bootmem(unsigned long size, unsigned long align,
610 unsigned long goal)
Andi Kleena8062232006-04-07 19:49:21 +0200611{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700612 return ___alloc_bootmem(size, align, goal, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613}
614
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700615static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
616 unsigned long size, unsigned long align,
617 unsigned long goal, unsigned long limit)
618{
619 void *ptr;
620
621 ptr = alloc_bootmem_core(bdata, size, align, goal, limit);
622 if (ptr)
623 return ptr;
624
625 return ___alloc_bootmem(size, align, goal, limit);
626}
627
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700628/**
629 * __alloc_bootmem_node - allocate boot memory from a specific node
630 * @pgdat: node to allocate from
631 * @size: size of the request in bytes
632 * @align: alignment of the region
633 * @goal: preferred starting address of the region
634 *
635 * The goal is dropped if it can not be satisfied and the allocation will
636 * fall back to memory below @goal.
637 *
638 * Allocation may fall back to any node in the system if the specified node
639 * can not hold the requested memory.
640 *
641 * The function panics if the request can not be satisfied.
642 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700643void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
644 unsigned long align, unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645{
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700646 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647}
648
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700649#ifdef CONFIG_SPARSEMEM
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700650/**
651 * alloc_bootmem_section - allocate boot memory from a specific section
652 * @size: size of the request in bytes
653 * @section_nr: sparse map section to allocate from
654 *
655 * Return NULL on failure.
656 */
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700657void * __init alloc_bootmem_section(unsigned long size,
658 unsigned long section_nr)
659{
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700660 bootmem_data_t *bdata;
661 unsigned long pfn, goal, limit;
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700662
663 pfn = section_nr_to_pfn(section_nr);
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700664 goal = pfn << PAGE_SHIFT;
665 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
666 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700667
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700668 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700669}
670#endif
671
Andi Kleenb54bbf72008-07-23 21:27:45 -0700672void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
673 unsigned long align, unsigned long goal)
674{
675 void *ptr;
676
677 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
678 if (ptr)
679 return ptr;
680
681 return __alloc_bootmem_nopanic(size, align, goal);
682}
683
Heiko Carstensdfd54cb2006-09-25 23:31:33 -0700684#ifndef ARCH_LOW_ADDRESS_LIMIT
685#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
686#endif
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800687
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700688/**
689 * __alloc_bootmem_low - allocate low boot memory
690 * @size: size of the request in bytes
691 * @align: alignment of the region
692 * @goal: preferred starting address of the region
693 *
694 * The goal is dropped if it can not be satisfied and the allocation will
695 * fall back to memory below @goal.
696 *
697 * Allocation may happen on any node in the system.
698 *
699 * The function panics if the request can not be satisfied.
700 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700701void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
702 unsigned long goal)
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800703{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700704 return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800705}
706
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700707/**
708 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
709 * @pgdat: node to allocate from
710 * @size: size of the request in bytes
711 * @align: alignment of the region
712 * @goal: preferred starting address of the region
713 *
714 * The goal is dropped if it can not be satisfied and the allocation will
715 * fall back to memory below @goal.
716 *
717 * Allocation may fall back to any node in the system if the specified node
718 * can not hold the requested memory.
719 *
720 * The function panics if the request can not be satisfied.
721 */
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800722void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
723 unsigned long align, unsigned long goal)
724{
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700725 return ___alloc_bootmem_node(pgdat->bdata, size, align,
726 goal, ARCH_LOW_ADDRESS_LIMIT);
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800727}