blob: e023c68b025555b6110e8d6352f9ad19e5c7f7f7 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Johannes Weiner57cfc292008-07-23 21:28:00 -07002 * bootmem - A boot-time physical memory allocator and configurator
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 1999 Ingo Molnar
Johannes Weiner57cfc292008-07-23 21:28:00 -07005 * 1999 Kanoj Sarcar, SGI
6 * 2008 Johannes Weiner
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Johannes Weiner57cfc292008-07-23 21:28:00 -07008 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070012#include <linux/pfn.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/module.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070015
16#include <asm/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <asm/io.h>
Heiko Carstensdfd54cb2006-09-25 23:31:33 -070018#include <asm/processor.h>
Franck Bui-Huue786e862006-09-25 23:31:06 -070019
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include "internal.h"
21
Linus Torvalds1da177e2005-04-16 15:20:36 -070022unsigned long max_low_pfn;
23unsigned long min_low_pfn;
24unsigned long max_pfn;
25
Vivek Goyal92aa63a2005-06-25 14:58:18 -070026#ifdef CONFIG_CRASH_DUMP
27/*
28 * If we have booted due to a crash, max_pfn will be a very low value. We need
29 * to know the amount of memory that the previous kernel used.
30 */
31unsigned long saved_max_pfn;
32#endif
33
Johannes Weinerb61bfa32008-07-23 21:26:55 -070034bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
35
Johannes Weiner636cc402008-07-23 21:28:03 -070036static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
37
Johannes Weiner2e5237d2008-07-23 21:28:02 -070038static int bootmem_debug;
39
40static int __init bootmem_debug_setup(char *buf)
41{
42 bootmem_debug = 1;
43 return 0;
44}
45early_param("bootmem_debug", bootmem_debug_setup);
46
47#define bdebug(fmt, args...) ({ \
48 if (unlikely(bootmem_debug)) \
49 printk(KERN_INFO \
50 "bootmem::%s " fmt, \
51 __FUNCTION__, ## args); \
52})
53
Johannes Weinerdf049a52008-07-23 21:28:02 -070054static unsigned long __init bootmap_bytes(unsigned long pages)
Johannes Weiner223e8dc2008-07-23 21:28:00 -070055{
Johannes Weinerdf049a52008-07-23 21:28:02 -070056 unsigned long bytes = (pages + 7) / 8;
Johannes Weiner223e8dc2008-07-23 21:28:00 -070057
Johannes Weinerdf049a52008-07-23 21:28:02 -070058 return ALIGN(bytes, sizeof(long));
Johannes Weiner223e8dc2008-07-23 21:28:00 -070059}
60
Johannes Weinera66fd7d2008-07-23 21:28:01 -070061/**
62 * bootmem_bootmap_pages - calculate bitmap size in pages
63 * @pages: number of pages the bitmap has to represent
64 */
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070065unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066{
Johannes Weinerdf049a52008-07-23 21:28:02 -070067 unsigned long bytes = bootmap_bytes(pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
Johannes Weinerdf049a52008-07-23 21:28:02 -070069 return PAGE_ALIGN(bytes) >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070}
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070071
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080072/*
73 * link bdata in order
74 */
Franck Bui-Huu69d49e62006-09-25 23:31:04 -070075static void __init link_bootmem(bootmem_data_t *bdata)
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080076{
Johannes Weiner636cc402008-07-23 21:28:03 -070077 struct list_head *iter;
Franck Bui-Huuf71bf0c2006-09-25 23:31:08 -070078
Johannes Weiner636cc402008-07-23 21:28:03 -070079 list_for_each(iter, &bdata_list) {
80 bootmem_data_t *ent;
81
82 ent = list_entry(iter, bootmem_data_t, list);
Johannes Weiner3560e242008-07-23 21:28:09 -070083 if (bdata->node_min_pfn < ent->node_min_pfn)
Johannes Weiner636cc402008-07-23 21:28:03 -070084 break;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080085 }
Johannes Weiner636cc402008-07-23 21:28:03 -070086 list_add_tail(&bdata->list, iter);
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -080087}
88
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070089/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 * Called once to set up the allocator itself.
91 */
Johannes Weiner8ae04462008-07-23 21:26:59 -070092static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 unsigned long mapstart, unsigned long start, unsigned long end)
94{
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070095 unsigned long mapsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Mel Gorman2dbb51c2008-07-23 21:26:52 -070097 mminit_validate_memmodel_limits(&start, &end);
Franck Bui-Huubbc7b922006-09-25 23:31:07 -070098 bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
Johannes Weiner3560e242008-07-23 21:28:09 -070099 bdata->node_min_pfn = start;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 bdata->node_low_pfn = end;
KAMEZAWA Hiroyuki679bc9f2006-03-27 01:15:58 -0800101 link_bootmem(bdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
103 /*
104 * Initially all pages are reserved - setup_arch() has to
105 * register free RAM areas explicitly.
106 */
Johannes Weinerdf049a52008-07-23 21:28:02 -0700107 mapsize = bootmap_bytes(end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 memset(bdata->node_bootmem_map, 0xff, mapsize);
109
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700110 bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n",
111 bdata - bootmem_node_data, start, mapstart, end, mapsize);
112
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 return mapsize;
114}
115
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700116/**
117 * init_bootmem_node - register a node as boot memory
118 * @pgdat: node to register
119 * @freepfn: pfn where the bitmap for this node is to be placed
120 * @startpfn: first pfn on the node
121 * @endpfn: first pfn after the node
122 *
123 * Returns the number of bytes needed to hold the bitmap for this node.
124 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700125unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
126 unsigned long startpfn, unsigned long endpfn)
127{
128 return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
129}
130
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700131/**
132 * init_bootmem - register boot memory
133 * @start: pfn where the bitmap is to be placed
134 * @pages: number of available physical pages
135 *
136 * Returns the number of bytes needed to hold the bitmap.
137 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700138unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
139{
140 max_low_pfn = pages;
141 min_low_pfn = start;
142 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
143}
144
145static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
146{
Johannes Weiner41546c12008-07-23 21:28:03 -0700147 int aligned;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700148 struct page *page;
Johannes Weiner41546c12008-07-23 21:28:03 -0700149 unsigned long start, end, pages, count = 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700150
Johannes Weiner41546c12008-07-23 21:28:03 -0700151 if (!bdata->node_bootmem_map)
152 return 0;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700153
Johannes Weiner3560e242008-07-23 21:28:09 -0700154 start = bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700155 end = bdata->node_low_pfn;
156
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700157 /*
Johannes Weiner41546c12008-07-23 21:28:03 -0700158 * If the start is aligned to the machines wordsize, we might
159 * be able to free pages in bulks of that order.
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700160 */
Johannes Weiner41546c12008-07-23 21:28:03 -0700161 aligned = !(start & (BITS_PER_LONG - 1));
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700162
Johannes Weiner41546c12008-07-23 21:28:03 -0700163 bdebug("nid=%td start=%lx end=%lx aligned=%d\n",
164 bdata - bootmem_node_data, start, end, aligned);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700165
Johannes Weiner41546c12008-07-23 21:28:03 -0700166 while (start < end) {
167 unsigned long *map, idx, vec;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700168
Johannes Weiner41546c12008-07-23 21:28:03 -0700169 map = bdata->node_bootmem_map;
Johannes Weiner3560e242008-07-23 21:28:09 -0700170 idx = start - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700171 vec = ~map[idx / BITS_PER_LONG];
172
173 if (aligned && vec == ~0UL && start + BITS_PER_LONG < end) {
174 int order = ilog2(BITS_PER_LONG);
175
176 __free_pages_bootmem(pfn_to_page(start), order);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700177 count += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700178 } else {
Johannes Weiner41546c12008-07-23 21:28:03 -0700179 unsigned long off = 0;
180
181 while (vec && off < BITS_PER_LONG) {
182 if (vec & 1) {
183 page = pfn_to_page(start + off);
184 __free_pages_bootmem(page, 0);
185 count++;
186 }
187 vec >>= 1;
188 off++;
189 }
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700190 }
Johannes Weiner41546c12008-07-23 21:28:03 -0700191 start += BITS_PER_LONG;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700192 }
193
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700194 page = virt_to_page(bdata->node_bootmem_map);
Johannes Weiner3560e242008-07-23 21:28:09 -0700195 pages = bdata->node_low_pfn - bdata->node_min_pfn;
Johannes Weiner41546c12008-07-23 21:28:03 -0700196 pages = bootmem_bootmap_pages(pages);
197 count += pages;
198 while (pages--)
199 __free_pages_bootmem(page++, 0);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700200
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700201 bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);
202
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700203 return count;
204}
205
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700206/**
207 * free_all_bootmem_node - release a node's free pages to the buddy allocator
208 * @pgdat: node to be released
209 *
210 * Returns the number of pages actually released.
211 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700212unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
213{
214 register_page_bootmem_info_node(pgdat);
215 return free_all_bootmem_core(pgdat->bdata);
216}
217
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700218/**
219 * free_all_bootmem - release free pages to the buddy allocator
220 *
221 * Returns the number of pages actually released.
222 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700223unsigned long __init free_all_bootmem(void)
224{
225 return free_all_bootmem_core(NODE_DATA(0)->bdata);
226}
227
Johannes Weinerd747fa42008-07-23 21:28:05 -0700228static void __init __free(bootmem_data_t *bdata,
229 unsigned long sidx, unsigned long eidx)
230{
231 unsigned long idx;
232
233 bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700234 sidx + bdata->node_min_pfn,
235 eidx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700236
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700237 if (bdata->hint_idx > sidx)
238 bdata->hint_idx = sidx;
239
Johannes Weinerd747fa42008-07-23 21:28:05 -0700240 for (idx = sidx; idx < eidx; idx++)
241 if (!test_and_clear_bit(idx, bdata->node_bootmem_map))
242 BUG();
243}
244
245static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,
246 unsigned long eidx, int flags)
247{
248 unsigned long idx;
249 int exclusive = flags & BOOTMEM_EXCLUSIVE;
250
251 bdebug("nid=%td start=%lx end=%lx flags=%x\n",
252 bdata - bootmem_node_data,
Johannes Weiner3560e242008-07-23 21:28:09 -0700253 sidx + bdata->node_min_pfn,
254 eidx + bdata->node_min_pfn,
Johannes Weinerd747fa42008-07-23 21:28:05 -0700255 flags);
256
257 for (idx = sidx; idx < eidx; idx++)
258 if (test_and_set_bit(idx, bdata->node_bootmem_map)) {
259 if (exclusive) {
260 __free(bdata, sidx, idx);
261 return -EBUSY;
262 }
263 bdebug("silent double reserve of PFN %lx\n",
Johannes Weiner3560e242008-07-23 21:28:09 -0700264 idx + bdata->node_min_pfn);
Johannes Weinerd747fa42008-07-23 21:28:05 -0700265 }
266 return 0;
267}
268
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700269static int __init mark_bootmem_node(bootmem_data_t *bdata,
270 unsigned long start, unsigned long end,
271 int reserve, int flags)
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700272{
273 unsigned long sidx, eidx;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700274
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700275 bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n",
276 bdata - bootmem_node_data, start, end, reserve, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700277
Johannes Weiner3560e242008-07-23 21:28:09 -0700278 BUG_ON(start < bdata->node_min_pfn);
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700279 BUG_ON(end > bdata->node_low_pfn);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700280
Johannes Weiner3560e242008-07-23 21:28:09 -0700281 sidx = start - bdata->node_min_pfn;
282 eidx = end - bdata->node_min_pfn;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700283
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700284 if (reserve)
285 return __reserve(bdata, sidx, eidx, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700286 else
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700287 __free(bdata, sidx, eidx);
288 return 0;
289}
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700290
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700291static int __init mark_bootmem(unsigned long start, unsigned long end,
292 int reserve, int flags)
293{
294 unsigned long pos;
295 bootmem_data_t *bdata;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700296
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700297 pos = start;
298 list_for_each_entry(bdata, &bdata_list, list) {
299 int err;
300 unsigned long max;
301
Johannes Weiner3560e242008-07-23 21:28:09 -0700302 if (pos < bdata->node_min_pfn ||
303 pos >= bdata->node_low_pfn) {
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700304 BUG_ON(pos != start);
305 continue;
306 }
307
308 max = min(bdata->node_low_pfn, end);
309
310 err = mark_bootmem_node(bdata, pos, max, reserve, flags);
311 if (reserve && err) {
312 mark_bootmem(start, pos, 0, 0);
313 return err;
314 }
315
316 if (max == end)
317 return 0;
318 pos = bdata->node_low_pfn;
319 }
320 BUG();
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700321}
322
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700323/**
324 * free_bootmem_node - mark a page range as usable
325 * @pgdat: node the range resides on
326 * @physaddr: starting address of the range
327 * @size: size of the range in bytes
328 *
329 * Partial pages will be considered reserved and left as they are.
330 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700331 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700332 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700333void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
334 unsigned long size)
335{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700336 unsigned long start, end;
337
338 start = PFN_UP(physaddr);
339 end = PFN_DOWN(physaddr + size);
340
341 mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700342}
343
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700344/**
345 * free_bootmem - mark a page range as usable
346 * @addr: starting address of the range
347 * @size: size of the range in bytes
348 *
349 * Partial pages will be considered reserved and left as they are.
350 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700351 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700352 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700353void __init free_bootmem(unsigned long addr, unsigned long size)
354{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700355 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700356
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700357 start = PFN_UP(addr);
358 end = PFN_DOWN(addr + size);
Yinghai Lua5645a62008-03-18 12:49:12 -0700359
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700360 mark_bootmem(start, end, 0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361}
362
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700363/**
364 * reserve_bootmem_node - mark a page range as reserved
365 * @pgdat: node the range resides on
366 * @physaddr: starting address of the range
367 * @size: size of the range in bytes
368 * @flags: reservation flags (see linux/bootmem.h)
369 *
370 * Partial pages will be reserved.
371 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700372 * The range must reside completely on the specified node.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700373 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700374int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
375 unsigned long size, int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700377 unsigned long start, end;
Franck Bui-Huubbc7b922006-09-25 23:31:07 -0700378
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700379 start = PFN_DOWN(physaddr);
380 end = PFN_UP(physaddr + size);
381
382 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700385#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700386/**
387 * reserve_bootmem - mark a page range as usable
388 * @addr: starting address of the range
389 * @size: size of the range in bytes
390 * @flags: reservation flags (see linux/bootmem.h)
391 *
392 * Partial pages will be reserved.
393 *
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700394 * The range must be contiguous but may span node boundaries.
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700395 */
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700396int __init reserve_bootmem(unsigned long addr, unsigned long size,
397 int flags)
398{
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700399 unsigned long start, end;
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700400
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700401 start = PFN_DOWN(addr);
402 end = PFN_UP(addr + size);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700403
Johannes Weinere2bf3ca2008-07-23 21:28:06 -0700404 return mark_bootmem(start, end, 1, flags);
Johannes Weiner223e8dc2008-07-23 21:28:00 -0700405}
406#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
407
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700408static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
409 unsigned long size, unsigned long align,
410 unsigned long goal, unsigned long limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700412 unsigned long fallback = 0;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700413 unsigned long min, max, start, sidx, midx, step;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700415 BUG_ON(!size);
416 BUG_ON(align & (align - 1));
417 BUG_ON(limit && goal + size > limit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418
Christian Krafft7c309a62006-12-06 20:32:41 -0800419 if (!bdata->node_bootmem_map)
420 return NULL;
421
Johannes Weiner2e5237d2008-07-23 21:28:02 -0700422 bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
423 bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
424 align, goal, limit);
425
Johannes Weiner3560e242008-07-23 21:28:09 -0700426 min = bdata->node_min_pfn;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700427 max = bdata->node_low_pfn;
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700428
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700429 goal >>= PAGE_SHIFT;
430 limit >>= PAGE_SHIFT;
431
432 if (limit && max > limit)
433 max = limit;
434 if (max <= min)
Yinghai Lu9a2dc042008-03-18 12:44:48 -0700435 return NULL;
436
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700437 step = max(align >> PAGE_SHIFT, 1UL);
Yasunori Goto281dd252005-10-19 15:52:18 -0700438
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700439 if (goal && min < goal && goal < max)
440 start = ALIGN(goal, step);
441 else
442 start = ALIGN(min, step);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443
Johannes Weiner3560e242008-07-23 21:28:09 -0700444 sidx = start - bdata->node_min_pfn;;
445 midx = max - bdata->node_min_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700447 if (bdata->hint_idx > sidx) {
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700448 /*
449 * Handle the valid case of sidx being zero and still
450 * catch the fallback below.
451 */
452 fallback = sidx + 1;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700453 sidx = ALIGN(bdata->hint_idx, step);
Yinghai Luad093152008-03-10 23:23:42 -0700454 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700456 while (1) {
457 int merge;
458 void *region;
459 unsigned long eidx, i, start_off, end_off;
460find_block:
461 sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
462 sidx = ALIGN(sidx, step);
463 eidx = sidx + PFN_UP(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700465 if (sidx >= midx || eidx > midx)
Haren Myneni66d43e92005-12-12 00:37:39 -0800466 break;
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700467
468 for (i = sidx; i < eidx; i++)
469 if (test_bit(i, bdata->node_bootmem_map)) {
470 sidx = ALIGN(i, step);
471 if (sidx == i)
472 sidx += step;
473 goto find_block;
474 }
475
Mikulas Patocka627240a2008-08-15 00:40:17 -0700476 if (bdata->last_end_off & (PAGE_SIZE - 1) &&
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700477 PFN_DOWN(bdata->last_end_off) + 1 == sidx)
478 start_off = ALIGN(bdata->last_end_off, align);
479 else
480 start_off = PFN_PHYS(sidx);
481
482 merge = PFN_DOWN(start_off) < sidx;
483 end_off = start_off + size;
484
485 bdata->last_end_off = end_off;
486 bdata->hint_idx = PFN_UP(end_off);
487
488 /*
489 * Reserve the area now:
490 */
Johannes Weinerd747fa42008-07-23 21:28:05 -0700491 if (__reserve(bdata, PFN_DOWN(start_off) + merge,
492 PFN_UP(end_off), BOOTMEM_EXCLUSIVE))
493 BUG();
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700494
Johannes Weiner3560e242008-07-23 21:28:09 -0700495 region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
496 start_off);
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700497 memset(region, 0, size);
498 return region;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 }
500
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700501 if (fallback) {
502 sidx = ALIGN(fallback - 1, step);
503 fallback = 0;
504 goto find_block;
505 }
506
507 return NULL;
508}
509
510static void * __init ___alloc_bootmem_nopanic(unsigned long size,
511 unsigned long align,
512 unsigned long goal,
513 unsigned long limit)
514{
515 bootmem_data_t *bdata;
516
517restart:
518 list_for_each_entry(bdata, &bdata_list, list) {
519 void *region;
520
521 if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
522 continue;
Johannes Weiner3560e242008-07-23 21:28:09 -0700523 if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700524 break;
525
526 region = alloc_bootmem_core(bdata, size, align, goal, limit);
527 if (region)
528 return region;
529 }
530
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700531 if (goal) {
532 goal = 0;
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700533 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 }
Johannes Weiner5f2809e2008-07-23 21:28:05 -0700535
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537}
538
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700539/**
540 * __alloc_bootmem_nopanic - allocate boot memory without panicking
541 * @size: size of the request in bytes
542 * @align: alignment of the region
543 * @goal: preferred starting address of the region
544 *
545 * The goal is dropped if it can not be satisfied and the allocation will
546 * fall back to memory below @goal.
547 *
548 * Allocation may happen on any node in the system.
549 *
550 * Returns NULL on failure.
551 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700552void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700553 unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700555 return ___alloc_bootmem_nopanic(size, align, goal, 0);
556}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700558static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
559 unsigned long goal, unsigned long limit)
560{
561 void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
562
563 if (mem)
564 return mem;
565 /*
566 * Whoops, we cannot satisfy the allocation request.
567 */
568 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
569 panic("Out of memory");
Andi Kleena8062232006-04-07 19:49:21 +0200570 return NULL;
571}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700573/**
574 * __alloc_bootmem - allocate boot memory
575 * @size: size of the request in bytes
576 * @align: alignment of the region
577 * @goal: preferred starting address of the region
578 *
579 * The goal is dropped if it can not be satisfied and the allocation will
580 * fall back to memory below @goal.
581 *
582 * Allocation may happen on any node in the system.
583 *
584 * The function panics if the request can not be satisfied.
585 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700586void * __init __alloc_bootmem(unsigned long size, unsigned long align,
587 unsigned long goal)
Andi Kleena8062232006-04-07 19:49:21 +0200588{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700589 return ___alloc_bootmem(size, align, goal, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590}
591
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700592static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
593 unsigned long size, unsigned long align,
594 unsigned long goal, unsigned long limit)
595{
596 void *ptr;
597
598 ptr = alloc_bootmem_core(bdata, size, align, goal, limit);
599 if (ptr)
600 return ptr;
601
602 return ___alloc_bootmem(size, align, goal, limit);
603}
604
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700605/**
606 * __alloc_bootmem_node - allocate boot memory from a specific node
607 * @pgdat: node to allocate from
608 * @size: size of the request in bytes
609 * @align: alignment of the region
610 * @goal: preferred starting address of the region
611 *
612 * The goal is dropped if it can not be satisfied and the allocation will
613 * fall back to memory below @goal.
614 *
615 * Allocation may fall back to any node in the system if the specified node
616 * can not hold the requested memory.
617 *
618 * The function panics if the request can not be satisfied.
619 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700620void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
621 unsigned long align, unsigned long goal)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622{
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700623 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624}
625
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700626#ifdef CONFIG_SPARSEMEM
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700627/**
628 * alloc_bootmem_section - allocate boot memory from a specific section
629 * @size: size of the request in bytes
630 * @section_nr: sparse map section to allocate from
631 *
632 * Return NULL on failure.
633 */
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700634void * __init alloc_bootmem_section(unsigned long size,
635 unsigned long section_nr)
636{
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700637 bootmem_data_t *bdata;
638 unsigned long pfn, goal, limit;
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700639
640 pfn = section_nr_to_pfn(section_nr);
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700641 goal = pfn << PAGE_SHIFT;
642 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
643 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700644
Johannes Weiner75a56cf2008-07-23 21:28:09 -0700645 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
Yasunori Gotoe70260a2008-04-28 02:13:32 -0700646}
647#endif
648
Andi Kleenb54bbf72008-07-23 21:27:45 -0700649void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
650 unsigned long align, unsigned long goal)
651{
652 void *ptr;
653
654 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
655 if (ptr)
656 return ptr;
657
658 return __alloc_bootmem_nopanic(size, align, goal);
659}
660
Heiko Carstensdfd54cb2006-09-25 23:31:33 -0700661#ifndef ARCH_LOW_ADDRESS_LIMIT
662#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
663#endif
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800664
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700665/**
666 * __alloc_bootmem_low - allocate low boot memory
667 * @size: size of the request in bytes
668 * @align: alignment of the region
669 * @goal: preferred starting address of the region
670 *
671 * The goal is dropped if it can not be satisfied and the allocation will
672 * fall back to memory below @goal.
673 *
674 * Allocation may happen on any node in the system.
675 *
676 * The function panics if the request can not be satisfied.
677 */
Franck Bui-Huubb0923a2006-09-25 23:31:05 -0700678void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
679 unsigned long goal)
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800680{
Johannes Weiner0f3caba2008-07-23 21:28:07 -0700681 return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800682}
683
Johannes Weinera66fd7d2008-07-23 21:28:01 -0700684/**
685 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
686 * @pgdat: node to allocate from
687 * @size: size of the request in bytes
688 * @align: alignment of the region
689 * @goal: preferred starting address of the region
690 *
691 * The goal is dropped if it can not be satisfied and the allocation will
692 * fall back to memory below @goal.
693 *
694 * Allocation may fall back to any node in the system if the specified node
695 * can not hold the requested memory.
696 *
697 * The function panics if the request can not be satisfied.
698 */
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800699void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
700 unsigned long align, unsigned long goal)
701{
Johannes Weiner4cc278b2008-07-23 21:28:08 -0700702 return ___alloc_bootmem_node(pgdat->bdata, size, align,
703 goal, ARCH_LOW_ADDRESS_LIMIT);
Ravikiran G Thirumalai008857c2006-01-06 00:11:01 -0800704}