blob: 8cc7be0e95906c18068e926490a8509651e3904c [file] [log] [blame]
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001/*
2 * sparse memory mappings.
3 */
Andy Whitcroftd41dee32005-06-23 00:07:54 -07004#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09005#include <linux/slab.h>
Andy Whitcroftd41dee32005-06-23 00:07:54 -07006#include <linux/mmzone.h>
7#include <linux/bootmem.h>
Dave Hansen0b0acbe2005-10-29 18:16:55 -07008#include <linux/highmem.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -04009#include <linux/export.h>
Dave Hansen28ae55c2005-09-03 15:54:29 -070010#include <linux/spinlock.h>
Dave Hansen0b0acbe2005-10-29 18:16:55 -070011#include <linux/vmalloc.h>
Yasunori Goto0c0a4a52008-04-28 02:13:34 -070012#include "internal.h"
Andy Whitcroftd41dee32005-06-23 00:07:54 -070013#include <asm/dma.h>
Christoph Lameter8f6aac42007-10-16 01:24:13 -070014#include <asm/pgalloc.h>
15#include <asm/pgtable.h>
Andy Whitcroftd41dee32005-06-23 00:07:54 -070016
17/*
18 * Permanent SPARSEMEM data:
19 *
20 * 1) mem_section - memory sections, mem_map's for valid memory
21 */
Bob Picco3e347262005-09-03 15:54:28 -070022#ifdef CONFIG_SPARSEMEM_EXTREME
Bob Picco802f1922005-09-03 15:54:26 -070023struct mem_section *mem_section[NR_SECTION_ROOTS]
Ravikiran G Thirumalai22fc6ec2006-01-08 01:01:27 -080024 ____cacheline_internodealigned_in_smp;
Bob Picco3e347262005-09-03 15:54:28 -070025#else
26struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
Ravikiran G Thirumalai22fc6ec2006-01-08 01:01:27 -080027 ____cacheline_internodealigned_in_smp;
Bob Picco3e347262005-09-03 15:54:28 -070028#endif
29EXPORT_SYMBOL(mem_section);
30
Christoph Lameter89689ae2006-12-06 20:31:45 -080031#ifdef NODE_NOT_IN_PAGE_FLAGS
32/*
33 * If we did not store the node number in the page then we have to
34 * do a lookup in the section_to_node_table in order to find which
35 * node the page belongs to.
36 */
37#if MAX_NUMNODES <= 256
38static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
39#else
40static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
41#endif
42
Ian Campbell33dd4e02011-07-25 17:11:51 -070043int page_to_nid(const struct page *page)
Christoph Lameter89689ae2006-12-06 20:31:45 -080044{
45 return section_to_node_table[page_to_section(page)];
46}
47EXPORT_SYMBOL(page_to_nid);
Andy Whitcroft85770ff2007-08-22 14:01:03 -070048
49static void set_section_nid(unsigned long section_nr, int nid)
50{
51 section_to_node_table[section_nr] = nid;
52}
53#else /* !NODE_NOT_IN_PAGE_FLAGS */
54static inline void set_section_nid(unsigned long section_nr, int nid)
55{
56}
Christoph Lameter89689ae2006-12-06 20:31:45 -080057#endif
58
Bob Picco3e347262005-09-03 15:54:28 -070059#ifdef CONFIG_SPARSEMEM_EXTREME
Sam Ravnborg577a32f2007-05-17 23:29:25 +020060static struct mem_section noinline __init_refok *sparse_index_alloc(int nid)
Bob Picco802f1922005-09-03 15:54:26 -070061{
Dave Hansen28ae55c2005-09-03 15:54:29 -070062 struct mem_section *section = NULL;
63 unsigned long array_size = SECTIONS_PER_ROOT *
64 sizeof(struct mem_section);
Bob Picco802f1922005-09-03 15:54:26 -070065
Shaohua Lif52407c2009-09-21 17:01:19 -070066 if (slab_is_available()) {
67 if (node_state(nid, N_HIGH_MEMORY))
Gavin Shan5b760e62012-07-31 16:46:02 -070068 section = kzalloc_node(array_size, GFP_KERNEL, nid);
Shaohua Lif52407c2009-09-21 17:01:19 -070069 else
Gavin Shan5b760e62012-07-31 16:46:02 -070070 section = kzalloc(array_size, GFP_KERNEL);
71 } else {
Mike Kravetz46a66ee2006-05-01 12:16:09 -070072 section = alloc_bootmem_node(NODE_DATA(nid), array_size);
Gavin Shan5b760e62012-07-31 16:46:02 -070073 }
Bob Picco3e347262005-09-03 15:54:28 -070074
Dave Hansen28ae55c2005-09-03 15:54:29 -070075 return section;
Bob Picco802f1922005-09-03 15:54:26 -070076}
Dave Hansen28ae55c2005-09-03 15:54:29 -070077
Yasunori Gotoa3142c82007-05-08 00:23:07 -070078static int __meminit sparse_index_init(unsigned long section_nr, int nid)
Dave Hansen28ae55c2005-09-03 15:54:29 -070079{
Dave Hansen28ae55c2005-09-03 15:54:29 -070080 unsigned long root = SECTION_NR_TO_ROOT(section_nr);
81 struct mem_section *section;
Dave Hansen28ae55c2005-09-03 15:54:29 -070082
83 if (mem_section[root])
84 return -EEXIST;
85
86 section = sparse_index_alloc(nid);
WANG Congaf0cd5a2007-12-17 16:19:58 -080087 if (!section)
88 return -ENOMEM;
Dave Hansen28ae55c2005-09-03 15:54:29 -070089
90 mem_section[root] = section;
Gavin Shanc1c95182012-07-31 16:46:06 -070091
Zhang Yanfei9d1936c2013-05-17 22:10:38 +080092 return 0;
Dave Hansen28ae55c2005-09-03 15:54:29 -070093}
94#else /* !SPARSEMEM_EXTREME */
95static inline int sparse_index_init(unsigned long section_nr, int nid)
96{
97 return 0;
98}
99#endif
100
Dave Hansen4ca644d2005-10-29 18:16:51 -0700101/*
102 * Although written for the SPARSEMEM_EXTREME case, this happens
Andy Whitcroftcd881a62007-10-16 01:24:10 -0700103 * to also work for the flat array case because
Dave Hansen4ca644d2005-10-29 18:16:51 -0700104 * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
105 */
106int __section_nr(struct mem_section* ms)
107{
108 unsigned long root_nr;
109 struct mem_section* root;
110
Mike Kravetz12783b02006-05-20 15:00:05 -0700111 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {
112 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);
Dave Hansen4ca644d2005-10-29 18:16:51 -0700113 if (!root)
114 continue;
115
116 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))
117 break;
118 }
119
Gavin Shandb36a462012-07-31 16:46:04 -0700120 VM_BUG_ON(root_nr == NR_SECTION_ROOTS);
121
Dave Hansen4ca644d2005-10-29 18:16:51 -0700122 return (root_nr * SECTIONS_PER_ROOT) + (ms - root);
123}
124
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700125/*
126 * During early boot, before section_mem_map is used for an actual
127 * mem_map, we use section_mem_map to store the section's NUMA
128 * node. This keeps us from having to use another data structure. The
129 * node information is cleared just before we store the real mem_map.
130 */
131static inline unsigned long sparse_encode_early_nid(int nid)
132{
133 return (nid << SECTION_NID_SHIFT);
134}
135
136static inline int sparse_early_nid(struct mem_section *section)
137{
138 return (section->section_mem_map >> SECTION_NID_SHIFT);
139}
140
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700141/* Validate the physical addressing limitations of the model */
142void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
143 unsigned long *end_pfn)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700144{
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700145 unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT);
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700146
Ingo Molnarbead9a32008-04-16 01:40:00 +0200147 /*
148 * Sanity checks - do not allow an architecture to pass
149 * in larger pfns than the maximum scope of sparsemem:
150 */
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700151 if (*start_pfn > max_sparsemem_pfn) {
152 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
153 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
154 *start_pfn, *end_pfn, max_sparsemem_pfn);
155 WARN_ON_ONCE(1);
156 *start_pfn = max_sparsemem_pfn;
157 *end_pfn = max_sparsemem_pfn;
Cyrill Gorcunovef161a92009-03-31 15:19:25 -0700158 } else if (*end_pfn > max_sparsemem_pfn) {
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700159 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
160 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
161 *start_pfn, *end_pfn, max_sparsemem_pfn);
162 WARN_ON_ONCE(1);
163 *end_pfn = max_sparsemem_pfn;
164 }
165}
166
167/* Record a memory area against a node. */
168void __init memory_present(int nid, unsigned long start, unsigned long end)
169{
170 unsigned long pfn;
Ingo Molnarbead9a32008-04-16 01:40:00 +0200171
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700172 start &= PAGE_SECTION_MASK;
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700173 mminit_validate_memmodel_limits(&start, &end);
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700174 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
175 unsigned long section = pfn_to_section_nr(pfn);
Bob Picco802f1922005-09-03 15:54:26 -0700176 struct mem_section *ms;
177
178 sparse_index_init(section, nid);
Andy Whitcroft85770ff2007-08-22 14:01:03 -0700179 set_section_nid(section, nid);
Bob Picco802f1922005-09-03 15:54:26 -0700180
181 ms = __nr_to_section(section);
182 if (!ms->section_mem_map)
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700183 ms->section_mem_map = sparse_encode_early_nid(nid) |
184 SECTION_MARKED_PRESENT;
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700185 }
186}
187
188/*
189 * Only used by the i386 NUMA architecures, but relatively
190 * generic code.
191 */
192unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,
193 unsigned long end_pfn)
194{
195 unsigned long pfn;
196 unsigned long nr_pages = 0;
197
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700198 mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700199 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
200 if (nid != early_pfn_to_nid(pfn))
201 continue;
202
Andy Whitcroft540557b2007-10-16 01:24:11 -0700203 if (pfn_present(pfn))
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700204 nr_pages += PAGES_PER_SECTION;
205 }
206
207 return nr_pages * sizeof(struct page);
208}
209
210/*
Andy Whitcroft29751f62005-06-23 00:08:00 -0700211 * Subtle, we encode the real pfn into the mem_map such that
212 * the identity pfn - section_mem_map will return the actual
213 * physical page frame number.
214 */
215static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
216{
217 return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
218}
219
220/*
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700221 * Decode mem_map from the coded memmap
Andy Whitcroft29751f62005-06-23 00:08:00 -0700222 */
Andy Whitcroft29751f62005-06-23 00:08:00 -0700223struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
224{
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700225 /* mask off the extra low bits of information */
226 coded_mem_map &= SECTION_MAP_MASK;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700227 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
228}
229
Yasunori Gotoa3142c82007-05-08 00:23:07 -0700230static int __meminit sparse_init_one_section(struct mem_section *ms,
Mel Gorman5c0e3062007-10-16 01:25:56 -0700231 unsigned long pnum, struct page *mem_map,
232 unsigned long *pageblock_bitmap)
Andy Whitcroft29751f62005-06-23 00:08:00 -0700233{
Andy Whitcroft540557b2007-10-16 01:24:11 -0700234 if (!present_section(ms))
Andy Whitcroft29751f62005-06-23 00:08:00 -0700235 return -EINVAL;
236
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700237 ms->section_mem_map &= ~SECTION_MAP_MASK;
Andy Whitcroft540557b2007-10-16 01:24:11 -0700238 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |
239 SECTION_HAS_MEM_MAP;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700240 ms->pageblock_flags = pageblock_bitmap;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700241
242 return 1;
243}
244
Yasunori Goto04753272008-04-28 02:13:31 -0700245unsigned long usemap_size(void)
Mel Gorman5c0e3062007-10-16 01:25:56 -0700246{
247 unsigned long size_bytes;
248 size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8;
249 size_bytes = roundup(size_bytes, sizeof(unsigned long));
250 return size_bytes;
251}
252
253#ifdef CONFIG_MEMORY_HOTPLUG
254static unsigned long *__kmalloc_section_usemap(void)
255{
256 return kmalloc(usemap_size(), GFP_KERNEL);
257}
258#endif /* CONFIG_MEMORY_HOTPLUG */
259
Yasunori Goto48c90682008-07-23 21:28:15 -0700260#ifdef CONFIG_MEMORY_HOTREMOVE
261static unsigned long * __init
Yinghai Lua4322e12010-02-10 01:20:21 -0800262sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
Johannes Weiner238305b2012-05-29 15:06:36 -0700263 unsigned long size)
Yasunori Goto48c90682008-07-23 21:28:15 -0700264{
Yinghai Lu99ab7b12012-07-11 14:02:53 -0700265 unsigned long goal, limit;
266 unsigned long *p;
267 int nid;
Yasunori Goto48c90682008-07-23 21:28:15 -0700268 /*
269 * A page may contain usemaps for other sections preventing the
270 * page being freed and making a section unremovable while
271 * other sections referencing the usemap retmain active. Similarly,
272 * a pgdat can prevent a section being removed. If section A
273 * contains a pgdat and section B contains the usemap, both
274 * sections become inter-dependent. This allocates usemaps
275 * from the same section as the pgdat where possible to avoid
276 * this problem.
277 */
Yinghai Lu07b4e2b2012-07-11 14:02:51 -0700278 goal = __pa(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT);
Yinghai Lu99ab7b12012-07-11 14:02:53 -0700279 limit = goal + (1UL << PA_SECTION_SHIFT);
280 nid = early_pfn_to_nid(goal >> PAGE_SHIFT);
281again:
282 p = ___alloc_bootmem_node_nopanic(NODE_DATA(nid), size,
283 SMP_CACHE_BYTES, goal, limit);
284 if (!p && limit) {
285 limit = 0;
286 goto again;
287 }
288 return p;
Yasunori Goto48c90682008-07-23 21:28:15 -0700289}
290
291static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
292{
293 unsigned long usemap_snr, pgdat_snr;
294 static unsigned long old_usemap_snr = NR_MEM_SECTIONS;
295 static unsigned long old_pgdat_snr = NR_MEM_SECTIONS;
296 struct pglist_data *pgdat = NODE_DATA(nid);
297 int usemap_nid;
298
299 usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT);
300 pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);
301 if (usemap_snr == pgdat_snr)
302 return;
303
304 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)
305 /* skip redundant message */
306 return;
307
308 old_usemap_snr = usemap_snr;
309 old_pgdat_snr = pgdat_snr;
310
311 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
312 if (usemap_nid != nid) {
313 printk(KERN_INFO
314 "node %d must be removed before remove section %ld\n",
315 nid, usemap_snr);
316 return;
317 }
318 /*
319 * There is a circular dependency.
320 * Some platforms allow un-removable section because they will just
321 * gather other removable sections for dynamic partitioning.
322 * Just notify un-removable section's number here.
323 */
324 printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr,
325 pgdat_snr, nid);
326 printk(KERN_CONT
327 " have a circular dependency on usemap and pgdat allocations\n");
328}
329#else
330static unsigned long * __init
Yinghai Lua4322e12010-02-10 01:20:21 -0800331sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
Johannes Weiner238305b2012-05-29 15:06:36 -0700332 unsigned long size)
Yasunori Goto48c90682008-07-23 21:28:15 -0700333{
Johannes Weiner238305b2012-05-29 15:06:36 -0700334 return alloc_bootmem_node_nopanic(pgdat, size);
Yasunori Goto48c90682008-07-23 21:28:15 -0700335}
336
337static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
338{
339}
340#endif /* CONFIG_MEMORY_HOTREMOVE */
341
Wanpeng Li18732092013-09-11 14:22:38 -0700342static void __init sparse_early_usemaps_alloc_node(void *data,
Yinghai Lua4322e12010-02-10 01:20:21 -0800343 unsigned long pnum_begin,
344 unsigned long pnum_end,
345 unsigned long usemap_count, int nodeid)
Mel Gorman5c0e3062007-10-16 01:25:56 -0700346{
Yinghai Lua4322e12010-02-10 01:20:21 -0800347 void *usemap;
348 unsigned long pnum;
Wanpeng Li18732092013-09-11 14:22:38 -0700349 unsigned long **usemap_map = (unsigned long **)data;
Yinghai Lua4322e12010-02-10 01:20:21 -0800350 int size = usemap_size();
Mel Gorman5c0e3062007-10-16 01:25:56 -0700351
Yinghai Lua4322e12010-02-10 01:20:21 -0800352 usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
Johannes Weiner238305b2012-05-29 15:06:36 -0700353 size * usemap_count);
Nishanth Aravamudanf5bf18f2012-03-21 16:34:07 -0700354 if (!usemap) {
Johannes Weiner238305b2012-05-29 15:06:36 -0700355 printk(KERN_WARNING "%s: allocation failed\n", __func__);
356 return;
Yasunori Goto48c90682008-07-23 21:28:15 -0700357 }
358
Nishanth Aravamudanf5bf18f2012-03-21 16:34:07 -0700359 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
360 if (!present_section_nr(pnum))
361 continue;
362 usemap_map[pnum] = usemap;
363 usemap += size;
364 check_usemap_section_nr(nodeid, usemap_map[pnum]);
Yinghai Lua4322e12010-02-10 01:20:21 -0800365 }
Mel Gorman5c0e3062007-10-16 01:25:56 -0700366}
367
Christoph Lameter8f6aac42007-10-16 01:24:13 -0700368#ifndef CONFIG_SPARSEMEM_VMEMMAP
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700369struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
Andy Whitcroft29751f62005-06-23 00:08:00 -0700370{
371 struct page *map;
Yinghai Lue48e67e2010-05-24 14:31:57 -0700372 unsigned long size;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700373
374 map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);
375 if (map)
376 return map;
377
Yinghai Lue48e67e2010-05-24 14:31:57 -0700378 size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
379 map = __alloc_bootmem_node_high(NODE_DATA(nid), size,
380 PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
Christoph Lameter8f6aac42007-10-16 01:24:13 -0700381 return map;
382}
Yinghai Lu9bdac912010-02-10 01:20:22 -0800383void __init sparse_mem_maps_populate_node(struct page **map_map,
384 unsigned long pnum_begin,
385 unsigned long pnum_end,
386 unsigned long map_count, int nodeid)
387{
388 void *map;
389 unsigned long pnum;
390 unsigned long size = sizeof(struct page) * PAGES_PER_SECTION;
391
392 map = alloc_remap(nodeid, size * map_count);
393 if (map) {
394 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
395 if (!present_section_nr(pnum))
396 continue;
397 map_map[pnum] = map;
398 map += size;
399 }
400 return;
401 }
402
403 size = PAGE_ALIGN(size);
Yinghai Lue48e67e2010-05-24 14:31:57 -0700404 map = __alloc_bootmem_node_high(NODE_DATA(nodeid), size * map_count,
405 PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
Yinghai Lu9bdac912010-02-10 01:20:22 -0800406 if (map) {
407 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
408 if (!present_section_nr(pnum))
409 continue;
410 map_map[pnum] = map;
411 map += size;
412 }
413 return;
414 }
415
416 /* fallback */
417 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
418 struct mem_section *ms;
419
420 if (!present_section_nr(pnum))
421 continue;
422 map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
423 if (map_map[pnum])
424 continue;
425 ms = __nr_to_section(pnum);
426 printk(KERN_ERR "%s: sparsemem memory map backing failed "
427 "some memory will not be available.\n", __func__);
428 ms->section_mem_map = 0;
429 }
430}
Christoph Lameter8f6aac42007-10-16 01:24:13 -0700431#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
432
Yinghai Lu81d0d952010-02-27 09:29:38 -0800433#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
Wanpeng Li18732092013-09-11 14:22:38 -0700434static void __init sparse_early_mem_maps_alloc_node(void *data,
Yinghai Lu9bdac912010-02-10 01:20:22 -0800435 unsigned long pnum_begin,
436 unsigned long pnum_end,
437 unsigned long map_count, int nodeid)
438{
Wanpeng Li18732092013-09-11 14:22:38 -0700439 struct page **map_map = (struct page **)data;
Yinghai Lu9bdac912010-02-10 01:20:22 -0800440 sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end,
441 map_count, nodeid);
442}
Yinghai Lu81d0d952010-02-27 09:29:38 -0800443#else
Adrian Bunk9e5c6da2008-07-25 19:46:22 -0700444static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
Christoph Lameter8f6aac42007-10-16 01:24:13 -0700445{
446 struct page *map;
447 struct mem_section *ms = __nr_to_section(pnum);
448 int nid = sparse_early_nid(ms);
449
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700450 map = sparse_mem_map_populate(pnum, nid);
Andy Whitcroft29751f62005-06-23 00:08:00 -0700451 if (map)
452 return map;
453
Christoph Lameter8f6aac42007-10-16 01:24:13 -0700454 printk(KERN_ERR "%s: sparsemem memory map backing failed "
Harvey Harrisond40cee22008-04-30 00:55:07 -0700455 "some memory will not be available.\n", __func__);
Bob Picco802f1922005-09-03 15:54:26 -0700456 ms->section_mem_map = 0;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700457 return NULL;
458}
Yinghai Lu9bdac912010-02-10 01:20:22 -0800459#endif
Andy Whitcroft29751f62005-06-23 00:08:00 -0700460
Yinghai Luc2b91e22008-04-12 01:19:24 -0700461void __attribute__((weak)) __meminit vmemmap_populate_print_last(void)
462{
463}
Yinghai Lua4322e12010-02-10 01:20:21 -0800464
Wanpeng Li18732092013-09-11 14:22:38 -0700465/**
466 * alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap
467 * @map: usemap_map for pageblock flags or mmap_map for vmemmap
Stephen Rothwell193faea2007-06-08 13:46:51 -0700468 */
Wanpeng Li18732092013-09-11 14:22:38 -0700469static void __init alloc_usemap_and_memmap(void (*alloc_func)
470 (void *, unsigned long, unsigned long,
471 unsigned long, int), void *data)
Stephen Rothwell193faea2007-06-08 13:46:51 -0700472{
473 unsigned long pnum;
Wanpeng Li18732092013-09-11 14:22:38 -0700474 unsigned long map_count;
Yinghai Lua4322e12010-02-10 01:20:21 -0800475 int nodeid_begin = 0;
476 unsigned long pnum_begin = 0;
Yinghai Lu9bdac912010-02-10 01:20:22 -0800477
478 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
479 struct mem_section *ms;
480
481 if (!present_section_nr(pnum))
482 continue;
483 ms = __nr_to_section(pnum);
484 nodeid_begin = sparse_early_nid(ms);
485 pnum_begin = pnum;
486 break;
487 }
488 map_count = 1;
489 for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) {
490 struct mem_section *ms;
491 int nodeid;
492
493 if (!present_section_nr(pnum))
494 continue;
495 ms = __nr_to_section(pnum);
496 nodeid = sparse_early_nid(ms);
497 if (nodeid == nodeid_begin) {
498 map_count++;
499 continue;
500 }
501 /* ok, we need to take cake of from pnum_begin to pnum - 1*/
Wanpeng Li18732092013-09-11 14:22:38 -0700502 alloc_func(data, pnum_begin, pnum,
503 map_count, nodeid_begin);
Yinghai Lu9bdac912010-02-10 01:20:22 -0800504 /* new start, update count etc*/
505 nodeid_begin = nodeid;
506 pnum_begin = pnum;
507 map_count = 1;
508 }
509 /* ok, last chunk */
Wanpeng Li18732092013-09-11 14:22:38 -0700510 alloc_func(data, pnum_begin, NR_MEM_SECTIONS,
511 map_count, nodeid_begin);
512}
513
514/*
515 * Allocate the accumulated non-linear sections, allocate a mem_map
516 * for each and record the physical to section mapping.
517 */
518void __init sparse_init(void)
519{
520 unsigned long pnum;
521 struct page *map;
522 unsigned long *usemap;
523 unsigned long **usemap_map;
524 int size;
525#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
526 int size2;
527 struct page **map_map;
528#endif
529
530 /* see include/linux/mmzone.h 'struct mem_section' definition */
531 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
532
533 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
534 set_pageblock_order();
535
536 /*
537 * map is using big page (aka 2M in x86 64 bit)
538 * usemap is less one page (aka 24 bytes)
539 * so alloc 2M (with 2M align) and 24 bytes in turn will
540 * make next 2M slip to one more 2M later.
541 * then in big system, the memory will have a lot of holes...
542 * here try to allocate 2M pages continuously.
543 *
544 * powerpc need to call sparse_init_one_section right after each
545 * sparse_early_mem_map_alloc, so allocate usemap_map at first.
546 */
547 size = sizeof(unsigned long *) * NR_MEM_SECTIONS;
548 usemap_map = alloc_bootmem(size);
549 if (!usemap_map)
550 panic("can not allocate usemap_map\n");
551 alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node,
552 (void *)usemap_map);
553
554#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
555 size2 = sizeof(struct page *) * NR_MEM_SECTIONS;
556 map_map = alloc_bootmem(size2);
557 if (!map_map)
558 panic("can not allocate map_map\n");
559 alloc_usemap_and_memmap(sparse_early_mem_maps_alloc_node,
560 (void *)map_map);
Yinghai Lu9bdac912010-02-10 01:20:22 -0800561#endif
562
Yinghai Lue123dd32008-04-13 11:51:06 -0700563 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
564 if (!present_section_nr(pnum))
565 continue;
566
567 usemap = usemap_map[pnum];
568 if (!usemap)
569 continue;
Stephen Rothwell193faea2007-06-08 13:46:51 -0700570
Yinghai Lu9bdac912010-02-10 01:20:22 -0800571#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
572 map = map_map[pnum];
573#else
Stephen Rothwell193faea2007-06-08 13:46:51 -0700574 map = sparse_early_mem_map_alloc(pnum);
Yinghai Lu9bdac912010-02-10 01:20:22 -0800575#endif
Stephen Rothwell193faea2007-06-08 13:46:51 -0700576 if (!map)
577 continue;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700578
Mel Gorman5c0e3062007-10-16 01:25:56 -0700579 sparse_init_one_section(__nr_to_section(pnum), pnum, map,
580 usemap);
Stephen Rothwell193faea2007-06-08 13:46:51 -0700581 }
Yinghai Lue123dd32008-04-13 11:51:06 -0700582
Yinghai Luc2b91e22008-04-12 01:19:24 -0700583 vmemmap_populate_print_last();
584
Yinghai Lu9bdac912010-02-10 01:20:22 -0800585#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
586 free_bootmem(__pa(map_map), size2);
587#endif
Yinghai Lue123dd32008-04-13 11:51:06 -0700588 free_bootmem(__pa(usemap_map), size);
Stephen Rothwell193faea2007-06-08 13:46:51 -0700589}
590
591#ifdef CONFIG_MEMORY_HOTPLUG
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700592#ifdef CONFIG_SPARSEMEM_VMEMMAP
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800593static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid)
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700594{
595 /* This will make the necessary allocations eventually. */
596 return sparse_mem_map_populate(pnum, nid);
597}
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800598static void __kfree_section_memmap(struct page *memmap)
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700599{
Johannes Weiner0aad8182013-04-29 15:07:50 -0700600 unsigned long start = (unsigned long)memmap;
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800601 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
Johannes Weiner0aad8182013-04-29 15:07:50 -0700602
603 vmemmap_free(start, end);
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700604}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700605#ifdef CONFIG_MEMORY_HOTREMOVE
Zhang Yanfei81556b02013-11-12 15:07:43 -0800606static void free_map_bootmem(struct page *memmap)
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700607{
Johannes Weiner0aad8182013-04-29 15:07:50 -0700608 unsigned long start = (unsigned long)memmap;
Zhang Yanfei81556b02013-11-12 15:07:43 -0800609 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
Johannes Weiner0aad8182013-04-29 15:07:50 -0700610
611 vmemmap_free(start, end);
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700612}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700613#endif /* CONFIG_MEMORY_HOTREMOVE */
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700614#else
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800615static struct page *__kmalloc_section_memmap(void)
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700616{
617 struct page *page, *ret;
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800618 unsigned long memmap_size = sizeof(struct page) * PAGES_PER_SECTION;
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700619
Yasunori Gotof2d0aa52006-10-28 10:38:32 -0700620 page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700621 if (page)
622 goto got_map_page;
623
624 ret = vmalloc(memmap_size);
625 if (ret)
626 goto got_map_ptr;
627
628 return NULL;
629got_map_page:
630 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
631got_map_ptr:
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700632
633 return ret;
634}
635
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800636static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid)
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700637{
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800638 return __kmalloc_section_memmap();
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700639}
640
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800641static void __kfree_section_memmap(struct page *memmap)
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700642{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800643 if (is_vmalloc_addr(memmap))
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700644 vfree(memmap);
645 else
646 free_pages((unsigned long)memmap,
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800647 get_order(sizeof(struct page) * PAGES_PER_SECTION));
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700648}
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700649
David Rientjes4edd7ce2013-04-29 15:08:22 -0700650#ifdef CONFIG_MEMORY_HOTREMOVE
Zhang Yanfei81556b02013-11-12 15:07:43 -0800651static void free_map_bootmem(struct page *memmap)
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700652{
653 unsigned long maps_section_nr, removing_section_nr, i;
Zhang Yanfei81556b02013-11-12 15:07:43 -0800654 unsigned long magic, nr_pages;
Jianguo Wuae64ffc2012-11-29 13:54:21 -0800655 struct page *page = virt_to_page(memmap);
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700656
Zhang Yanfei81556b02013-11-12 15:07:43 -0800657 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page))
658 >> PAGE_SHIFT;
659
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700660 for (i = 0; i < nr_pages; i++, page++) {
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800661 magic = (unsigned long) page->lru.next;
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700662
663 BUG_ON(magic == NODE_INFO);
664
665 maps_section_nr = pfn_to_section_nr(page_to_pfn(page));
666 removing_section_nr = page->private;
667
668 /*
669 * When this function is called, the removing section is
670 * logical offlined state. This means all pages are isolated
671 * from page allocator. If removing section's memmap is placed
672 * on the same section, it must not be freed.
673 * If it is freed, page allocator may allocate it which will
674 * be removed physically soon.
675 */
676 if (maps_section_nr != removing_section_nr)
677 put_page_bootmem(page);
678 }
679}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700680#endif /* CONFIG_MEMORY_HOTREMOVE */
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700681#endif /* CONFIG_SPARSEMEM_VMEMMAP */
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700682
Andy Whitcroft29751f62005-06-23 00:08:00 -0700683/*
Andy Whitcroft29751f62005-06-23 00:08:00 -0700684 * returns the number of sections whose mem_maps were properly
685 * set. If this is <=0, then that means that the passed-in
686 * map was not consumed and must be freed.
687 */
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800688int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
Andy Whitcroft29751f62005-06-23 00:08:00 -0700689{
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700690 unsigned long section_nr = pfn_to_section_nr(start_pfn);
691 struct pglist_data *pgdat = zone->zone_pgdat;
692 struct mem_section *ms;
693 struct page *memmap;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700694 unsigned long *usemap;
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700695 unsigned long flags;
696 int ret;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700697
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700698 /*
699 * no locking for this, because it does its own
700 * plus, it does a kmalloc
701 */
WANG Congbbd06822007-12-17 16:19:59 -0800702 ret = sparse_index_init(section_nr, pgdat->node_id);
703 if (ret < 0 && ret != -EEXIST)
704 return ret;
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800705 memmap = kmalloc_section_memmap(section_nr, pgdat->node_id);
WANG Congbbd06822007-12-17 16:19:59 -0800706 if (!memmap)
707 return -ENOMEM;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700708 usemap = __kmalloc_section_usemap();
WANG Congbbd06822007-12-17 16:19:59 -0800709 if (!usemap) {
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800710 __kfree_section_memmap(memmap);
WANG Congbbd06822007-12-17 16:19:59 -0800711 return -ENOMEM;
712 }
Andy Whitcroft29751f62005-06-23 00:08:00 -0700713
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700714 pgdat_resize_lock(pgdat, &flags);
715
716 ms = __pfn_to_section(start_pfn);
717 if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
718 ret = -EEXIST;
719 goto out;
720 }
Mel Gorman5c0e3062007-10-16 01:25:56 -0700721
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800722 memset(memmap, 0, sizeof(struct page) * PAGES_PER_SECTION);
Wen Congyang3ac19f82012-12-11 16:00:59 -0800723
Andy Whitcroft29751f62005-06-23 00:08:00 -0700724 ms->section_mem_map |= SECTION_MARKED_PRESENT;
725
Mel Gorman5c0e3062007-10-16 01:25:56 -0700726 ret = sparse_init_one_section(ms, section_nr, memmap, usemap);
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700727
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700728out:
729 pgdat_resize_unlock(pgdat, &flags);
WANG Congbbd06822007-12-17 16:19:59 -0800730 if (ret <= 0) {
731 kfree(usemap);
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800732 __kfree_section_memmap(memmap);
WANG Congbbd06822007-12-17 16:19:59 -0800733 }
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700734 return ret;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700735}
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700736
Zhang Yanfeif3deb682013-07-08 16:00:10 -0700737#ifdef CONFIG_MEMORY_HOTREMOVE
Wen Congyang95a47742012-12-11 16:00:47 -0800738#ifdef CONFIG_MEMORY_FAILURE
739static void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
740{
741 int i;
742
743 if (!memmap)
744 return;
745
746 for (i = 0; i < PAGES_PER_SECTION; i++) {
747 if (PageHWPoison(&memmap[i])) {
Xishi Qiu293c07e2013-02-22 16:34:02 -0800748 atomic_long_sub(1, &num_poisoned_pages);
Wen Congyang95a47742012-12-11 16:00:47 -0800749 ClearPageHWPoison(&memmap[i]);
750 }
751 }
752}
753#else
754static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
755{
756}
757#endif
758
David Rientjes4edd7ce2013-04-29 15:08:22 -0700759static void free_section_usemap(struct page *memmap, unsigned long *usemap)
760{
761 struct page *usemap_page;
David Rientjes4edd7ce2013-04-29 15:08:22 -0700762
763 if (!usemap)
764 return;
765
766 usemap_page = virt_to_page(usemap);
767 /*
768 * Check to see if allocation came from hot-plug-add
769 */
770 if (PageSlab(usemap_page) || PageCompound(usemap_page)) {
771 kfree(usemap);
772 if (memmap)
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800773 __kfree_section_memmap(memmap);
David Rientjes4edd7ce2013-04-29 15:08:22 -0700774 return;
775 }
776
777 /*
778 * The usemap came from bootmem. This is packed with other usemaps
779 * on the section which has pgdat at boot time. Just keep it as is now.
780 */
781
Zhang Yanfei81556b02013-11-12 15:07:43 -0800782 if (memmap)
783 free_map_bootmem(memmap);
David Rientjes4edd7ce2013-04-29 15:08:22 -0700784}
785
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700786void sparse_remove_one_section(struct zone *zone, struct mem_section *ms)
787{
788 struct page *memmap = NULL;
Tang Chencd099682013-02-22 16:33:02 -0800789 unsigned long *usemap = NULL, flags;
790 struct pglist_data *pgdat = zone->zone_pgdat;
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700791
Tang Chencd099682013-02-22 16:33:02 -0800792 pgdat_resize_lock(pgdat, &flags);
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700793 if (ms->section_mem_map) {
794 usemap = ms->pageblock_flags;
795 memmap = sparse_decode_mem_map(ms->section_mem_map,
796 __section_nr(ms));
797 ms->section_mem_map = 0;
798 ms->pageblock_flags = NULL;
799 }
Tang Chencd099682013-02-22 16:33:02 -0800800 pgdat_resize_unlock(pgdat, &flags);
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700801
Wen Congyang95a47742012-12-11 16:00:47 -0800802 clear_hwpoisoned_pages(memmap, PAGES_PER_SECTION);
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700803 free_section_usemap(memmap, usemap);
804}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700805#endif /* CONFIG_MEMORY_HOTREMOVE */
806#endif /* CONFIG_MEMORY_HOTPLUG */