blob: b3c82ba300124ea28d1cb952f337e387b1c4b9eb [file] [log] [blame]
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001/*
2 * sparse memory mappings.
3 */
Andy Whitcroftd41dee32005-06-23 00:07:54 -07004#include <linux/mm.h>
5#include <linux/mmzone.h>
6#include <linux/bootmem.h>
Dave Hansen0b0acbe2005-10-29 18:16:55 -07007#include <linux/highmem.h>
Andy Whitcroftd41dee32005-06-23 00:07:54 -07008#include <linux/module.h>
Dave Hansen28ae55c2005-09-03 15:54:29 -07009#include <linux/spinlock.h>
Dave Hansen0b0acbe2005-10-29 18:16:55 -070010#include <linux/vmalloc.h>
Andy Whitcroftd41dee32005-06-23 00:07:54 -070011#include <asm/dma.h>
12
13/*
14 * Permanent SPARSEMEM data:
15 *
16 * 1) mem_section - memory sections, mem_map's for valid memory
17 */
Bob Picco3e347262005-09-03 15:54:28 -070018#ifdef CONFIG_SPARSEMEM_EXTREME
Bob Picco802f1922005-09-03 15:54:26 -070019struct mem_section *mem_section[NR_SECTION_ROOTS]
Ravikiran G Thirumalai22fc6ec2006-01-08 01:01:27 -080020 ____cacheline_internodealigned_in_smp;
Bob Picco3e347262005-09-03 15:54:28 -070021#else
22struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
Ravikiran G Thirumalai22fc6ec2006-01-08 01:01:27 -080023 ____cacheline_internodealigned_in_smp;
Bob Picco3e347262005-09-03 15:54:28 -070024#endif
25EXPORT_SYMBOL(mem_section);
26
Bob Picco3e347262005-09-03 15:54:28 -070027#ifdef CONFIG_SPARSEMEM_EXTREME
Dave Hansen28ae55c2005-09-03 15:54:29 -070028static struct mem_section *sparse_index_alloc(int nid)
Bob Picco802f1922005-09-03 15:54:26 -070029{
Dave Hansen28ae55c2005-09-03 15:54:29 -070030 struct mem_section *section = NULL;
31 unsigned long array_size = SECTIONS_PER_ROOT *
32 sizeof(struct mem_section);
Bob Picco802f1922005-09-03 15:54:26 -070033
Mike Kravetz39d24e62006-05-15 09:44:13 -070034 if (slab_is_available())
Mike Kravetz46a66ee2006-05-01 12:16:09 -070035 section = kmalloc_node(array_size, GFP_KERNEL, nid);
36 else
37 section = alloc_bootmem_node(NODE_DATA(nid), array_size);
Bob Picco3e347262005-09-03 15:54:28 -070038
Dave Hansen28ae55c2005-09-03 15:54:29 -070039 if (section)
40 memset(section, 0, array_size);
Bob Picco3e347262005-09-03 15:54:28 -070041
Dave Hansen28ae55c2005-09-03 15:54:29 -070042 return section;
Bob Picco802f1922005-09-03 15:54:26 -070043}
Dave Hansen28ae55c2005-09-03 15:54:29 -070044
45static int sparse_index_init(unsigned long section_nr, int nid)
46{
Ingo Molnar34af9462006-06-27 02:53:55 -070047 static DEFINE_SPINLOCK(index_init_lock);
Dave Hansen28ae55c2005-09-03 15:54:29 -070048 unsigned long root = SECTION_NR_TO_ROOT(section_nr);
49 struct mem_section *section;
50 int ret = 0;
51
52 if (mem_section[root])
53 return -EEXIST;
54
55 section = sparse_index_alloc(nid);
56 /*
57 * This lock keeps two different sections from
58 * reallocating for the same index
59 */
60 spin_lock(&index_init_lock);
61
62 if (mem_section[root]) {
63 ret = -EEXIST;
64 goto out;
65 }
66
67 mem_section[root] = section;
68out:
69 spin_unlock(&index_init_lock);
70 return ret;
71}
72#else /* !SPARSEMEM_EXTREME */
73static inline int sparse_index_init(unsigned long section_nr, int nid)
74{
75 return 0;
76}
77#endif
78
Dave Hansen4ca644d2005-10-29 18:16:51 -070079/*
80 * Although written for the SPARSEMEM_EXTREME case, this happens
81 * to also work for the flat array case becase
82 * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
83 */
84int __section_nr(struct mem_section* ms)
85{
86 unsigned long root_nr;
87 struct mem_section* root;
88
Mike Kravetz12783b02006-05-20 15:00:05 -070089 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {
90 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);
Dave Hansen4ca644d2005-10-29 18:16:51 -070091 if (!root)
92 continue;
93
94 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))
95 break;
96 }
97
98 return (root_nr * SECTIONS_PER_ROOT) + (ms - root);
99}
100
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700101/*
102 * During early boot, before section_mem_map is used for an actual
103 * mem_map, we use section_mem_map to store the section's NUMA
104 * node. This keeps us from having to use another data structure. The
105 * node information is cleared just before we store the real mem_map.
106 */
107static inline unsigned long sparse_encode_early_nid(int nid)
108{
109 return (nid << SECTION_NID_SHIFT);
110}
111
112static inline int sparse_early_nid(struct mem_section *section)
113{
114 return (section->section_mem_map >> SECTION_NID_SHIFT);
115}
116
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700117/* Record a memory area against a node. */
118void memory_present(int nid, unsigned long start, unsigned long end)
119{
120 unsigned long pfn;
121
122 start &= PAGE_SECTION_MASK;
123 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
124 unsigned long section = pfn_to_section_nr(pfn);
Bob Picco802f1922005-09-03 15:54:26 -0700125 struct mem_section *ms;
126
127 sparse_index_init(section, nid);
128
129 ms = __nr_to_section(section);
130 if (!ms->section_mem_map)
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700131 ms->section_mem_map = sparse_encode_early_nid(nid) |
132 SECTION_MARKED_PRESENT;
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700133 }
134}
135
136/*
137 * Only used by the i386 NUMA architecures, but relatively
138 * generic code.
139 */
140unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,
141 unsigned long end_pfn)
142{
143 unsigned long pfn;
144 unsigned long nr_pages = 0;
145
146 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
147 if (nid != early_pfn_to_nid(pfn))
148 continue;
149
150 if (pfn_valid(pfn))
151 nr_pages += PAGES_PER_SECTION;
152 }
153
154 return nr_pages * sizeof(struct page);
155}
156
157/*
Andy Whitcroft29751f62005-06-23 00:08:00 -0700158 * Subtle, we encode the real pfn into the mem_map such that
159 * the identity pfn - section_mem_map will return the actual
160 * physical page frame number.
161 */
162static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
163{
164 return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
165}
166
167/*
168 * We need this if we ever free the mem_maps. While not implemented yet,
169 * this function is included for parity with its sibling.
170 */
171static __attribute((unused))
172struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
173{
174 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
175}
176
177static int sparse_init_one_section(struct mem_section *ms,
178 unsigned long pnum, struct page *mem_map)
179{
180 if (!valid_section(ms))
181 return -EINVAL;
182
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700183 ms->section_mem_map &= ~SECTION_MAP_MASK;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700184 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum);
185
186 return 1;
187}
188
189static struct page *sparse_early_mem_map_alloc(unsigned long pnum)
190{
191 struct page *map;
Bob Picco802f1922005-09-03 15:54:26 -0700192 struct mem_section *ms = __nr_to_section(pnum);
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700193 int nid = sparse_early_nid(ms);
Andy Whitcroft29751f62005-06-23 00:08:00 -0700194
195 map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);
196 if (map)
197 return map;
198
199 map = alloc_bootmem_node(NODE_DATA(nid),
200 sizeof(struct page) * PAGES_PER_SECTION);
201 if (map)
202 return map;
203
204 printk(KERN_WARNING "%s: allocation failed\n", __FUNCTION__);
Bob Picco802f1922005-09-03 15:54:26 -0700205 ms->section_mem_map = 0;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700206 return NULL;
207}
208
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700209static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
210{
211 struct page *page, *ret;
212 unsigned long memmap_size = sizeof(struct page) * nr_pages;
213
Yasunori Gotof2d0aa52006-10-28 10:38:32 -0700214 page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700215 if (page)
216 goto got_map_page;
217
218 ret = vmalloc(memmap_size);
219 if (ret)
220 goto got_map_ptr;
221
222 return NULL;
223got_map_page:
224 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
225got_map_ptr:
226 memset(ret, 0, memmap_size);
227
228 return ret;
229}
230
231static int vaddr_in_vmalloc_area(void *addr)
232{
233 if (addr >= (void *)VMALLOC_START &&
234 addr < (void *)VMALLOC_END)
235 return 1;
236 return 0;
237}
238
239static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
240{
241 if (vaddr_in_vmalloc_area(memmap))
242 vfree(memmap);
243 else
244 free_pages((unsigned long)memmap,
245 get_order(sizeof(struct page) * nr_pages));
246}
247
Andy Whitcroft29751f62005-06-23 00:08:00 -0700248/*
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700249 * Allocate the accumulated non-linear sections, allocate a mem_map
250 * for each and record the physical to section mapping.
251 */
252void sparse_init(void)
253{
254 unsigned long pnum;
255 struct page *map;
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700256
257 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
Andy Whitcroft29751f62005-06-23 00:08:00 -0700258 if (!valid_section_nr(pnum))
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700259 continue;
260
Andy Whitcroft29751f62005-06-23 00:08:00 -0700261 map = sparse_early_mem_map_alloc(pnum);
Bob Picco802f1922005-09-03 15:54:26 -0700262 if (!map)
263 continue;
264 sparse_init_one_section(__nr_to_section(pnum), pnum, map);
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700265 }
266}
Andy Whitcroft29751f62005-06-23 00:08:00 -0700267
268/*
269 * returns the number of sections whose mem_maps were properly
270 * set. If this is <=0, then that means that the passed-in
271 * map was not consumed and must be freed.
272 */
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700273int sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
274 int nr_pages)
Andy Whitcroft29751f62005-06-23 00:08:00 -0700275{
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700276 unsigned long section_nr = pfn_to_section_nr(start_pfn);
277 struct pglist_data *pgdat = zone->zone_pgdat;
278 struct mem_section *ms;
279 struct page *memmap;
280 unsigned long flags;
281 int ret;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700282
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700283 /*
284 * no locking for this, because it does its own
285 * plus, it does a kmalloc
286 */
287 sparse_index_init(section_nr, pgdat->node_id);
288 memmap = __kmalloc_section_memmap(nr_pages);
Andy Whitcroft29751f62005-06-23 00:08:00 -0700289
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700290 pgdat_resize_lock(pgdat, &flags);
291
292 ms = __pfn_to_section(start_pfn);
293 if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
294 ret = -EEXIST;
295 goto out;
296 }
Andy Whitcroft29751f62005-06-23 00:08:00 -0700297 ms->section_mem_map |= SECTION_MARKED_PRESENT;
298
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700299 ret = sparse_init_one_section(ms, section_nr, memmap);
300
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700301out:
302 pgdat_resize_unlock(pgdat, &flags);
Mike Kravetz46a66ee2006-05-01 12:16:09 -0700303 if (ret <= 0)
304 __kfree_section_memmap(memmap, nr_pages);
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700305 return ret;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700306}