blob: 6e45b0f3d1256e0bdc3a3caffa74597cf96a5138 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
Christoph Lameter930fc452005-10-29 18:15:41 -07008 * Numa awareness, Christoph Lameter, SGI, June 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/highmem.h>
14#include <linux/slab.h>
15#include <linux/spinlock.h>
16#include <linux/interrupt.h>
Christoph Lametera10aa572008-04-28 02:12:40 -070017#include <linux/seq_file.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070018#include <linux/debugobjects.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/vmalloc.h>
Christoph Lameter23016962008-04-28 02:12:42 -070020#include <linux/kallsyms.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22#include <asm/uaccess.h>
23#include <asm/tlbflush.h>
24
25
26DEFINE_RWLOCK(vmlist_lock);
27struct vm_struct *vmlist;
28
Adrian Bunkb2213852006-09-25 23:31:02 -070029static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
Christoph Lameter23016962008-04-28 02:12:42 -070030 int node, void *caller);
Adrian Bunkb2213852006-09-25 23:31:02 -070031
Linus Torvalds1da177e2005-04-16 15:20:36 -070032static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
33{
34 pte_t *pte;
35
36 pte = pte_offset_kernel(pmd, addr);
37 do {
38 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
39 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
40 } while (pte++, addr += PAGE_SIZE, addr != end);
41}
42
43static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
44 unsigned long end)
45{
46 pmd_t *pmd;
47 unsigned long next;
48
49 pmd = pmd_offset(pud, addr);
50 do {
51 next = pmd_addr_end(addr, end);
52 if (pmd_none_or_clear_bad(pmd))
53 continue;
54 vunmap_pte_range(pmd, addr, next);
55 } while (pmd++, addr = next, addr != end);
56}
57
58static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
59 unsigned long end)
60{
61 pud_t *pud;
62 unsigned long next;
63
64 pud = pud_offset(pgd, addr);
65 do {
66 next = pud_addr_end(addr, end);
67 if (pud_none_or_clear_bad(pud))
68 continue;
69 vunmap_pmd_range(pud, addr, next);
70 } while (pud++, addr = next, addr != end);
71}
72
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100073void unmap_kernel_range(unsigned long addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -070074{
75 pgd_t *pgd;
76 unsigned long next;
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100077 unsigned long start = addr;
78 unsigned long end = addr + size;
Linus Torvalds1da177e2005-04-16 15:20:36 -070079
80 BUG_ON(addr >= end);
81 pgd = pgd_offset_k(addr);
82 flush_cache_vunmap(addr, end);
83 do {
84 next = pgd_addr_end(addr, end);
85 if (pgd_none_or_clear_bad(pgd))
86 continue;
87 vunmap_pud_range(pgd, addr, next);
88 } while (pgd++, addr = next, addr != end);
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100089 flush_tlb_kernel_range(start, end);
90}
91
92static void unmap_vm_area(struct vm_struct *area)
93{
94 unmap_kernel_range((unsigned long)area->addr, area->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -070095}
96
97static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
98 unsigned long end, pgprot_t prot, struct page ***pages)
99{
100 pte_t *pte;
101
Hugh Dickins872fec12005-10-29 18:16:21 -0700102 pte = pte_alloc_kernel(pmd, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 if (!pte)
104 return -ENOMEM;
105 do {
106 struct page *page = **pages;
107 WARN_ON(!pte_none(*pte));
108 if (!page)
109 return -ENOMEM;
110 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
111 (*pages)++;
112 } while (pte++, addr += PAGE_SIZE, addr != end);
113 return 0;
114}
115
116static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
117 unsigned long end, pgprot_t prot, struct page ***pages)
118{
119 pmd_t *pmd;
120 unsigned long next;
121
122 pmd = pmd_alloc(&init_mm, pud, addr);
123 if (!pmd)
124 return -ENOMEM;
125 do {
126 next = pmd_addr_end(addr, end);
127 if (vmap_pte_range(pmd, addr, next, prot, pages))
128 return -ENOMEM;
129 } while (pmd++, addr = next, addr != end);
130 return 0;
131}
132
133static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
134 unsigned long end, pgprot_t prot, struct page ***pages)
135{
136 pud_t *pud;
137 unsigned long next;
138
139 pud = pud_alloc(&init_mm, pgd, addr);
140 if (!pud)
141 return -ENOMEM;
142 do {
143 next = pud_addr_end(addr, end);
144 if (vmap_pmd_range(pud, addr, next, prot, pages))
145 return -ENOMEM;
146 } while (pud++, addr = next, addr != end);
147 return 0;
148}
149
150int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
151{
152 pgd_t *pgd;
153 unsigned long next;
154 unsigned long addr = (unsigned long) area->addr;
155 unsigned long end = addr + area->size - PAGE_SIZE;
156 int err;
157
158 BUG_ON(addr >= end);
159 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 do {
161 next = pgd_addr_end(addr, end);
162 err = vmap_pud_range(pgd, addr, next, prot, pages);
163 if (err)
164 break;
165 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 flush_cache_vmap((unsigned long) area->addr, end);
167 return err;
168}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700169EXPORT_SYMBOL_GPL(map_vm_area);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
Christoph Lameter48667e72008-02-04 22:28:31 -0800171/*
172 * Map a vmalloc()-space virtual address to the physical page.
173 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800174struct page *vmalloc_to_page(const void *vmalloc_addr)
Christoph Lameter48667e72008-02-04 22:28:31 -0800175{
176 unsigned long addr = (unsigned long) vmalloc_addr;
177 struct page *page = NULL;
178 pgd_t *pgd = pgd_offset_k(addr);
179 pud_t *pud;
180 pmd_t *pmd;
181 pte_t *ptep, pte;
182
183 if (!pgd_none(*pgd)) {
184 pud = pud_offset(pgd, addr);
185 if (!pud_none(*pud)) {
186 pmd = pmd_offset(pud, addr);
187 if (!pmd_none(*pmd)) {
188 ptep = pte_offset_map(pmd, addr);
189 pte = *ptep;
190 if (pte_present(pte))
191 page = pte_page(pte);
192 pte_unmap(ptep);
193 }
194 }
195 }
196 return page;
197}
198EXPORT_SYMBOL(vmalloc_to_page);
199
200/*
201 * Map a vmalloc()-space virtual address to the physical page frame number.
202 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800203unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
Christoph Lameter48667e72008-02-04 22:28:31 -0800204{
205 return page_to_pfn(vmalloc_to_page(vmalloc_addr));
206}
207EXPORT_SYMBOL(vmalloc_to_pfn);
208
Christoph Lameter23016962008-04-28 02:12:42 -0700209static struct vm_struct *
210__get_vm_area_node(unsigned long size, unsigned long flags, unsigned long start,
211 unsigned long end, int node, gfp_t gfp_mask, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212{
213 struct vm_struct **p, *tmp, *area;
214 unsigned long align = 1;
215 unsigned long addr;
216
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700217 BUG_ON(in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218 if (flags & VM_IOREMAP) {
219 int bit = fls(size);
220
221 if (bit > IOREMAP_MAX_ORDER)
222 bit = IOREMAP_MAX_ORDER;
223 else if (bit < PAGE_SHIFT)
224 bit = PAGE_SHIFT;
225
226 align = 1ul << bit;
227 }
228 addr = ALIGN(start, align);
229 size = PAGE_ALIGN(size);
OGAWA Hirofumi31be8302006-11-16 01:19:29 -0800230 if (unlikely(!size))
231 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232
Christoph Lameter6cb06222007-10-16 01:25:41 -0700233 area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
234
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 if (unlikely(!area))
236 return NULL;
237
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 /*
239 * We always allocate a guard page.
240 */
241 size += PAGE_SIZE;
242
243 write_lock(&vmlist_lock);
244 for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
245 if ((unsigned long)tmp->addr < addr) {
246 if((unsigned long)tmp->addr + tmp->size >= addr)
247 addr = ALIGN(tmp->size +
248 (unsigned long)tmp->addr, align);
249 continue;
250 }
251 if ((size + addr) < addr)
252 goto out;
253 if (size + addr <= (unsigned long)tmp->addr)
254 goto found;
255 addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
256 if (addr > end - size)
257 goto out;
258 }
Robert Bragg5dc33182008-02-04 22:29:18 -0800259 if ((size + addr) < addr)
260 goto out;
261 if (addr > end - size)
262 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263
264found:
265 area->next = *p;
266 *p = area;
267
268 area->flags = flags;
269 area->addr = (void *)addr;
270 area->size = size;
271 area->pages = NULL;
272 area->nr_pages = 0;
273 area->phys_addr = 0;
Christoph Lameter23016962008-04-28 02:12:42 -0700274 area->caller = caller;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 write_unlock(&vmlist_lock);
276
277 return area;
278
279out:
280 write_unlock(&vmlist_lock);
281 kfree(area);
282 if (printk_ratelimit())
283 printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
284 return NULL;
285}
286
Christoph Lameter930fc452005-10-29 18:15:41 -0700287struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
288 unsigned long start, unsigned long end)
289{
Christoph Lameter23016962008-04-28 02:12:42 -0700290 return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL,
291 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700292}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700293EXPORT_SYMBOL_GPL(__get_vm_area);
Christoph Lameter930fc452005-10-29 18:15:41 -0700294
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295/**
Simon Arlott183ff222007-10-20 01:27:18 +0200296 * get_vm_area - reserve a contiguous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 * @size: size of the area
298 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
299 *
300 * Search an area of @size in the kernel virtual mapping area,
301 * and reserved it for out purposes. Returns the area descriptor
302 * on success or %NULL on failure.
303 */
304struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
305{
Christoph Lameter23016962008-04-28 02:12:42 -0700306 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
307 -1, GFP_KERNEL, __builtin_return_address(0));
308}
309
310struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
311 void *caller)
312{
313 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
314 -1, GFP_KERNEL, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315}
316
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700317struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
318 int node, gfp_t gfp_mask)
Christoph Lameter930fc452005-10-29 18:15:41 -0700319{
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700320 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
Christoph Lameter23016962008-04-28 02:12:42 -0700321 gfp_mask, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700322}
323
Andi Kleen7856dfe2005-05-20 14:27:57 -0700324/* Caller must hold vmlist_lock */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800325static struct vm_struct *__find_vm_area(const void *addr)
Nick Piggin83342312006-06-23 02:03:20 -0700326{
327 struct vm_struct *tmp;
328
329 for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
330 if (tmp->addr == addr)
331 break;
332 }
333
334 return tmp;
335}
336
337/* Caller must hold vmlist_lock */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800338static struct vm_struct *__remove_vm_area(const void *addr)
Andi Kleen7856dfe2005-05-20 14:27:57 -0700339{
340 struct vm_struct **p, *tmp;
341
342 for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
343 if (tmp->addr == addr)
344 goto found;
345 }
346 return NULL;
347
348found:
349 unmap_vm_area(tmp);
350 *p = tmp->next;
351
352 /*
353 * Remove the guard page.
354 */
355 tmp->size -= PAGE_SIZE;
356 return tmp;
357}
358
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359/**
Simon Arlott183ff222007-10-20 01:27:18 +0200360 * remove_vm_area - find and remove a continuous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 * @addr: base address
362 *
363 * Search for the kernel VM area starting at @addr, and remove it.
364 * This function returns the found VM area, but using it is NOT safe
Andi Kleen7856dfe2005-05-20 14:27:57 -0700365 * on SMP machines, except for its size or flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800367struct vm_struct *remove_vm_area(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368{
Andi Kleen7856dfe2005-05-20 14:27:57 -0700369 struct vm_struct *v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 write_lock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700371 v = __remove_vm_area(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 write_unlock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700373 return v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374}
375
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800376static void __vunmap(const void *addr, int deallocate_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377{
378 struct vm_struct *area;
379
380 if (!addr)
381 return;
382
383 if ((PAGE_SIZE-1) & (unsigned long)addr) {
384 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
385 WARN_ON(1);
386 return;
387 }
388
389 area = remove_vm_area(addr);
390 if (unlikely(!area)) {
391 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
392 addr);
393 WARN_ON(1);
394 return;
395 }
396
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700397 debug_check_no_locks_freed(addr, area->size);
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -0700398 debug_check_no_obj_freed(addr, area->size);
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700399
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 if (deallocate_pages) {
401 int i;
402
403 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800404 struct page *page = area->pages[i];
405
406 BUG_ON(!page);
407 __free_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 }
409
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700410 if (area->flags & VM_VPAGES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 vfree(area->pages);
412 else
413 kfree(area->pages);
414 }
415
416 kfree(area);
417 return;
418}
419
420/**
421 * vfree - release memory allocated by vmalloc()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 * @addr: memory base address
423 *
Simon Arlott183ff222007-10-20 01:27:18 +0200424 * Free the virtually continuous memory area starting at @addr, as
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700425 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
426 * NULL, no operation is performed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700428 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800430void vfree(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431{
432 BUG_ON(in_interrupt());
433 __vunmap(addr, 1);
434}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435EXPORT_SYMBOL(vfree);
436
437/**
438 * vunmap - release virtual mapping obtained by vmap()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 * @addr: memory base address
440 *
441 * Free the virtually contiguous memory area starting at @addr,
442 * which was created from the page array passed to vmap().
443 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700444 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800446void vunmap(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447{
448 BUG_ON(in_interrupt());
449 __vunmap(addr, 0);
450}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451EXPORT_SYMBOL(vunmap);
452
453/**
454 * vmap - map an array of pages into virtually contiguous space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 * @pages: array of page pointers
456 * @count: number of pages to map
457 * @flags: vm_area->flags
458 * @prot: page protection for the mapping
459 *
460 * Maps @count pages from @pages into contiguous kernel virtual
461 * space.
462 */
463void *vmap(struct page **pages, unsigned int count,
464 unsigned long flags, pgprot_t prot)
465{
466 struct vm_struct *area;
467
468 if (count > num_physpages)
469 return NULL;
470
Christoph Lameter23016962008-04-28 02:12:42 -0700471 area = get_vm_area_caller((count << PAGE_SHIFT), flags,
472 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 if (!area)
474 return NULL;
Christoph Lameter23016962008-04-28 02:12:42 -0700475
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 if (map_vm_area(area, prot, &pages)) {
477 vunmap(area->addr);
478 return NULL;
479 }
480
481 return area->addr;
482}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483EXPORT_SYMBOL(vmap);
484
Adrian Bunke31d9eb2008-02-04 22:29:09 -0800485static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
Christoph Lameter23016962008-04-28 02:12:42 -0700486 pgprot_t prot, int node, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487{
488 struct page **pages;
489 unsigned int nr_pages, array_size, i;
490
491 nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
492 array_size = (nr_pages * sizeof(struct page *));
493
494 area->nr_pages = nr_pages;
495 /* Please note that the recursion is strictly bounded. */
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700496 if (array_size > PAGE_SIZE) {
Christoph Lameter94f60302007-07-17 04:03:29 -0700497 pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
Christoph Lameter23016962008-04-28 02:12:42 -0700498 PAGE_KERNEL, node, caller);
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700499 area->flags |= VM_VPAGES;
Andrew Morton286e1ea2006-10-17 00:09:57 -0700500 } else {
501 pages = kmalloc_node(array_size,
Christoph Lameter6cb06222007-10-16 01:25:41 -0700502 (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO,
Andrew Morton286e1ea2006-10-17 00:09:57 -0700503 node);
504 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 area->pages = pages;
Christoph Lameter23016962008-04-28 02:12:42 -0700506 area->caller = caller;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 if (!area->pages) {
508 remove_vm_area(area->addr);
509 kfree(area);
510 return NULL;
511 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512
513 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800514 struct page *page;
515
Christoph Lameter930fc452005-10-29 18:15:41 -0700516 if (node < 0)
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800517 page = alloc_page(gfp_mask);
Christoph Lameter930fc452005-10-29 18:15:41 -0700518 else
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800519 page = alloc_pages_node(node, gfp_mask, 0);
520
521 if (unlikely(!page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 /* Successfully allocated i pages, free them in __vunmap() */
523 area->nr_pages = i;
524 goto fail;
525 }
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800526 area->pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 }
528
529 if (map_vm_area(area, prot, &pages))
530 goto fail;
531 return area->addr;
532
533fail:
534 vfree(area->addr);
535 return NULL;
536}
537
Christoph Lameter930fc452005-10-29 18:15:41 -0700538void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
539{
Christoph Lameter23016962008-04-28 02:12:42 -0700540 return __vmalloc_area_node(area, gfp_mask, prot, -1,
541 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700542}
543
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700545 * __vmalloc_node - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546 * @size: allocation size
547 * @gfp_mask: flags for the page level allocator
548 * @prot: protection mask for the allocated pages
Randy Dunlapd44e0782005-11-07 01:01:10 -0800549 * @node: node to use for allocation or -1
Randy Dunlapc85d1942008-05-01 04:34:48 -0700550 * @caller: caller's return address
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 *
552 * Allocate enough pages to cover @size from the page level
553 * allocator with @gfp_mask flags. Map them into contiguous
554 * kernel virtual space, using a pagetable protection of @prot.
555 */
Adrian Bunkb2213852006-09-25 23:31:02 -0700556static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
Christoph Lameter23016962008-04-28 02:12:42 -0700557 int node, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558{
559 struct vm_struct *area;
560
561 size = PAGE_ALIGN(size);
562 if (!size || (size >> PAGE_SHIFT) > num_physpages)
563 return NULL;
564
Christoph Lameter23016962008-04-28 02:12:42 -0700565 area = __get_vm_area_node(size, VM_ALLOC, VMALLOC_START, VMALLOC_END,
566 node, gfp_mask, caller);
567
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568 if (!area)
569 return NULL;
570
Christoph Lameter23016962008-04-28 02:12:42 -0700571 return __vmalloc_area_node(area, gfp_mask, prot, node, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572}
573
Christoph Lameter930fc452005-10-29 18:15:41 -0700574void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
575{
Christoph Lameter23016962008-04-28 02:12:42 -0700576 return __vmalloc_node(size, gfp_mask, prot, -1,
577 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700578}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579EXPORT_SYMBOL(__vmalloc);
580
581/**
582 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 * @size: allocation size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 * Allocate enough pages to cover @size from the page level
585 * allocator and map them into contiguous kernel virtual space.
586 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200587 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 * use __vmalloc() instead.
589 */
590void *vmalloc(unsigned long size)
591{
Christoph Lameter23016962008-04-28 02:12:42 -0700592 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
593 -1, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595EXPORT_SYMBOL(vmalloc);
596
Christoph Lameter930fc452005-10-29 18:15:41 -0700597/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700598 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
599 * @size: allocation size
Nick Piggin83342312006-06-23 02:03:20 -0700600 *
Rolf Eike Beeread04082006-09-27 01:50:13 -0700601 * The resulting memory area is zeroed so it can be mapped to userspace
602 * without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700603 */
604void *vmalloc_user(unsigned long size)
605{
606 struct vm_struct *area;
607 void *ret;
608
609 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800610 if (ret) {
611 write_lock(&vmlist_lock);
612 area = __find_vm_area(ret);
613 area->flags |= VM_USERMAP;
614 write_unlock(&vmlist_lock);
615 }
Nick Piggin83342312006-06-23 02:03:20 -0700616 return ret;
617}
618EXPORT_SYMBOL(vmalloc_user);
619
620/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700621 * vmalloc_node - allocate memory on a specific node
Christoph Lameter930fc452005-10-29 18:15:41 -0700622 * @size: allocation size
Randy Dunlapd44e0782005-11-07 01:01:10 -0800623 * @node: numa node
Christoph Lameter930fc452005-10-29 18:15:41 -0700624 *
625 * Allocate enough pages to cover @size from the page level
626 * allocator and map them into contiguous kernel virtual space.
627 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200628 * For tight control over page level allocator and protection flags
Christoph Lameter930fc452005-10-29 18:15:41 -0700629 * use __vmalloc() instead.
630 */
631void *vmalloc_node(unsigned long size, int node)
632{
Christoph Lameter23016962008-04-28 02:12:42 -0700633 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
634 node, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700635}
636EXPORT_SYMBOL(vmalloc_node);
637
Pavel Pisa4dc3b162005-05-01 08:59:25 -0700638#ifndef PAGE_KERNEL_EXEC
639# define PAGE_KERNEL_EXEC PAGE_KERNEL
640#endif
641
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642/**
643 * vmalloc_exec - allocate virtually contiguous, executable memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 * @size: allocation size
645 *
646 * Kernel-internal function to allocate enough pages to cover @size
647 * the page level allocator and map them into contiguous and
648 * executable kernel virtual space.
649 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200650 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 * use __vmalloc() instead.
652 */
653
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654void *vmalloc_exec(unsigned long size)
655{
656 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
657}
658
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200659#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700660#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200661#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700662#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200663#else
664#define GFP_VMALLOC32 GFP_KERNEL
665#endif
666
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667/**
668 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 * @size: allocation size
670 *
671 * Allocate enough 32bit PA addressable pages to cover @size from the
672 * page level allocator and map them into contiguous kernel virtual space.
673 */
674void *vmalloc_32(unsigned long size)
675{
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200676 return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678EXPORT_SYMBOL(vmalloc_32);
679
Nick Piggin83342312006-06-23 02:03:20 -0700680/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700681 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
Nick Piggin83342312006-06-23 02:03:20 -0700682 * @size: allocation size
Rolf Eike Beeread04082006-09-27 01:50:13 -0700683 *
684 * The resulting memory area is 32bit addressable and zeroed so it can be
685 * mapped to userspace without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700686 */
687void *vmalloc_32_user(unsigned long size)
688{
689 struct vm_struct *area;
690 void *ret;
691
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200692 ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800693 if (ret) {
694 write_lock(&vmlist_lock);
695 area = __find_vm_area(ret);
696 area->flags |= VM_USERMAP;
697 write_unlock(&vmlist_lock);
698 }
Nick Piggin83342312006-06-23 02:03:20 -0700699 return ret;
700}
701EXPORT_SYMBOL(vmalloc_32_user);
702
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703long vread(char *buf, char *addr, unsigned long count)
704{
705 struct vm_struct *tmp;
706 char *vaddr, *buf_start = buf;
707 unsigned long n;
708
709 /* Don't allow overflow */
710 if ((unsigned long) addr + count < count)
711 count = -(unsigned long) addr;
712
713 read_lock(&vmlist_lock);
714 for (tmp = vmlist; tmp; tmp = tmp->next) {
715 vaddr = (char *) tmp->addr;
716 if (addr >= vaddr + tmp->size - PAGE_SIZE)
717 continue;
718 while (addr < vaddr) {
719 if (count == 0)
720 goto finished;
721 *buf = '\0';
722 buf++;
723 addr++;
724 count--;
725 }
726 n = vaddr + tmp->size - PAGE_SIZE - addr;
727 do {
728 if (count == 0)
729 goto finished;
730 *buf = *addr;
731 buf++;
732 addr++;
733 count--;
734 } while (--n > 0);
735 }
736finished:
737 read_unlock(&vmlist_lock);
738 return buf - buf_start;
739}
740
741long vwrite(char *buf, char *addr, unsigned long count)
742{
743 struct vm_struct *tmp;
744 char *vaddr, *buf_start = buf;
745 unsigned long n;
746
747 /* Don't allow overflow */
748 if ((unsigned long) addr + count < count)
749 count = -(unsigned long) addr;
750
751 read_lock(&vmlist_lock);
752 for (tmp = vmlist; tmp; tmp = tmp->next) {
753 vaddr = (char *) tmp->addr;
754 if (addr >= vaddr + tmp->size - PAGE_SIZE)
755 continue;
756 while (addr < vaddr) {
757 if (count == 0)
758 goto finished;
759 buf++;
760 addr++;
761 count--;
762 }
763 n = vaddr + tmp->size - PAGE_SIZE - addr;
764 do {
765 if (count == 0)
766 goto finished;
767 *addr = *buf;
768 buf++;
769 addr++;
770 count--;
771 } while (--n > 0);
772 }
773finished:
774 read_unlock(&vmlist_lock);
775 return buf - buf_start;
776}
Nick Piggin83342312006-06-23 02:03:20 -0700777
778/**
779 * remap_vmalloc_range - map vmalloc pages to userspace
Nick Piggin83342312006-06-23 02:03:20 -0700780 * @vma: vma to cover (map full range of vma)
781 * @addr: vmalloc memory
782 * @pgoff: number of pages into addr before first page to map
Randy Dunlap76824862008-03-19 17:00:40 -0700783 *
784 * Returns: 0 for success, -Exxx on failure
Nick Piggin83342312006-06-23 02:03:20 -0700785 *
786 * This function checks that addr is a valid vmalloc'ed area, and
787 * that it is big enough to cover the vma. Will return failure if
788 * that criteria isn't met.
789 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800790 * Similar to remap_pfn_range() (see mm/memory.c)
Nick Piggin83342312006-06-23 02:03:20 -0700791 */
792int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
793 unsigned long pgoff)
794{
795 struct vm_struct *area;
796 unsigned long uaddr = vma->vm_start;
797 unsigned long usize = vma->vm_end - vma->vm_start;
798 int ret;
799
800 if ((PAGE_SIZE-1) & (unsigned long)addr)
801 return -EINVAL;
802
803 read_lock(&vmlist_lock);
804 area = __find_vm_area(addr);
805 if (!area)
806 goto out_einval_locked;
807
808 if (!(area->flags & VM_USERMAP))
809 goto out_einval_locked;
810
811 if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
812 goto out_einval_locked;
813 read_unlock(&vmlist_lock);
814
815 addr += pgoff << PAGE_SHIFT;
816 do {
817 struct page *page = vmalloc_to_page(addr);
818 ret = vm_insert_page(vma, uaddr, page);
819 if (ret)
820 return ret;
821
822 uaddr += PAGE_SIZE;
823 addr += PAGE_SIZE;
824 usize -= PAGE_SIZE;
825 } while (usize > 0);
826
827 /* Prevent "things" like memory migration? VM_flags need a cleanup... */
828 vma->vm_flags |= VM_RESERVED;
829
830 return ret;
831
832out_einval_locked:
833 read_unlock(&vmlist_lock);
834 return -EINVAL;
835}
836EXPORT_SYMBOL(remap_vmalloc_range);
837
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -0700838/*
839 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
840 * have one.
841 */
842void __attribute__((weak)) vmalloc_sync_all(void)
843{
844}
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700845
846
Martin Schwidefsky2f569af2008-02-08 04:22:04 -0800847static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700848{
849 /* apply_to_page_range() does all the hard work. */
850 return 0;
851}
852
853/**
854 * alloc_vm_area - allocate a range of kernel address space
855 * @size: size of the area
Randy Dunlap76824862008-03-19 17:00:40 -0700856 *
857 * Returns: NULL on failure, vm_struct on success
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700858 *
859 * This function reserves a range of kernel address space, and
860 * allocates pagetables to map that range. No actual mappings
861 * are created. If the kernel address space is not shared
862 * between processes, it syncs the pagetable across all
863 * processes.
864 */
865struct vm_struct *alloc_vm_area(size_t size)
866{
867 struct vm_struct *area;
868
Christoph Lameter23016962008-04-28 02:12:42 -0700869 area = get_vm_area_caller(size, VM_IOREMAP,
870 __builtin_return_address(0));
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700871 if (area == NULL)
872 return NULL;
873
874 /*
875 * This ensures that page tables are constructed for this region
876 * of kernel virtual address space and mapped into init_mm.
877 */
878 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
879 area->size, f, NULL)) {
880 free_vm_area(area);
881 return NULL;
882 }
883
884 /* Make sure the pagetables are constructed in process kernel
885 mappings */
886 vmalloc_sync_all();
887
888 return area;
889}
890EXPORT_SYMBOL_GPL(alloc_vm_area);
891
892void free_vm_area(struct vm_struct *area)
893{
894 struct vm_struct *ret;
895 ret = remove_vm_area(area->addr);
896 BUG_ON(ret != area);
897 kfree(area);
898}
899EXPORT_SYMBOL_GPL(free_vm_area);
Christoph Lametera10aa572008-04-28 02:12:40 -0700900
901
902#ifdef CONFIG_PROC_FS
903static void *s_start(struct seq_file *m, loff_t *pos)
904{
905 loff_t n = *pos;
906 struct vm_struct *v;
907
908 read_lock(&vmlist_lock);
909 v = vmlist;
910 while (n > 0 && v) {
911 n--;
912 v = v->next;
913 }
914 if (!n)
915 return v;
916
917 return NULL;
918
919}
920
921static void *s_next(struct seq_file *m, void *p, loff_t *pos)
922{
923 struct vm_struct *v = p;
924
925 ++*pos;
926 return v->next;
927}
928
929static void s_stop(struct seq_file *m, void *p)
930{
931 read_unlock(&vmlist_lock);
932}
933
934static int s_show(struct seq_file *m, void *p)
935{
936 struct vm_struct *v = p;
937
938 seq_printf(m, "0x%p-0x%p %7ld",
939 v->addr, v->addr + v->size, v->size);
940
Christoph Lameter23016962008-04-28 02:12:42 -0700941 if (v->caller) {
942 char buff[2 * KSYM_NAME_LEN];
943
944 seq_putc(m, ' ');
945 sprint_symbol(buff, (unsigned long)v->caller);
946 seq_puts(m, buff);
947 }
948
Christoph Lametera10aa572008-04-28 02:12:40 -0700949 if (v->nr_pages)
950 seq_printf(m, " pages=%d", v->nr_pages);
951
952 if (v->phys_addr)
953 seq_printf(m, " phys=%lx", v->phys_addr);
954
955 if (v->flags & VM_IOREMAP)
956 seq_printf(m, " ioremap");
957
958 if (v->flags & VM_ALLOC)
959 seq_printf(m, " vmalloc");
960
961 if (v->flags & VM_MAP)
962 seq_printf(m, " vmap");
963
964 if (v->flags & VM_USERMAP)
965 seq_printf(m, " user");
966
967 if (v->flags & VM_VPAGES)
968 seq_printf(m, " vpages");
969
970 seq_putc(m, '\n');
971 return 0;
972}
973
974const struct seq_operations vmalloc_op = {
975 .start = s_start,
976 .next = s_next,
977 .stop = s_stop,
978 .show = s_show,
979};
980#endif
981