blob: e33e0ae69ad142d6241921d64cfaed37abfdb51a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
Christoph Lameter930fc452005-10-29 18:15:41 -07008 * Numa awareness, Christoph Lameter, SGI, June 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/highmem.h>
14#include <linux/slab.h>
15#include <linux/spinlock.h>
16#include <linux/interrupt.h>
Christoph Lametera10aa572008-04-28 02:12:40 -070017#include <linux/seq_file.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/vmalloc.h>
Christoph Lameter23016962008-04-28 02:12:42 -070019#include <linux/kallsyms.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21#include <asm/uaccess.h>
22#include <asm/tlbflush.h>
23
24
25DEFINE_RWLOCK(vmlist_lock);
26struct vm_struct *vmlist;
27
Adrian Bunkb2213852006-09-25 23:31:02 -070028static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
Christoph Lameter23016962008-04-28 02:12:42 -070029 int node, void *caller);
Adrian Bunkb2213852006-09-25 23:31:02 -070030
Linus Torvalds1da177e2005-04-16 15:20:36 -070031static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
32{
33 pte_t *pte;
34
35 pte = pte_offset_kernel(pmd, addr);
36 do {
37 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
38 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
39 } while (pte++, addr += PAGE_SIZE, addr != end);
40}
41
42static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
43 unsigned long end)
44{
45 pmd_t *pmd;
46 unsigned long next;
47
48 pmd = pmd_offset(pud, addr);
49 do {
50 next = pmd_addr_end(addr, end);
51 if (pmd_none_or_clear_bad(pmd))
52 continue;
53 vunmap_pte_range(pmd, addr, next);
54 } while (pmd++, addr = next, addr != end);
55}
56
57static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
58 unsigned long end)
59{
60 pud_t *pud;
61 unsigned long next;
62
63 pud = pud_offset(pgd, addr);
64 do {
65 next = pud_addr_end(addr, end);
66 if (pud_none_or_clear_bad(pud))
67 continue;
68 vunmap_pmd_range(pud, addr, next);
69 } while (pud++, addr = next, addr != end);
70}
71
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100072void unmap_kernel_range(unsigned long addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -070073{
74 pgd_t *pgd;
75 unsigned long next;
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100076 unsigned long start = addr;
77 unsigned long end = addr + size;
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
79 BUG_ON(addr >= end);
80 pgd = pgd_offset_k(addr);
81 flush_cache_vunmap(addr, end);
82 do {
83 next = pgd_addr_end(addr, end);
84 if (pgd_none_or_clear_bad(pgd))
85 continue;
86 vunmap_pud_range(pgd, addr, next);
87 } while (pgd++, addr = next, addr != end);
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100088 flush_tlb_kernel_range(start, end);
89}
90
91static void unmap_vm_area(struct vm_struct *area)
92{
93 unmap_kernel_range((unsigned long)area->addr, area->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -070094}
95
96static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
97 unsigned long end, pgprot_t prot, struct page ***pages)
98{
99 pte_t *pte;
100
Hugh Dickins872fec12005-10-29 18:16:21 -0700101 pte = pte_alloc_kernel(pmd, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 if (!pte)
103 return -ENOMEM;
104 do {
105 struct page *page = **pages;
106 WARN_ON(!pte_none(*pte));
107 if (!page)
108 return -ENOMEM;
109 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
110 (*pages)++;
111 } while (pte++, addr += PAGE_SIZE, addr != end);
112 return 0;
113}
114
115static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
116 unsigned long end, pgprot_t prot, struct page ***pages)
117{
118 pmd_t *pmd;
119 unsigned long next;
120
121 pmd = pmd_alloc(&init_mm, pud, addr);
122 if (!pmd)
123 return -ENOMEM;
124 do {
125 next = pmd_addr_end(addr, end);
126 if (vmap_pte_range(pmd, addr, next, prot, pages))
127 return -ENOMEM;
128 } while (pmd++, addr = next, addr != end);
129 return 0;
130}
131
132static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
133 unsigned long end, pgprot_t prot, struct page ***pages)
134{
135 pud_t *pud;
136 unsigned long next;
137
138 pud = pud_alloc(&init_mm, pgd, addr);
139 if (!pud)
140 return -ENOMEM;
141 do {
142 next = pud_addr_end(addr, end);
143 if (vmap_pmd_range(pud, addr, next, prot, pages))
144 return -ENOMEM;
145 } while (pud++, addr = next, addr != end);
146 return 0;
147}
148
149int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
150{
151 pgd_t *pgd;
152 unsigned long next;
153 unsigned long addr = (unsigned long) area->addr;
154 unsigned long end = addr + area->size - PAGE_SIZE;
155 int err;
156
157 BUG_ON(addr >= end);
158 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 do {
160 next = pgd_addr_end(addr, end);
161 err = vmap_pud_range(pgd, addr, next, prot, pages);
162 if (err)
163 break;
164 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 flush_cache_vmap((unsigned long) area->addr, end);
166 return err;
167}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700168EXPORT_SYMBOL_GPL(map_vm_area);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169
Christoph Lameter48667e72008-02-04 22:28:31 -0800170/*
171 * Map a vmalloc()-space virtual address to the physical page.
172 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800173struct page *vmalloc_to_page(const void *vmalloc_addr)
Christoph Lameter48667e72008-02-04 22:28:31 -0800174{
175 unsigned long addr = (unsigned long) vmalloc_addr;
176 struct page *page = NULL;
177 pgd_t *pgd = pgd_offset_k(addr);
178 pud_t *pud;
179 pmd_t *pmd;
180 pte_t *ptep, pte;
181
182 if (!pgd_none(*pgd)) {
183 pud = pud_offset(pgd, addr);
184 if (!pud_none(*pud)) {
185 pmd = pmd_offset(pud, addr);
186 if (!pmd_none(*pmd)) {
187 ptep = pte_offset_map(pmd, addr);
188 pte = *ptep;
189 if (pte_present(pte))
190 page = pte_page(pte);
191 pte_unmap(ptep);
192 }
193 }
194 }
195 return page;
196}
197EXPORT_SYMBOL(vmalloc_to_page);
198
199/*
200 * Map a vmalloc()-space virtual address to the physical page frame number.
201 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800202unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
Christoph Lameter48667e72008-02-04 22:28:31 -0800203{
204 return page_to_pfn(vmalloc_to_page(vmalloc_addr));
205}
206EXPORT_SYMBOL(vmalloc_to_pfn);
207
Christoph Lameter23016962008-04-28 02:12:42 -0700208static struct vm_struct *
209__get_vm_area_node(unsigned long size, unsigned long flags, unsigned long start,
210 unsigned long end, int node, gfp_t gfp_mask, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211{
212 struct vm_struct **p, *tmp, *area;
213 unsigned long align = 1;
214 unsigned long addr;
215
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700216 BUG_ON(in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 if (flags & VM_IOREMAP) {
218 int bit = fls(size);
219
220 if (bit > IOREMAP_MAX_ORDER)
221 bit = IOREMAP_MAX_ORDER;
222 else if (bit < PAGE_SHIFT)
223 bit = PAGE_SHIFT;
224
225 align = 1ul << bit;
226 }
227 addr = ALIGN(start, align);
228 size = PAGE_ALIGN(size);
OGAWA Hirofumi31be8302006-11-16 01:19:29 -0800229 if (unlikely(!size))
230 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231
Christoph Lameter6cb06222007-10-16 01:25:41 -0700232 area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
233
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 if (unlikely(!area))
235 return NULL;
236
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 /*
238 * We always allocate a guard page.
239 */
240 size += PAGE_SIZE;
241
242 write_lock(&vmlist_lock);
243 for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
244 if ((unsigned long)tmp->addr < addr) {
245 if((unsigned long)tmp->addr + tmp->size >= addr)
246 addr = ALIGN(tmp->size +
247 (unsigned long)tmp->addr, align);
248 continue;
249 }
250 if ((size + addr) < addr)
251 goto out;
252 if (size + addr <= (unsigned long)tmp->addr)
253 goto found;
254 addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
255 if (addr > end - size)
256 goto out;
257 }
Robert Bragg5dc33182008-02-04 22:29:18 -0800258 if ((size + addr) < addr)
259 goto out;
260 if (addr > end - size)
261 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262
263found:
264 area->next = *p;
265 *p = area;
266
267 area->flags = flags;
268 area->addr = (void *)addr;
269 area->size = size;
270 area->pages = NULL;
271 area->nr_pages = 0;
272 area->phys_addr = 0;
Christoph Lameter23016962008-04-28 02:12:42 -0700273 area->caller = caller;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 write_unlock(&vmlist_lock);
275
276 return area;
277
278out:
279 write_unlock(&vmlist_lock);
280 kfree(area);
281 if (printk_ratelimit())
282 printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
283 return NULL;
284}
285
Christoph Lameter930fc452005-10-29 18:15:41 -0700286struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
287 unsigned long start, unsigned long end)
288{
Christoph Lameter23016962008-04-28 02:12:42 -0700289 return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL,
290 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700291}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700292EXPORT_SYMBOL_GPL(__get_vm_area);
Christoph Lameter930fc452005-10-29 18:15:41 -0700293
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294/**
Simon Arlott183ff222007-10-20 01:27:18 +0200295 * get_vm_area - reserve a contiguous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 * @size: size of the area
297 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
298 *
299 * Search an area of @size in the kernel virtual mapping area,
300 * and reserved it for out purposes. Returns the area descriptor
301 * on success or %NULL on failure.
302 */
303struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
304{
Christoph Lameter23016962008-04-28 02:12:42 -0700305 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
306 -1, GFP_KERNEL, __builtin_return_address(0));
307}
308
309struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
310 void *caller)
311{
312 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
313 -1, GFP_KERNEL, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314}
315
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700316struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
317 int node, gfp_t gfp_mask)
Christoph Lameter930fc452005-10-29 18:15:41 -0700318{
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700319 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
Christoph Lameter23016962008-04-28 02:12:42 -0700320 gfp_mask, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700321}
322
Andi Kleen7856dfe2005-05-20 14:27:57 -0700323/* Caller must hold vmlist_lock */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800324static struct vm_struct *__find_vm_area(const void *addr)
Nick Piggin83342312006-06-23 02:03:20 -0700325{
326 struct vm_struct *tmp;
327
328 for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
329 if (tmp->addr == addr)
330 break;
331 }
332
333 return tmp;
334}
335
336/* Caller must hold vmlist_lock */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800337static struct vm_struct *__remove_vm_area(const void *addr)
Andi Kleen7856dfe2005-05-20 14:27:57 -0700338{
339 struct vm_struct **p, *tmp;
340
341 for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
342 if (tmp->addr == addr)
343 goto found;
344 }
345 return NULL;
346
347found:
348 unmap_vm_area(tmp);
349 *p = tmp->next;
350
351 /*
352 * Remove the guard page.
353 */
354 tmp->size -= PAGE_SIZE;
355 return tmp;
356}
357
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358/**
Simon Arlott183ff222007-10-20 01:27:18 +0200359 * remove_vm_area - find and remove a continuous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 * @addr: base address
361 *
362 * Search for the kernel VM area starting at @addr, and remove it.
363 * This function returns the found VM area, but using it is NOT safe
Andi Kleen7856dfe2005-05-20 14:27:57 -0700364 * on SMP machines, except for its size or flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800366struct vm_struct *remove_vm_area(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367{
Andi Kleen7856dfe2005-05-20 14:27:57 -0700368 struct vm_struct *v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 write_lock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700370 v = __remove_vm_area(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 write_unlock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700372 return v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373}
374
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800375static void __vunmap(const void *addr, int deallocate_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
377 struct vm_struct *area;
378
379 if (!addr)
380 return;
381
382 if ((PAGE_SIZE-1) & (unsigned long)addr) {
383 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
384 WARN_ON(1);
385 return;
386 }
387
388 area = remove_vm_area(addr);
389 if (unlikely(!area)) {
390 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
391 addr);
392 WARN_ON(1);
393 return;
394 }
395
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700396 debug_check_no_locks_freed(addr, area->size);
397
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 if (deallocate_pages) {
399 int i;
400
401 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800402 struct page *page = area->pages[i];
403
404 BUG_ON(!page);
405 __free_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 }
407
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700408 if (area->flags & VM_VPAGES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 vfree(area->pages);
410 else
411 kfree(area->pages);
412 }
413
414 kfree(area);
415 return;
416}
417
418/**
419 * vfree - release memory allocated by vmalloc()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 * @addr: memory base address
421 *
Simon Arlott183ff222007-10-20 01:27:18 +0200422 * Free the virtually continuous memory area starting at @addr, as
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700423 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
424 * NULL, no operation is performed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700426 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800428void vfree(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429{
430 BUG_ON(in_interrupt());
431 __vunmap(addr, 1);
432}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433EXPORT_SYMBOL(vfree);
434
435/**
436 * vunmap - release virtual mapping obtained by vmap()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 * @addr: memory base address
438 *
439 * Free the virtually contiguous memory area starting at @addr,
440 * which was created from the page array passed to vmap().
441 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700442 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800444void vunmap(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445{
446 BUG_ON(in_interrupt());
447 __vunmap(addr, 0);
448}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449EXPORT_SYMBOL(vunmap);
450
451/**
452 * vmap - map an array of pages into virtually contiguous space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 * @pages: array of page pointers
454 * @count: number of pages to map
455 * @flags: vm_area->flags
456 * @prot: page protection for the mapping
457 *
458 * Maps @count pages from @pages into contiguous kernel virtual
459 * space.
460 */
461void *vmap(struct page **pages, unsigned int count,
462 unsigned long flags, pgprot_t prot)
463{
464 struct vm_struct *area;
465
466 if (count > num_physpages)
467 return NULL;
468
Christoph Lameter23016962008-04-28 02:12:42 -0700469 area = get_vm_area_caller((count << PAGE_SHIFT), flags,
470 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 if (!area)
472 return NULL;
Christoph Lameter23016962008-04-28 02:12:42 -0700473
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 if (map_vm_area(area, prot, &pages)) {
475 vunmap(area->addr);
476 return NULL;
477 }
478
479 return area->addr;
480}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481EXPORT_SYMBOL(vmap);
482
Adrian Bunke31d9eb2008-02-04 22:29:09 -0800483static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
Christoph Lameter23016962008-04-28 02:12:42 -0700484 pgprot_t prot, int node, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485{
486 struct page **pages;
487 unsigned int nr_pages, array_size, i;
488
489 nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
490 array_size = (nr_pages * sizeof(struct page *));
491
492 area->nr_pages = nr_pages;
493 /* Please note that the recursion is strictly bounded. */
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700494 if (array_size > PAGE_SIZE) {
Christoph Lameter94f60302007-07-17 04:03:29 -0700495 pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
Christoph Lameter23016962008-04-28 02:12:42 -0700496 PAGE_KERNEL, node, caller);
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700497 area->flags |= VM_VPAGES;
Andrew Morton286e1ea2006-10-17 00:09:57 -0700498 } else {
499 pages = kmalloc_node(array_size,
Christoph Lameter6cb06222007-10-16 01:25:41 -0700500 (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO,
Andrew Morton286e1ea2006-10-17 00:09:57 -0700501 node);
502 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 area->pages = pages;
Christoph Lameter23016962008-04-28 02:12:42 -0700504 area->caller = caller;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 if (!area->pages) {
506 remove_vm_area(area->addr);
507 kfree(area);
508 return NULL;
509 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510
511 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800512 struct page *page;
513
Christoph Lameter930fc452005-10-29 18:15:41 -0700514 if (node < 0)
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800515 page = alloc_page(gfp_mask);
Christoph Lameter930fc452005-10-29 18:15:41 -0700516 else
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800517 page = alloc_pages_node(node, gfp_mask, 0);
518
519 if (unlikely(!page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 /* Successfully allocated i pages, free them in __vunmap() */
521 area->nr_pages = i;
522 goto fail;
523 }
Christoph Lameterbf53d6f2008-02-04 22:28:34 -0800524 area->pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 }
526
527 if (map_vm_area(area, prot, &pages))
528 goto fail;
529 return area->addr;
530
531fail:
532 vfree(area->addr);
533 return NULL;
534}
535
Christoph Lameter930fc452005-10-29 18:15:41 -0700536void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
537{
Christoph Lameter23016962008-04-28 02:12:42 -0700538 return __vmalloc_area_node(area, gfp_mask, prot, -1,
539 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700540}
541
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700543 * __vmalloc_node - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 * @size: allocation size
545 * @gfp_mask: flags for the page level allocator
546 * @prot: protection mask for the allocated pages
Randy Dunlapd44e0782005-11-07 01:01:10 -0800547 * @node: node to use for allocation or -1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 *
549 * Allocate enough pages to cover @size from the page level
550 * allocator with @gfp_mask flags. Map them into contiguous
551 * kernel virtual space, using a pagetable protection of @prot.
552 */
Adrian Bunkb2213852006-09-25 23:31:02 -0700553static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
Christoph Lameter23016962008-04-28 02:12:42 -0700554 int node, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555{
556 struct vm_struct *area;
557
558 size = PAGE_ALIGN(size);
559 if (!size || (size >> PAGE_SHIFT) > num_physpages)
560 return NULL;
561
Christoph Lameter23016962008-04-28 02:12:42 -0700562 area = __get_vm_area_node(size, VM_ALLOC, VMALLOC_START, VMALLOC_END,
563 node, gfp_mask, caller);
564
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 if (!area)
566 return NULL;
567
Christoph Lameter23016962008-04-28 02:12:42 -0700568 return __vmalloc_area_node(area, gfp_mask, prot, node, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569}
570
Christoph Lameter930fc452005-10-29 18:15:41 -0700571void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
572{
Christoph Lameter23016962008-04-28 02:12:42 -0700573 return __vmalloc_node(size, gfp_mask, prot, -1,
574 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700575}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576EXPORT_SYMBOL(__vmalloc);
577
578/**
579 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 * @size: allocation size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 * Allocate enough pages to cover @size from the page level
582 * allocator and map them into contiguous kernel virtual space.
583 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200584 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585 * use __vmalloc() instead.
586 */
587void *vmalloc(unsigned long size)
588{
Christoph Lameter23016962008-04-28 02:12:42 -0700589 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
590 -1, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592EXPORT_SYMBOL(vmalloc);
593
Christoph Lameter930fc452005-10-29 18:15:41 -0700594/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700595 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
596 * @size: allocation size
Nick Piggin83342312006-06-23 02:03:20 -0700597 *
Rolf Eike Beeread04082006-09-27 01:50:13 -0700598 * The resulting memory area is zeroed so it can be mapped to userspace
599 * without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700600 */
601void *vmalloc_user(unsigned long size)
602{
603 struct vm_struct *area;
604 void *ret;
605
606 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800607 if (ret) {
608 write_lock(&vmlist_lock);
609 area = __find_vm_area(ret);
610 area->flags |= VM_USERMAP;
611 write_unlock(&vmlist_lock);
612 }
Nick Piggin83342312006-06-23 02:03:20 -0700613 return ret;
614}
615EXPORT_SYMBOL(vmalloc_user);
616
617/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700618 * vmalloc_node - allocate memory on a specific node
Christoph Lameter930fc452005-10-29 18:15:41 -0700619 * @size: allocation size
Randy Dunlapd44e0782005-11-07 01:01:10 -0800620 * @node: numa node
Christoph Lameter930fc452005-10-29 18:15:41 -0700621 *
622 * Allocate enough pages to cover @size from the page level
623 * allocator and map them into contiguous kernel virtual space.
624 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200625 * For tight control over page level allocator and protection flags
Christoph Lameter930fc452005-10-29 18:15:41 -0700626 * use __vmalloc() instead.
627 */
628void *vmalloc_node(unsigned long size, int node)
629{
Christoph Lameter23016962008-04-28 02:12:42 -0700630 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
631 node, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -0700632}
633EXPORT_SYMBOL(vmalloc_node);
634
Pavel Pisa4dc3b162005-05-01 08:59:25 -0700635#ifndef PAGE_KERNEL_EXEC
636# define PAGE_KERNEL_EXEC PAGE_KERNEL
637#endif
638
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639/**
640 * vmalloc_exec - allocate virtually contiguous, executable memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641 * @size: allocation size
642 *
643 * Kernel-internal function to allocate enough pages to cover @size
644 * the page level allocator and map them into contiguous and
645 * executable kernel virtual space.
646 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200647 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 * use __vmalloc() instead.
649 */
650
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651void *vmalloc_exec(unsigned long size)
652{
653 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
654}
655
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200656#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700657#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200658#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700659#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200660#else
661#define GFP_VMALLOC32 GFP_KERNEL
662#endif
663
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664/**
665 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 * @size: allocation size
667 *
668 * Allocate enough 32bit PA addressable pages to cover @size from the
669 * page level allocator and map them into contiguous kernel virtual space.
670 */
671void *vmalloc_32(unsigned long size)
672{
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200673 return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675EXPORT_SYMBOL(vmalloc_32);
676
Nick Piggin83342312006-06-23 02:03:20 -0700677/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700678 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
Nick Piggin83342312006-06-23 02:03:20 -0700679 * @size: allocation size
Rolf Eike Beeread04082006-09-27 01:50:13 -0700680 *
681 * The resulting memory area is 32bit addressable and zeroed so it can be
682 * mapped to userspace without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700683 */
684void *vmalloc_32_user(unsigned long size)
685{
686 struct vm_struct *area;
687 void *ret;
688
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200689 ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800690 if (ret) {
691 write_lock(&vmlist_lock);
692 area = __find_vm_area(ret);
693 area->flags |= VM_USERMAP;
694 write_unlock(&vmlist_lock);
695 }
Nick Piggin83342312006-06-23 02:03:20 -0700696 return ret;
697}
698EXPORT_SYMBOL(vmalloc_32_user);
699
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700long vread(char *buf, char *addr, unsigned long count)
701{
702 struct vm_struct *tmp;
703 char *vaddr, *buf_start = buf;
704 unsigned long n;
705
706 /* Don't allow overflow */
707 if ((unsigned long) addr + count < count)
708 count = -(unsigned long) addr;
709
710 read_lock(&vmlist_lock);
711 for (tmp = vmlist; tmp; tmp = tmp->next) {
712 vaddr = (char *) tmp->addr;
713 if (addr >= vaddr + tmp->size - PAGE_SIZE)
714 continue;
715 while (addr < vaddr) {
716 if (count == 0)
717 goto finished;
718 *buf = '\0';
719 buf++;
720 addr++;
721 count--;
722 }
723 n = vaddr + tmp->size - PAGE_SIZE - addr;
724 do {
725 if (count == 0)
726 goto finished;
727 *buf = *addr;
728 buf++;
729 addr++;
730 count--;
731 } while (--n > 0);
732 }
733finished:
734 read_unlock(&vmlist_lock);
735 return buf - buf_start;
736}
737
738long vwrite(char *buf, char *addr, unsigned long count)
739{
740 struct vm_struct *tmp;
741 char *vaddr, *buf_start = buf;
742 unsigned long n;
743
744 /* Don't allow overflow */
745 if ((unsigned long) addr + count < count)
746 count = -(unsigned long) addr;
747
748 read_lock(&vmlist_lock);
749 for (tmp = vmlist; tmp; tmp = tmp->next) {
750 vaddr = (char *) tmp->addr;
751 if (addr >= vaddr + tmp->size - PAGE_SIZE)
752 continue;
753 while (addr < vaddr) {
754 if (count == 0)
755 goto finished;
756 buf++;
757 addr++;
758 count--;
759 }
760 n = vaddr + tmp->size - PAGE_SIZE - addr;
761 do {
762 if (count == 0)
763 goto finished;
764 *addr = *buf;
765 buf++;
766 addr++;
767 count--;
768 } while (--n > 0);
769 }
770finished:
771 read_unlock(&vmlist_lock);
772 return buf - buf_start;
773}
Nick Piggin83342312006-06-23 02:03:20 -0700774
775/**
776 * remap_vmalloc_range - map vmalloc pages to userspace
Nick Piggin83342312006-06-23 02:03:20 -0700777 * @vma: vma to cover (map full range of vma)
778 * @addr: vmalloc memory
779 * @pgoff: number of pages into addr before first page to map
Randy Dunlap76824862008-03-19 17:00:40 -0700780 *
781 * Returns: 0 for success, -Exxx on failure
Nick Piggin83342312006-06-23 02:03:20 -0700782 *
783 * This function checks that addr is a valid vmalloc'ed area, and
784 * that it is big enough to cover the vma. Will return failure if
785 * that criteria isn't met.
786 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800787 * Similar to remap_pfn_range() (see mm/memory.c)
Nick Piggin83342312006-06-23 02:03:20 -0700788 */
789int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
790 unsigned long pgoff)
791{
792 struct vm_struct *area;
793 unsigned long uaddr = vma->vm_start;
794 unsigned long usize = vma->vm_end - vma->vm_start;
795 int ret;
796
797 if ((PAGE_SIZE-1) & (unsigned long)addr)
798 return -EINVAL;
799
800 read_lock(&vmlist_lock);
801 area = __find_vm_area(addr);
802 if (!area)
803 goto out_einval_locked;
804
805 if (!(area->flags & VM_USERMAP))
806 goto out_einval_locked;
807
808 if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
809 goto out_einval_locked;
810 read_unlock(&vmlist_lock);
811
812 addr += pgoff << PAGE_SHIFT;
813 do {
814 struct page *page = vmalloc_to_page(addr);
815 ret = vm_insert_page(vma, uaddr, page);
816 if (ret)
817 return ret;
818
819 uaddr += PAGE_SIZE;
820 addr += PAGE_SIZE;
821 usize -= PAGE_SIZE;
822 } while (usize > 0);
823
824 /* Prevent "things" like memory migration? VM_flags need a cleanup... */
825 vma->vm_flags |= VM_RESERVED;
826
827 return ret;
828
829out_einval_locked:
830 read_unlock(&vmlist_lock);
831 return -EINVAL;
832}
833EXPORT_SYMBOL(remap_vmalloc_range);
834
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -0700835/*
836 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
837 * have one.
838 */
839void __attribute__((weak)) vmalloc_sync_all(void)
840{
841}
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700842
843
Martin Schwidefsky2f569af2008-02-08 04:22:04 -0800844static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700845{
846 /* apply_to_page_range() does all the hard work. */
847 return 0;
848}
849
850/**
851 * alloc_vm_area - allocate a range of kernel address space
852 * @size: size of the area
Randy Dunlap76824862008-03-19 17:00:40 -0700853 *
854 * Returns: NULL on failure, vm_struct on success
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700855 *
856 * This function reserves a range of kernel address space, and
857 * allocates pagetables to map that range. No actual mappings
858 * are created. If the kernel address space is not shared
859 * between processes, it syncs the pagetable across all
860 * processes.
861 */
862struct vm_struct *alloc_vm_area(size_t size)
863{
864 struct vm_struct *area;
865
Christoph Lameter23016962008-04-28 02:12:42 -0700866 area = get_vm_area_caller(size, VM_IOREMAP,
867 __builtin_return_address(0));
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700868 if (area == NULL)
869 return NULL;
870
871 /*
872 * This ensures that page tables are constructed for this region
873 * of kernel virtual address space and mapped into init_mm.
874 */
875 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
876 area->size, f, NULL)) {
877 free_vm_area(area);
878 return NULL;
879 }
880
881 /* Make sure the pagetables are constructed in process kernel
882 mappings */
883 vmalloc_sync_all();
884
885 return area;
886}
887EXPORT_SYMBOL_GPL(alloc_vm_area);
888
889void free_vm_area(struct vm_struct *area)
890{
891 struct vm_struct *ret;
892 ret = remove_vm_area(area->addr);
893 BUG_ON(ret != area);
894 kfree(area);
895}
896EXPORT_SYMBOL_GPL(free_vm_area);
Christoph Lametera10aa572008-04-28 02:12:40 -0700897
898
899#ifdef CONFIG_PROC_FS
900static void *s_start(struct seq_file *m, loff_t *pos)
901{
902 loff_t n = *pos;
903 struct vm_struct *v;
904
905 read_lock(&vmlist_lock);
906 v = vmlist;
907 while (n > 0 && v) {
908 n--;
909 v = v->next;
910 }
911 if (!n)
912 return v;
913
914 return NULL;
915
916}
917
918static void *s_next(struct seq_file *m, void *p, loff_t *pos)
919{
920 struct vm_struct *v = p;
921
922 ++*pos;
923 return v->next;
924}
925
926static void s_stop(struct seq_file *m, void *p)
927{
928 read_unlock(&vmlist_lock);
929}
930
931static int s_show(struct seq_file *m, void *p)
932{
933 struct vm_struct *v = p;
934
935 seq_printf(m, "0x%p-0x%p %7ld",
936 v->addr, v->addr + v->size, v->size);
937
Christoph Lameter23016962008-04-28 02:12:42 -0700938 if (v->caller) {
939 char buff[2 * KSYM_NAME_LEN];
940
941 seq_putc(m, ' ');
942 sprint_symbol(buff, (unsigned long)v->caller);
943 seq_puts(m, buff);
944 }
945
Christoph Lametera10aa572008-04-28 02:12:40 -0700946 if (v->nr_pages)
947 seq_printf(m, " pages=%d", v->nr_pages);
948
949 if (v->phys_addr)
950 seq_printf(m, " phys=%lx", v->phys_addr);
951
952 if (v->flags & VM_IOREMAP)
953 seq_printf(m, " ioremap");
954
955 if (v->flags & VM_ALLOC)
956 seq_printf(m, " vmalloc");
957
958 if (v->flags & VM_MAP)
959 seq_printf(m, " vmap");
960
961 if (v->flags & VM_USERMAP)
962 seq_printf(m, " user");
963
964 if (v->flags & VM_VPAGES)
965 seq_printf(m, " vpages");
966
967 seq_putc(m, '\n');
968 return 0;
969}
970
971const struct seq_operations vmalloc_op = {
972 .start = s_start,
973 .next = s_next,
974 .stop = s_stop,
975 .show = s_show,
976};
977#endif
978