blob: 3cee76a8c9f0b621e3479cac9f6320b0d01bdddd [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
Christoph Lameter930fc452005-10-29 18:15:41 -07008 * Numa awareness, Christoph Lameter, SGI, June 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/highmem.h>
14#include <linux/slab.h>
15#include <linux/spinlock.h>
16#include <linux/interrupt.h>
17
18#include <linux/vmalloc.h>
19
20#include <asm/uaccess.h>
21#include <asm/tlbflush.h>
22
23
24DEFINE_RWLOCK(vmlist_lock);
25struct vm_struct *vmlist;
26
Adrian Bunkb2213852006-09-25 23:31:02 -070027static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
28 int node);
29
Linus Torvalds1da177e2005-04-16 15:20:36 -070030static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
31{
32 pte_t *pte;
33
34 pte = pte_offset_kernel(pmd, addr);
35 do {
36 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
37 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
38 } while (pte++, addr += PAGE_SIZE, addr != end);
39}
40
41static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
42 unsigned long end)
43{
44 pmd_t *pmd;
45 unsigned long next;
46
47 pmd = pmd_offset(pud, addr);
48 do {
49 next = pmd_addr_end(addr, end);
50 if (pmd_none_or_clear_bad(pmd))
51 continue;
52 vunmap_pte_range(pmd, addr, next);
53 } while (pmd++, addr = next, addr != end);
54}
55
56static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
57 unsigned long end)
58{
59 pud_t *pud;
60 unsigned long next;
61
62 pud = pud_offset(pgd, addr);
63 do {
64 next = pud_addr_end(addr, end);
65 if (pud_none_or_clear_bad(pud))
66 continue;
67 vunmap_pmd_range(pud, addr, next);
68 } while (pud++, addr = next, addr != end);
69}
70
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100071void unmap_kernel_range(unsigned long addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072{
73 pgd_t *pgd;
74 unsigned long next;
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100075 unsigned long start = addr;
76 unsigned long end = addr + size;
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78 BUG_ON(addr >= end);
79 pgd = pgd_offset_k(addr);
80 flush_cache_vunmap(addr, end);
81 do {
82 next = pgd_addr_end(addr, end);
83 if (pgd_none_or_clear_bad(pgd))
84 continue;
85 vunmap_pud_range(pgd, addr, next);
86 } while (pgd++, addr = next, addr != end);
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100087 flush_tlb_kernel_range(start, end);
88}
89
90static void unmap_vm_area(struct vm_struct *area)
91{
92 unmap_kernel_range((unsigned long)area->addr, area->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -070093}
94
95static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
96 unsigned long end, pgprot_t prot, struct page ***pages)
97{
98 pte_t *pte;
99
Hugh Dickins872fec12005-10-29 18:16:21 -0700100 pte = pte_alloc_kernel(pmd, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 if (!pte)
102 return -ENOMEM;
103 do {
104 struct page *page = **pages;
105 WARN_ON(!pte_none(*pte));
106 if (!page)
107 return -ENOMEM;
108 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
109 (*pages)++;
110 } while (pte++, addr += PAGE_SIZE, addr != end);
111 return 0;
112}
113
114static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
115 unsigned long end, pgprot_t prot, struct page ***pages)
116{
117 pmd_t *pmd;
118 unsigned long next;
119
120 pmd = pmd_alloc(&init_mm, pud, addr);
121 if (!pmd)
122 return -ENOMEM;
123 do {
124 next = pmd_addr_end(addr, end);
125 if (vmap_pte_range(pmd, addr, next, prot, pages))
126 return -ENOMEM;
127 } while (pmd++, addr = next, addr != end);
128 return 0;
129}
130
131static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
132 unsigned long end, pgprot_t prot, struct page ***pages)
133{
134 pud_t *pud;
135 unsigned long next;
136
137 pud = pud_alloc(&init_mm, pgd, addr);
138 if (!pud)
139 return -ENOMEM;
140 do {
141 next = pud_addr_end(addr, end);
142 if (vmap_pmd_range(pud, addr, next, prot, pages))
143 return -ENOMEM;
144 } while (pud++, addr = next, addr != end);
145 return 0;
146}
147
148int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
149{
150 pgd_t *pgd;
151 unsigned long next;
152 unsigned long addr = (unsigned long) area->addr;
153 unsigned long end = addr + area->size - PAGE_SIZE;
154 int err;
155
156 BUG_ON(addr >= end);
157 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 do {
159 next = pgd_addr_end(addr, end);
160 err = vmap_pud_range(pgd, addr, next, prot, pages);
161 if (err)
162 break;
163 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 flush_cache_vmap((unsigned long) area->addr, end);
165 return err;
166}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700167EXPORT_SYMBOL_GPL(map_vm_area);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700169static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags,
170 unsigned long start, unsigned long end,
171 int node, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172{
173 struct vm_struct **p, *tmp, *area;
174 unsigned long align = 1;
175 unsigned long addr;
176
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700177 BUG_ON(in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 if (flags & VM_IOREMAP) {
179 int bit = fls(size);
180
181 if (bit > IOREMAP_MAX_ORDER)
182 bit = IOREMAP_MAX_ORDER;
183 else if (bit < PAGE_SHIFT)
184 bit = PAGE_SHIFT;
185
186 align = 1ul << bit;
187 }
188 addr = ALIGN(start, align);
189 size = PAGE_ALIGN(size);
OGAWA Hirofumi31be8302006-11-16 01:19:29 -0800190 if (unlikely(!size))
191 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
Giridhar Pemmasani5211e6e2006-10-29 04:46:55 -0800193 area = kmalloc_node(sizeof(*area), gfp_mask & GFP_LEVEL_MASK, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 if (unlikely(!area))
195 return NULL;
196
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 /*
198 * We always allocate a guard page.
199 */
200 size += PAGE_SIZE;
201
202 write_lock(&vmlist_lock);
203 for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
204 if ((unsigned long)tmp->addr < addr) {
205 if((unsigned long)tmp->addr + tmp->size >= addr)
206 addr = ALIGN(tmp->size +
207 (unsigned long)tmp->addr, align);
208 continue;
209 }
210 if ((size + addr) < addr)
211 goto out;
212 if (size + addr <= (unsigned long)tmp->addr)
213 goto found;
214 addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
215 if (addr > end - size)
216 goto out;
217 }
218
219found:
220 area->next = *p;
221 *p = area;
222
223 area->flags = flags;
224 area->addr = (void *)addr;
225 area->size = size;
226 area->pages = NULL;
227 area->nr_pages = 0;
228 area->phys_addr = 0;
229 write_unlock(&vmlist_lock);
230
231 return area;
232
233out:
234 write_unlock(&vmlist_lock);
235 kfree(area);
236 if (printk_ratelimit())
237 printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
238 return NULL;
239}
240
Christoph Lameter930fc452005-10-29 18:15:41 -0700241struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
242 unsigned long start, unsigned long end)
243{
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700244 return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL);
Christoph Lameter930fc452005-10-29 18:15:41 -0700245}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700246EXPORT_SYMBOL_GPL(__get_vm_area);
Christoph Lameter930fc452005-10-29 18:15:41 -0700247
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248/**
249 * get_vm_area - reserve a contingous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 * @size: size of the area
251 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
252 *
253 * Search an area of @size in the kernel virtual mapping area,
254 * and reserved it for out purposes. Returns the area descriptor
255 * on success or %NULL on failure.
256 */
257struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
258{
259 return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
260}
261
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700262struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
263 int node, gfp_t gfp_mask)
Christoph Lameter930fc452005-10-29 18:15:41 -0700264{
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700265 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
266 gfp_mask);
Christoph Lameter930fc452005-10-29 18:15:41 -0700267}
268
Andi Kleen7856dfe2005-05-20 14:27:57 -0700269/* Caller must hold vmlist_lock */
Nick Piggin83342312006-06-23 02:03:20 -0700270static struct vm_struct *__find_vm_area(void *addr)
271{
272 struct vm_struct *tmp;
273
274 for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
275 if (tmp->addr == addr)
276 break;
277 }
278
279 return tmp;
280}
281
282/* Caller must hold vmlist_lock */
Rolf Eike Beerd24afc52006-09-27 01:50:13 -0700283static struct vm_struct *__remove_vm_area(void *addr)
Andi Kleen7856dfe2005-05-20 14:27:57 -0700284{
285 struct vm_struct **p, *tmp;
286
287 for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
288 if (tmp->addr == addr)
289 goto found;
290 }
291 return NULL;
292
293found:
294 unmap_vm_area(tmp);
295 *p = tmp->next;
296
297 /*
298 * Remove the guard page.
299 */
300 tmp->size -= PAGE_SIZE;
301 return tmp;
302}
303
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304/**
305 * remove_vm_area - find and remove a contingous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 * @addr: base address
307 *
308 * Search for the kernel VM area starting at @addr, and remove it.
309 * This function returns the found VM area, but using it is NOT safe
Andi Kleen7856dfe2005-05-20 14:27:57 -0700310 * on SMP machines, except for its size or flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 */
312struct vm_struct *remove_vm_area(void *addr)
313{
Andi Kleen7856dfe2005-05-20 14:27:57 -0700314 struct vm_struct *v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 write_lock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700316 v = __remove_vm_area(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 write_unlock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700318 return v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319}
320
Benjamin Herrenschmidtd55e2ca2007-05-16 22:11:07 -0700321static void __vunmap(void *addr, int deallocate_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322{
323 struct vm_struct *area;
324
325 if (!addr)
326 return;
327
328 if ((PAGE_SIZE-1) & (unsigned long)addr) {
329 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
330 WARN_ON(1);
331 return;
332 }
333
334 area = remove_vm_area(addr);
335 if (unlikely(!area)) {
336 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
337 addr);
338 WARN_ON(1);
339 return;
340 }
341
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700342 debug_check_no_locks_freed(addr, area->size);
343
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 if (deallocate_pages) {
345 int i;
346
347 for (i = 0; i < area->nr_pages; i++) {
Eric Sesterhenn5aae2772006-04-01 01:26:09 +0200348 BUG_ON(!area->pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 __free_page(area->pages[i]);
350 }
351
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700352 if (area->flags & VM_VPAGES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 vfree(area->pages);
354 else
355 kfree(area->pages);
356 }
357
358 kfree(area);
359 return;
360}
361
362/**
363 * vfree - release memory allocated by vmalloc()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 * @addr: memory base address
365 *
366 * Free the virtually contiguous memory area starting at @addr, as
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700367 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
368 * NULL, no operation is performed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700370 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 */
372void vfree(void *addr)
373{
374 BUG_ON(in_interrupt());
375 __vunmap(addr, 1);
376}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377EXPORT_SYMBOL(vfree);
378
379/**
380 * vunmap - release virtual mapping obtained by vmap()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 * @addr: memory base address
382 *
383 * Free the virtually contiguous memory area starting at @addr,
384 * which was created from the page array passed to vmap().
385 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700386 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 */
388void vunmap(void *addr)
389{
390 BUG_ON(in_interrupt());
391 __vunmap(addr, 0);
392}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393EXPORT_SYMBOL(vunmap);
394
395/**
396 * vmap - map an array of pages into virtually contiguous space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397 * @pages: array of page pointers
398 * @count: number of pages to map
399 * @flags: vm_area->flags
400 * @prot: page protection for the mapping
401 *
402 * Maps @count pages from @pages into contiguous kernel virtual
403 * space.
404 */
405void *vmap(struct page **pages, unsigned int count,
406 unsigned long flags, pgprot_t prot)
407{
408 struct vm_struct *area;
409
410 if (count > num_physpages)
411 return NULL;
412
413 area = get_vm_area((count << PAGE_SHIFT), flags);
414 if (!area)
415 return NULL;
416 if (map_vm_area(area, prot, &pages)) {
417 vunmap(area->addr);
418 return NULL;
419 }
420
421 return area->addr;
422}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423EXPORT_SYMBOL(vmap);
424
Christoph Lameter930fc452005-10-29 18:15:41 -0700425void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
426 pgprot_t prot, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427{
428 struct page **pages;
429 unsigned int nr_pages, array_size, i;
430
431 nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
432 array_size = (nr_pages * sizeof(struct page *));
433
434 area->nr_pages = nr_pages;
435 /* Please note that the recursion is strictly bounded. */
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700436 if (array_size > PAGE_SIZE) {
Christoph Lameter94f60302007-07-17 04:03:29 -0700437 pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
438 PAGE_KERNEL, node);
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700439 area->flags |= VM_VPAGES;
Andrew Morton286e1ea2006-10-17 00:09:57 -0700440 } else {
441 pages = kmalloc_node(array_size,
Christoph Lameter94f60302007-07-17 04:03:29 -0700442 (gfp_mask & GFP_LEVEL_MASK) | __GFP_ZERO,
Andrew Morton286e1ea2006-10-17 00:09:57 -0700443 node);
444 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 area->pages = pages;
446 if (!area->pages) {
447 remove_vm_area(area->addr);
448 kfree(area);
449 return NULL;
450 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451
452 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameter930fc452005-10-29 18:15:41 -0700453 if (node < 0)
454 area->pages[i] = alloc_page(gfp_mask);
455 else
456 area->pages[i] = alloc_pages_node(node, gfp_mask, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 if (unlikely(!area->pages[i])) {
458 /* Successfully allocated i pages, free them in __vunmap() */
459 area->nr_pages = i;
460 goto fail;
461 }
462 }
463
464 if (map_vm_area(area, prot, &pages))
465 goto fail;
466 return area->addr;
467
468fail:
469 vfree(area->addr);
470 return NULL;
471}
472
Christoph Lameter930fc452005-10-29 18:15:41 -0700473void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
474{
475 return __vmalloc_area_node(area, gfp_mask, prot, -1);
476}
477
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700479 * __vmalloc_node - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 * @size: allocation size
481 * @gfp_mask: flags for the page level allocator
482 * @prot: protection mask for the allocated pages
Randy Dunlapd44e0782005-11-07 01:01:10 -0800483 * @node: node to use for allocation or -1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 *
485 * Allocate enough pages to cover @size from the page level
486 * allocator with @gfp_mask flags. Map them into contiguous
487 * kernel virtual space, using a pagetable protection of @prot.
488 */
Adrian Bunkb2213852006-09-25 23:31:02 -0700489static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
490 int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491{
492 struct vm_struct *area;
493
494 size = PAGE_ALIGN(size);
495 if (!size || (size >> PAGE_SHIFT) > num_physpages)
496 return NULL;
497
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700498 area = get_vm_area_node(size, VM_ALLOC, node, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 if (!area)
500 return NULL;
501
Christoph Lameter930fc452005-10-29 18:15:41 -0700502 return __vmalloc_area_node(area, gfp_mask, prot, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503}
504
Christoph Lameter930fc452005-10-29 18:15:41 -0700505void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
506{
507 return __vmalloc_node(size, gfp_mask, prot, -1);
508}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509EXPORT_SYMBOL(__vmalloc);
510
511/**
512 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 * @size: allocation size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 * Allocate enough pages to cover @size from the page level
515 * allocator and map them into contiguous kernel virtual space.
516 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200517 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 * use __vmalloc() instead.
519 */
520void *vmalloc(unsigned long size)
521{
Nick Piggin83342312006-06-23 02:03:20 -0700522 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524EXPORT_SYMBOL(vmalloc);
525
Christoph Lameter930fc452005-10-29 18:15:41 -0700526/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700527 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
528 * @size: allocation size
Nick Piggin83342312006-06-23 02:03:20 -0700529 *
Rolf Eike Beeread04082006-09-27 01:50:13 -0700530 * The resulting memory area is zeroed so it can be mapped to userspace
531 * without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700532 */
533void *vmalloc_user(unsigned long size)
534{
535 struct vm_struct *area;
536 void *ret;
537
538 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800539 if (ret) {
540 write_lock(&vmlist_lock);
541 area = __find_vm_area(ret);
542 area->flags |= VM_USERMAP;
543 write_unlock(&vmlist_lock);
544 }
Nick Piggin83342312006-06-23 02:03:20 -0700545 return ret;
546}
547EXPORT_SYMBOL(vmalloc_user);
548
549/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700550 * vmalloc_node - allocate memory on a specific node
Christoph Lameter930fc452005-10-29 18:15:41 -0700551 * @size: allocation size
Randy Dunlapd44e0782005-11-07 01:01:10 -0800552 * @node: numa node
Christoph Lameter930fc452005-10-29 18:15:41 -0700553 *
554 * Allocate enough pages to cover @size from the page level
555 * allocator and map them into contiguous kernel virtual space.
556 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200557 * For tight control over page level allocator and protection flags
Christoph Lameter930fc452005-10-29 18:15:41 -0700558 * use __vmalloc() instead.
559 */
560void *vmalloc_node(unsigned long size, int node)
561{
Nick Piggin83342312006-06-23 02:03:20 -0700562 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, node);
Christoph Lameter930fc452005-10-29 18:15:41 -0700563}
564EXPORT_SYMBOL(vmalloc_node);
565
Pavel Pisa4dc3b162005-05-01 08:59:25 -0700566#ifndef PAGE_KERNEL_EXEC
567# define PAGE_KERNEL_EXEC PAGE_KERNEL
568#endif
569
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570/**
571 * vmalloc_exec - allocate virtually contiguous, executable memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 * @size: allocation size
573 *
574 * Kernel-internal function to allocate enough pages to cover @size
575 * the page level allocator and map them into contiguous and
576 * executable kernel virtual space.
577 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200578 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 * use __vmalloc() instead.
580 */
581
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582void *vmalloc_exec(unsigned long size)
583{
584 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
585}
586
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200587#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700588#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200589#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700590#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200591#else
592#define GFP_VMALLOC32 GFP_KERNEL
593#endif
594
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595/**
596 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 * @size: allocation size
598 *
599 * Allocate enough 32bit PA addressable pages to cover @size from the
600 * page level allocator and map them into contiguous kernel virtual space.
601 */
602void *vmalloc_32(unsigned long size)
603{
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200604 return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606EXPORT_SYMBOL(vmalloc_32);
607
Nick Piggin83342312006-06-23 02:03:20 -0700608/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700609 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
Nick Piggin83342312006-06-23 02:03:20 -0700610 * @size: allocation size
Rolf Eike Beeread04082006-09-27 01:50:13 -0700611 *
612 * The resulting memory area is 32bit addressable and zeroed so it can be
613 * mapped to userspace without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700614 */
615void *vmalloc_32_user(unsigned long size)
616{
617 struct vm_struct *area;
618 void *ret;
619
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200620 ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800621 if (ret) {
622 write_lock(&vmlist_lock);
623 area = __find_vm_area(ret);
624 area->flags |= VM_USERMAP;
625 write_unlock(&vmlist_lock);
626 }
Nick Piggin83342312006-06-23 02:03:20 -0700627 return ret;
628}
629EXPORT_SYMBOL(vmalloc_32_user);
630
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631long vread(char *buf, char *addr, unsigned long count)
632{
633 struct vm_struct *tmp;
634 char *vaddr, *buf_start = buf;
635 unsigned long n;
636
637 /* Don't allow overflow */
638 if ((unsigned long) addr + count < count)
639 count = -(unsigned long) addr;
640
641 read_lock(&vmlist_lock);
642 for (tmp = vmlist; tmp; tmp = tmp->next) {
643 vaddr = (char *) tmp->addr;
644 if (addr >= vaddr + tmp->size - PAGE_SIZE)
645 continue;
646 while (addr < vaddr) {
647 if (count == 0)
648 goto finished;
649 *buf = '\0';
650 buf++;
651 addr++;
652 count--;
653 }
654 n = vaddr + tmp->size - PAGE_SIZE - addr;
655 do {
656 if (count == 0)
657 goto finished;
658 *buf = *addr;
659 buf++;
660 addr++;
661 count--;
662 } while (--n > 0);
663 }
664finished:
665 read_unlock(&vmlist_lock);
666 return buf - buf_start;
667}
668
669long vwrite(char *buf, char *addr, unsigned long count)
670{
671 struct vm_struct *tmp;
672 char *vaddr, *buf_start = buf;
673 unsigned long n;
674
675 /* Don't allow overflow */
676 if ((unsigned long) addr + count < count)
677 count = -(unsigned long) addr;
678
679 read_lock(&vmlist_lock);
680 for (tmp = vmlist; tmp; tmp = tmp->next) {
681 vaddr = (char *) tmp->addr;
682 if (addr >= vaddr + tmp->size - PAGE_SIZE)
683 continue;
684 while (addr < vaddr) {
685 if (count == 0)
686 goto finished;
687 buf++;
688 addr++;
689 count--;
690 }
691 n = vaddr + tmp->size - PAGE_SIZE - addr;
692 do {
693 if (count == 0)
694 goto finished;
695 *addr = *buf;
696 buf++;
697 addr++;
698 count--;
699 } while (--n > 0);
700 }
701finished:
702 read_unlock(&vmlist_lock);
703 return buf - buf_start;
704}
Nick Piggin83342312006-06-23 02:03:20 -0700705
706/**
707 * remap_vmalloc_range - map vmalloc pages to userspace
Nick Piggin83342312006-06-23 02:03:20 -0700708 * @vma: vma to cover (map full range of vma)
709 * @addr: vmalloc memory
710 * @pgoff: number of pages into addr before first page to map
711 * @returns: 0 for success, -Exxx on failure
712 *
713 * This function checks that addr is a valid vmalloc'ed area, and
714 * that it is big enough to cover the vma. Will return failure if
715 * that criteria isn't met.
716 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800717 * Similar to remap_pfn_range() (see mm/memory.c)
Nick Piggin83342312006-06-23 02:03:20 -0700718 */
719int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
720 unsigned long pgoff)
721{
722 struct vm_struct *area;
723 unsigned long uaddr = vma->vm_start;
724 unsigned long usize = vma->vm_end - vma->vm_start;
725 int ret;
726
727 if ((PAGE_SIZE-1) & (unsigned long)addr)
728 return -EINVAL;
729
730 read_lock(&vmlist_lock);
731 area = __find_vm_area(addr);
732 if (!area)
733 goto out_einval_locked;
734
735 if (!(area->flags & VM_USERMAP))
736 goto out_einval_locked;
737
738 if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
739 goto out_einval_locked;
740 read_unlock(&vmlist_lock);
741
742 addr += pgoff << PAGE_SHIFT;
743 do {
744 struct page *page = vmalloc_to_page(addr);
745 ret = vm_insert_page(vma, uaddr, page);
746 if (ret)
747 return ret;
748
749 uaddr += PAGE_SIZE;
750 addr += PAGE_SIZE;
751 usize -= PAGE_SIZE;
752 } while (usize > 0);
753
754 /* Prevent "things" like memory migration? VM_flags need a cleanup... */
755 vma->vm_flags |= VM_RESERVED;
756
757 return ret;
758
759out_einval_locked:
760 read_unlock(&vmlist_lock);
761 return -EINVAL;
762}
763EXPORT_SYMBOL(remap_vmalloc_range);
764
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -0700765/*
766 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
767 * have one.
768 */
769void __attribute__((weak)) vmalloc_sync_all(void)
770{
771}
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700772
773
774static int f(pte_t *pte, struct page *pmd_page, unsigned long addr, void *data)
775{
776 /* apply_to_page_range() does all the hard work. */
777 return 0;
778}
779
780/**
781 * alloc_vm_area - allocate a range of kernel address space
782 * @size: size of the area
783 * @returns: NULL on failure, vm_struct on success
784 *
785 * This function reserves a range of kernel address space, and
786 * allocates pagetables to map that range. No actual mappings
787 * are created. If the kernel address space is not shared
788 * between processes, it syncs the pagetable across all
789 * processes.
790 */
791struct vm_struct *alloc_vm_area(size_t size)
792{
793 struct vm_struct *area;
794
795 area = get_vm_area(size, VM_IOREMAP);
796 if (area == NULL)
797 return NULL;
798
799 /*
800 * This ensures that page tables are constructed for this region
801 * of kernel virtual address space and mapped into init_mm.
802 */
803 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
804 area->size, f, NULL)) {
805 free_vm_area(area);
806 return NULL;
807 }
808
809 /* Make sure the pagetables are constructed in process kernel
810 mappings */
811 vmalloc_sync_all();
812
813 return area;
814}
815EXPORT_SYMBOL_GPL(alloc_vm_area);
816
817void free_vm_area(struct vm_struct *area)
818{
819 struct vm_struct *ret;
820 ret = remove_vm_area(area->addr);
821 BUG_ON(ret != area);
822 kfree(area);
823}
824EXPORT_SYMBOL_GPL(free_vm_area);