blob: af77e171e339a357933d7eb6ba4b001efae5e390 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
Christoph Lameter930fc452005-10-29 18:15:41 -07008 * Numa awareness, Christoph Lameter, SGI, June 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/highmem.h>
14#include <linux/slab.h>
15#include <linux/spinlock.h>
16#include <linux/interrupt.h>
17
18#include <linux/vmalloc.h>
19
20#include <asm/uaccess.h>
21#include <asm/tlbflush.h>
22
23
24DEFINE_RWLOCK(vmlist_lock);
25struct vm_struct *vmlist;
26
Adrian Bunkb2213852006-09-25 23:31:02 -070027static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
28 int node);
29
Linus Torvalds1da177e2005-04-16 15:20:36 -070030static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
31{
32 pte_t *pte;
33
34 pte = pte_offset_kernel(pmd, addr);
35 do {
36 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
37 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
38 } while (pte++, addr += PAGE_SIZE, addr != end);
39}
40
41static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
42 unsigned long end)
43{
44 pmd_t *pmd;
45 unsigned long next;
46
47 pmd = pmd_offset(pud, addr);
48 do {
49 next = pmd_addr_end(addr, end);
50 if (pmd_none_or_clear_bad(pmd))
51 continue;
52 vunmap_pte_range(pmd, addr, next);
53 } while (pmd++, addr = next, addr != end);
54}
55
56static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
57 unsigned long end)
58{
59 pud_t *pud;
60 unsigned long next;
61
62 pud = pud_offset(pgd, addr);
63 do {
64 next = pud_addr_end(addr, end);
65 if (pud_none_or_clear_bad(pud))
66 continue;
67 vunmap_pmd_range(pud, addr, next);
68 } while (pud++, addr = next, addr != end);
69}
70
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100071void unmap_kernel_range(unsigned long addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072{
73 pgd_t *pgd;
74 unsigned long next;
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100075 unsigned long start = addr;
76 unsigned long end = addr + size;
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78 BUG_ON(addr >= end);
79 pgd = pgd_offset_k(addr);
80 flush_cache_vunmap(addr, end);
81 do {
82 next = pgd_addr_end(addr, end);
83 if (pgd_none_or_clear_bad(pgd))
84 continue;
85 vunmap_pud_range(pgd, addr, next);
86 } while (pgd++, addr = next, addr != end);
Benjamin Herrenschmidtc19c03f2007-06-04 15:15:35 +100087 flush_tlb_kernel_range(start, end);
88}
89
90static void unmap_vm_area(struct vm_struct *area)
91{
92 unmap_kernel_range((unsigned long)area->addr, area->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -070093}
94
95static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
96 unsigned long end, pgprot_t prot, struct page ***pages)
97{
98 pte_t *pte;
99
Hugh Dickins872fec12005-10-29 18:16:21 -0700100 pte = pte_alloc_kernel(pmd, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 if (!pte)
102 return -ENOMEM;
103 do {
104 struct page *page = **pages;
105 WARN_ON(!pte_none(*pte));
106 if (!page)
107 return -ENOMEM;
108 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
109 (*pages)++;
110 } while (pte++, addr += PAGE_SIZE, addr != end);
111 return 0;
112}
113
114static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
115 unsigned long end, pgprot_t prot, struct page ***pages)
116{
117 pmd_t *pmd;
118 unsigned long next;
119
120 pmd = pmd_alloc(&init_mm, pud, addr);
121 if (!pmd)
122 return -ENOMEM;
123 do {
124 next = pmd_addr_end(addr, end);
125 if (vmap_pte_range(pmd, addr, next, prot, pages))
126 return -ENOMEM;
127 } while (pmd++, addr = next, addr != end);
128 return 0;
129}
130
131static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
132 unsigned long end, pgprot_t prot, struct page ***pages)
133{
134 pud_t *pud;
135 unsigned long next;
136
137 pud = pud_alloc(&init_mm, pgd, addr);
138 if (!pud)
139 return -ENOMEM;
140 do {
141 next = pud_addr_end(addr, end);
142 if (vmap_pmd_range(pud, addr, next, prot, pages))
143 return -ENOMEM;
144 } while (pud++, addr = next, addr != end);
145 return 0;
146}
147
148int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
149{
150 pgd_t *pgd;
151 unsigned long next;
152 unsigned long addr = (unsigned long) area->addr;
153 unsigned long end = addr + area->size - PAGE_SIZE;
154 int err;
155
156 BUG_ON(addr >= end);
157 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 do {
159 next = pgd_addr_end(addr, end);
160 err = vmap_pud_range(pgd, addr, next, prot, pages);
161 if (err)
162 break;
163 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 flush_cache_vmap((unsigned long) area->addr, end);
165 return err;
166}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700167EXPORT_SYMBOL_GPL(map_vm_area);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700169static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags,
170 unsigned long start, unsigned long end,
171 int node, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172{
173 struct vm_struct **p, *tmp, *area;
174 unsigned long align = 1;
175 unsigned long addr;
176
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700177 BUG_ON(in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 if (flags & VM_IOREMAP) {
179 int bit = fls(size);
180
181 if (bit > IOREMAP_MAX_ORDER)
182 bit = IOREMAP_MAX_ORDER;
183 else if (bit < PAGE_SHIFT)
184 bit = PAGE_SHIFT;
185
186 align = 1ul << bit;
187 }
188 addr = ALIGN(start, align);
189 size = PAGE_ALIGN(size);
OGAWA Hirofumi31be8302006-11-16 01:19:29 -0800190 if (unlikely(!size))
191 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
Christoph Lameter6cb06222007-10-16 01:25:41 -0700193 area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
194
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 if (unlikely(!area))
196 return NULL;
197
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 /*
199 * We always allocate a guard page.
200 */
201 size += PAGE_SIZE;
202
203 write_lock(&vmlist_lock);
204 for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
205 if ((unsigned long)tmp->addr < addr) {
206 if((unsigned long)tmp->addr + tmp->size >= addr)
207 addr = ALIGN(tmp->size +
208 (unsigned long)tmp->addr, align);
209 continue;
210 }
211 if ((size + addr) < addr)
212 goto out;
213 if (size + addr <= (unsigned long)tmp->addr)
214 goto found;
215 addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
216 if (addr > end - size)
217 goto out;
218 }
219
220found:
221 area->next = *p;
222 *p = area;
223
224 area->flags = flags;
225 area->addr = (void *)addr;
226 area->size = size;
227 area->pages = NULL;
228 area->nr_pages = 0;
229 area->phys_addr = 0;
230 write_unlock(&vmlist_lock);
231
232 return area;
233
234out:
235 write_unlock(&vmlist_lock);
236 kfree(area);
237 if (printk_ratelimit())
238 printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
239 return NULL;
240}
241
Christoph Lameter930fc452005-10-29 18:15:41 -0700242struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
243 unsigned long start, unsigned long end)
244{
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700245 return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL);
Christoph Lameter930fc452005-10-29 18:15:41 -0700246}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700247EXPORT_SYMBOL_GPL(__get_vm_area);
Christoph Lameter930fc452005-10-29 18:15:41 -0700248
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249/**
Simon Arlott183ff222007-10-20 01:27:18 +0200250 * get_vm_area - reserve a contiguous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 * @size: size of the area
252 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
253 *
254 * Search an area of @size in the kernel virtual mapping area,
255 * and reserved it for out purposes. Returns the area descriptor
256 * on success or %NULL on failure.
257 */
258struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
259{
260 return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
261}
262
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700263struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
264 int node, gfp_t gfp_mask)
Christoph Lameter930fc452005-10-29 18:15:41 -0700265{
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700266 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
267 gfp_mask);
Christoph Lameter930fc452005-10-29 18:15:41 -0700268}
269
Andi Kleen7856dfe2005-05-20 14:27:57 -0700270/* Caller must hold vmlist_lock */
Nick Piggin83342312006-06-23 02:03:20 -0700271static struct vm_struct *__find_vm_area(void *addr)
272{
273 struct vm_struct *tmp;
274
275 for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
276 if (tmp->addr == addr)
277 break;
278 }
279
280 return tmp;
281}
282
283/* Caller must hold vmlist_lock */
Rolf Eike Beerd24afc52006-09-27 01:50:13 -0700284static struct vm_struct *__remove_vm_area(void *addr)
Andi Kleen7856dfe2005-05-20 14:27:57 -0700285{
286 struct vm_struct **p, *tmp;
287
288 for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
289 if (tmp->addr == addr)
290 goto found;
291 }
292 return NULL;
293
294found:
295 unmap_vm_area(tmp);
296 *p = tmp->next;
297
298 /*
299 * Remove the guard page.
300 */
301 tmp->size -= PAGE_SIZE;
302 return tmp;
303}
304
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305/**
Simon Arlott183ff222007-10-20 01:27:18 +0200306 * remove_vm_area - find and remove a continuous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 * @addr: base address
308 *
309 * Search for the kernel VM area starting at @addr, and remove it.
310 * This function returns the found VM area, but using it is NOT safe
Andi Kleen7856dfe2005-05-20 14:27:57 -0700311 * on SMP machines, except for its size or flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 */
313struct vm_struct *remove_vm_area(void *addr)
314{
Andi Kleen7856dfe2005-05-20 14:27:57 -0700315 struct vm_struct *v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 write_lock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700317 v = __remove_vm_area(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 write_unlock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700319 return v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320}
321
Benjamin Herrenschmidtd55e2ca2007-05-16 22:11:07 -0700322static void __vunmap(void *addr, int deallocate_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323{
324 struct vm_struct *area;
325
326 if (!addr)
327 return;
328
329 if ((PAGE_SIZE-1) & (unsigned long)addr) {
330 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
331 WARN_ON(1);
332 return;
333 }
334
335 area = remove_vm_area(addr);
336 if (unlikely(!area)) {
337 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
338 addr);
339 WARN_ON(1);
340 return;
341 }
342
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700343 debug_check_no_locks_freed(addr, area->size);
344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 if (deallocate_pages) {
346 int i;
347
348 for (i = 0; i < area->nr_pages; i++) {
Eric Sesterhenn5aae2772006-04-01 01:26:09 +0200349 BUG_ON(!area->pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 __free_page(area->pages[i]);
351 }
352
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700353 if (area->flags & VM_VPAGES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 vfree(area->pages);
355 else
356 kfree(area->pages);
357 }
358
359 kfree(area);
360 return;
361}
362
363/**
364 * vfree - release memory allocated by vmalloc()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 * @addr: memory base address
366 *
Simon Arlott183ff222007-10-20 01:27:18 +0200367 * Free the virtually continuous memory area starting at @addr, as
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700368 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
369 * NULL, no operation is performed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700371 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 */
373void vfree(void *addr)
374{
375 BUG_ON(in_interrupt());
376 __vunmap(addr, 1);
377}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378EXPORT_SYMBOL(vfree);
379
380/**
381 * vunmap - release virtual mapping obtained by vmap()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 * @addr: memory base address
383 *
384 * Free the virtually contiguous memory area starting at @addr,
385 * which was created from the page array passed to vmap().
386 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700387 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 */
389void vunmap(void *addr)
390{
391 BUG_ON(in_interrupt());
392 __vunmap(addr, 0);
393}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394EXPORT_SYMBOL(vunmap);
395
396/**
397 * vmap - map an array of pages into virtually contiguous space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 * @pages: array of page pointers
399 * @count: number of pages to map
400 * @flags: vm_area->flags
401 * @prot: page protection for the mapping
402 *
403 * Maps @count pages from @pages into contiguous kernel virtual
404 * space.
405 */
406void *vmap(struct page **pages, unsigned int count,
407 unsigned long flags, pgprot_t prot)
408{
409 struct vm_struct *area;
410
411 if (count > num_physpages)
412 return NULL;
413
414 area = get_vm_area((count << PAGE_SHIFT), flags);
415 if (!area)
416 return NULL;
417 if (map_vm_area(area, prot, &pages)) {
418 vunmap(area->addr);
419 return NULL;
420 }
421
422 return area->addr;
423}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424EXPORT_SYMBOL(vmap);
425
Christoph Lameter930fc452005-10-29 18:15:41 -0700426void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
427 pgprot_t prot, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428{
429 struct page **pages;
430 unsigned int nr_pages, array_size, i;
431
432 nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
433 array_size = (nr_pages * sizeof(struct page *));
434
435 area->nr_pages = nr_pages;
436 /* Please note that the recursion is strictly bounded. */
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700437 if (array_size > PAGE_SIZE) {
Christoph Lameter94f60302007-07-17 04:03:29 -0700438 pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
439 PAGE_KERNEL, node);
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700440 area->flags |= VM_VPAGES;
Andrew Morton286e1ea2006-10-17 00:09:57 -0700441 } else {
442 pages = kmalloc_node(array_size,
Christoph Lameter6cb06222007-10-16 01:25:41 -0700443 (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO,
Andrew Morton286e1ea2006-10-17 00:09:57 -0700444 node);
445 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 area->pages = pages;
447 if (!area->pages) {
448 remove_vm_area(area->addr);
449 kfree(area);
450 return NULL;
451 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
453 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameter930fc452005-10-29 18:15:41 -0700454 if (node < 0)
455 area->pages[i] = alloc_page(gfp_mask);
456 else
457 area->pages[i] = alloc_pages_node(node, gfp_mask, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 if (unlikely(!area->pages[i])) {
459 /* Successfully allocated i pages, free them in __vunmap() */
460 area->nr_pages = i;
461 goto fail;
462 }
463 }
464
465 if (map_vm_area(area, prot, &pages))
466 goto fail;
467 return area->addr;
468
469fail:
470 vfree(area->addr);
471 return NULL;
472}
473
Christoph Lameter930fc452005-10-29 18:15:41 -0700474void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
475{
476 return __vmalloc_area_node(area, gfp_mask, prot, -1);
477}
478
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700480 * __vmalloc_node - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 * @size: allocation size
482 * @gfp_mask: flags for the page level allocator
483 * @prot: protection mask for the allocated pages
Randy Dunlapd44e0782005-11-07 01:01:10 -0800484 * @node: node to use for allocation or -1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 *
486 * Allocate enough pages to cover @size from the page level
487 * allocator with @gfp_mask flags. Map them into contiguous
488 * kernel virtual space, using a pagetable protection of @prot.
489 */
Adrian Bunkb2213852006-09-25 23:31:02 -0700490static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
491 int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492{
493 struct vm_struct *area;
494
495 size = PAGE_ALIGN(size);
496 if (!size || (size >> PAGE_SHIFT) > num_physpages)
497 return NULL;
498
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -0700499 area = get_vm_area_node(size, VM_ALLOC, node, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 if (!area)
501 return NULL;
502
Christoph Lameter930fc452005-10-29 18:15:41 -0700503 return __vmalloc_area_node(area, gfp_mask, prot, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504}
505
Christoph Lameter930fc452005-10-29 18:15:41 -0700506void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
507{
508 return __vmalloc_node(size, gfp_mask, prot, -1);
509}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510EXPORT_SYMBOL(__vmalloc);
511
512/**
513 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 * @size: allocation size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 * Allocate enough pages to cover @size from the page level
516 * allocator and map them into contiguous kernel virtual space.
517 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200518 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 * use __vmalloc() instead.
520 */
521void *vmalloc(unsigned long size)
522{
Nick Piggin83342312006-06-23 02:03:20 -0700523 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525EXPORT_SYMBOL(vmalloc);
526
Christoph Lameter930fc452005-10-29 18:15:41 -0700527/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700528 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
529 * @size: allocation size
Nick Piggin83342312006-06-23 02:03:20 -0700530 *
Rolf Eike Beeread04082006-09-27 01:50:13 -0700531 * The resulting memory area is zeroed so it can be mapped to userspace
532 * without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700533 */
534void *vmalloc_user(unsigned long size)
535{
536 struct vm_struct *area;
537 void *ret;
538
539 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800540 if (ret) {
541 write_lock(&vmlist_lock);
542 area = __find_vm_area(ret);
543 area->flags |= VM_USERMAP;
544 write_unlock(&vmlist_lock);
545 }
Nick Piggin83342312006-06-23 02:03:20 -0700546 return ret;
547}
548EXPORT_SYMBOL(vmalloc_user);
549
550/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700551 * vmalloc_node - allocate memory on a specific node
Christoph Lameter930fc452005-10-29 18:15:41 -0700552 * @size: allocation size
Randy Dunlapd44e0782005-11-07 01:01:10 -0800553 * @node: numa node
Christoph Lameter930fc452005-10-29 18:15:41 -0700554 *
555 * Allocate enough pages to cover @size from the page level
556 * allocator and map them into contiguous kernel virtual space.
557 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200558 * For tight control over page level allocator and protection flags
Christoph Lameter930fc452005-10-29 18:15:41 -0700559 * use __vmalloc() instead.
560 */
561void *vmalloc_node(unsigned long size, int node)
562{
Nick Piggin83342312006-06-23 02:03:20 -0700563 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, node);
Christoph Lameter930fc452005-10-29 18:15:41 -0700564}
565EXPORT_SYMBOL(vmalloc_node);
566
Pavel Pisa4dc3b162005-05-01 08:59:25 -0700567#ifndef PAGE_KERNEL_EXEC
568# define PAGE_KERNEL_EXEC PAGE_KERNEL
569#endif
570
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571/**
572 * vmalloc_exec - allocate virtually contiguous, executable memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 * @size: allocation size
574 *
575 * Kernel-internal function to allocate enough pages to cover @size
576 * the page level allocator and map them into contiguous and
577 * executable kernel virtual space.
578 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200579 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 * use __vmalloc() instead.
581 */
582
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583void *vmalloc_exec(unsigned long size)
584{
585 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
586}
587
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200588#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700589#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200590#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -0700591#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200592#else
593#define GFP_VMALLOC32 GFP_KERNEL
594#endif
595
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596/**
597 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 * @size: allocation size
599 *
600 * Allocate enough 32bit PA addressable pages to cover @size from the
601 * page level allocator and map them into contiguous kernel virtual space.
602 */
603void *vmalloc_32(unsigned long size)
604{
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200605 return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607EXPORT_SYMBOL(vmalloc_32);
608
Nick Piggin83342312006-06-23 02:03:20 -0700609/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700610 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
Nick Piggin83342312006-06-23 02:03:20 -0700611 * @size: allocation size
Rolf Eike Beeread04082006-09-27 01:50:13 -0700612 *
613 * The resulting memory area is 32bit addressable and zeroed so it can be
614 * mapped to userspace without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700615 */
616void *vmalloc_32_user(unsigned long size)
617{
618 struct vm_struct *area;
619 void *ret;
620
Andi Kleen0d08e0d2007-05-02 19:27:12 +0200621 ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL);
Eric Dumazet2b4ac442006-11-10 12:27:48 -0800622 if (ret) {
623 write_lock(&vmlist_lock);
624 area = __find_vm_area(ret);
625 area->flags |= VM_USERMAP;
626 write_unlock(&vmlist_lock);
627 }
Nick Piggin83342312006-06-23 02:03:20 -0700628 return ret;
629}
630EXPORT_SYMBOL(vmalloc_32_user);
631
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632long vread(char *buf, char *addr, unsigned long count)
633{
634 struct vm_struct *tmp;
635 char *vaddr, *buf_start = buf;
636 unsigned long n;
637
638 /* Don't allow overflow */
639 if ((unsigned long) addr + count < count)
640 count = -(unsigned long) addr;
641
642 read_lock(&vmlist_lock);
643 for (tmp = vmlist; tmp; tmp = tmp->next) {
644 vaddr = (char *) tmp->addr;
645 if (addr >= vaddr + tmp->size - PAGE_SIZE)
646 continue;
647 while (addr < vaddr) {
648 if (count == 0)
649 goto finished;
650 *buf = '\0';
651 buf++;
652 addr++;
653 count--;
654 }
655 n = vaddr + tmp->size - PAGE_SIZE - addr;
656 do {
657 if (count == 0)
658 goto finished;
659 *buf = *addr;
660 buf++;
661 addr++;
662 count--;
663 } while (--n > 0);
664 }
665finished:
666 read_unlock(&vmlist_lock);
667 return buf - buf_start;
668}
669
670long vwrite(char *buf, char *addr, unsigned long count)
671{
672 struct vm_struct *tmp;
673 char *vaddr, *buf_start = buf;
674 unsigned long n;
675
676 /* Don't allow overflow */
677 if ((unsigned long) addr + count < count)
678 count = -(unsigned long) addr;
679
680 read_lock(&vmlist_lock);
681 for (tmp = vmlist; tmp; tmp = tmp->next) {
682 vaddr = (char *) tmp->addr;
683 if (addr >= vaddr + tmp->size - PAGE_SIZE)
684 continue;
685 while (addr < vaddr) {
686 if (count == 0)
687 goto finished;
688 buf++;
689 addr++;
690 count--;
691 }
692 n = vaddr + tmp->size - PAGE_SIZE - addr;
693 do {
694 if (count == 0)
695 goto finished;
696 *addr = *buf;
697 buf++;
698 addr++;
699 count--;
700 } while (--n > 0);
701 }
702finished:
703 read_unlock(&vmlist_lock);
704 return buf - buf_start;
705}
Nick Piggin83342312006-06-23 02:03:20 -0700706
707/**
708 * remap_vmalloc_range - map vmalloc pages to userspace
Nick Piggin83342312006-06-23 02:03:20 -0700709 * @vma: vma to cover (map full range of vma)
710 * @addr: vmalloc memory
711 * @pgoff: number of pages into addr before first page to map
712 * @returns: 0 for success, -Exxx on failure
713 *
714 * This function checks that addr is a valid vmalloc'ed area, and
715 * that it is big enough to cover the vma. Will return failure if
716 * that criteria isn't met.
717 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800718 * Similar to remap_pfn_range() (see mm/memory.c)
Nick Piggin83342312006-06-23 02:03:20 -0700719 */
720int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
721 unsigned long pgoff)
722{
723 struct vm_struct *area;
724 unsigned long uaddr = vma->vm_start;
725 unsigned long usize = vma->vm_end - vma->vm_start;
726 int ret;
727
728 if ((PAGE_SIZE-1) & (unsigned long)addr)
729 return -EINVAL;
730
731 read_lock(&vmlist_lock);
732 area = __find_vm_area(addr);
733 if (!area)
734 goto out_einval_locked;
735
736 if (!(area->flags & VM_USERMAP))
737 goto out_einval_locked;
738
739 if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
740 goto out_einval_locked;
741 read_unlock(&vmlist_lock);
742
743 addr += pgoff << PAGE_SHIFT;
744 do {
745 struct page *page = vmalloc_to_page(addr);
746 ret = vm_insert_page(vma, uaddr, page);
747 if (ret)
748 return ret;
749
750 uaddr += PAGE_SIZE;
751 addr += PAGE_SIZE;
752 usize -= PAGE_SIZE;
753 } while (usize > 0);
754
755 /* Prevent "things" like memory migration? VM_flags need a cleanup... */
756 vma->vm_flags |= VM_RESERVED;
757
758 return ret;
759
760out_einval_locked:
761 read_unlock(&vmlist_lock);
762 return -EINVAL;
763}
764EXPORT_SYMBOL(remap_vmalloc_range);
765
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -0700766/*
767 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
768 * have one.
769 */
770void __attribute__((weak)) vmalloc_sync_all(void)
771{
772}
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -0700773
774
775static int f(pte_t *pte, struct page *pmd_page, unsigned long addr, void *data)
776{
777 /* apply_to_page_range() does all the hard work. */
778 return 0;
779}
780
781/**
782 * alloc_vm_area - allocate a range of kernel address space
783 * @size: size of the area
784 * @returns: NULL on failure, vm_struct on success
785 *
786 * This function reserves a range of kernel address space, and
787 * allocates pagetables to map that range. No actual mappings
788 * are created. If the kernel address space is not shared
789 * between processes, it syncs the pagetable across all
790 * processes.
791 */
792struct vm_struct *alloc_vm_area(size_t size)
793{
794 struct vm_struct *area;
795
796 area = get_vm_area(size, VM_IOREMAP);
797 if (area == NULL)
798 return NULL;
799
800 /*
801 * This ensures that page tables are constructed for this region
802 * of kernel virtual address space and mapped into init_mm.
803 */
804 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
805 area->size, f, NULL)) {
806 free_vm_area(area);
807 return NULL;
808 }
809
810 /* Make sure the pagetables are constructed in process kernel
811 mappings */
812 vmalloc_sync_all();
813
814 return area;
815}
816EXPORT_SYMBOL_GPL(alloc_vm_area);
817
818void free_vm_area(struct vm_struct *area)
819{
820 struct vm_struct *ret;
821 ret = remove_vm_area(area->addr);
822 BUG_ON(ret != area);
823 kfree(area);
824}
825EXPORT_SYMBOL_GPL(free_vm_area);