blob: 1ac191ce564142b46cf46ec7655231d76aad093e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
Christoph Lameter930fc452005-10-29 18:15:41 -07008 * Numa awareness, Christoph Lameter, SGI, June 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/highmem.h>
14#include <linux/slab.h>
15#include <linux/spinlock.h>
16#include <linux/interrupt.h>
17
18#include <linux/vmalloc.h>
19
20#include <asm/uaccess.h>
21#include <asm/tlbflush.h>
22
23
24DEFINE_RWLOCK(vmlist_lock);
25struct vm_struct *vmlist;
26
Adrian Bunkb2213852006-09-25 23:31:02 -070027static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
28 int node);
29
Linus Torvalds1da177e2005-04-16 15:20:36 -070030static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
31{
32 pte_t *pte;
33
34 pte = pte_offset_kernel(pmd, addr);
35 do {
36 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
37 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
38 } while (pte++, addr += PAGE_SIZE, addr != end);
39}
40
41static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
42 unsigned long end)
43{
44 pmd_t *pmd;
45 unsigned long next;
46
47 pmd = pmd_offset(pud, addr);
48 do {
49 next = pmd_addr_end(addr, end);
50 if (pmd_none_or_clear_bad(pmd))
51 continue;
52 vunmap_pte_range(pmd, addr, next);
53 } while (pmd++, addr = next, addr != end);
54}
55
56static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
57 unsigned long end)
58{
59 pud_t *pud;
60 unsigned long next;
61
62 pud = pud_offset(pgd, addr);
63 do {
64 next = pud_addr_end(addr, end);
65 if (pud_none_or_clear_bad(pud))
66 continue;
67 vunmap_pmd_range(pud, addr, next);
68 } while (pud++, addr = next, addr != end);
69}
70
71void unmap_vm_area(struct vm_struct *area)
72{
73 pgd_t *pgd;
74 unsigned long next;
75 unsigned long addr = (unsigned long) area->addr;
76 unsigned long end = addr + area->size;
77
78 BUG_ON(addr >= end);
79 pgd = pgd_offset_k(addr);
80 flush_cache_vunmap(addr, end);
81 do {
82 next = pgd_addr_end(addr, end);
83 if (pgd_none_or_clear_bad(pgd))
84 continue;
85 vunmap_pud_range(pgd, addr, next);
86 } while (pgd++, addr = next, addr != end);
87 flush_tlb_kernel_range((unsigned long) area->addr, end);
88}
89
90static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
91 unsigned long end, pgprot_t prot, struct page ***pages)
92{
93 pte_t *pte;
94
Hugh Dickins872fec12005-10-29 18:16:21 -070095 pte = pte_alloc_kernel(pmd, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 if (!pte)
97 return -ENOMEM;
98 do {
99 struct page *page = **pages;
100 WARN_ON(!pte_none(*pte));
101 if (!page)
102 return -ENOMEM;
103 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
104 (*pages)++;
105 } while (pte++, addr += PAGE_SIZE, addr != end);
106 return 0;
107}
108
109static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
110 unsigned long end, pgprot_t prot, struct page ***pages)
111{
112 pmd_t *pmd;
113 unsigned long next;
114
115 pmd = pmd_alloc(&init_mm, pud, addr);
116 if (!pmd)
117 return -ENOMEM;
118 do {
119 next = pmd_addr_end(addr, end);
120 if (vmap_pte_range(pmd, addr, next, prot, pages))
121 return -ENOMEM;
122 } while (pmd++, addr = next, addr != end);
123 return 0;
124}
125
126static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
127 unsigned long end, pgprot_t prot, struct page ***pages)
128{
129 pud_t *pud;
130 unsigned long next;
131
132 pud = pud_alloc(&init_mm, pgd, addr);
133 if (!pud)
134 return -ENOMEM;
135 do {
136 next = pud_addr_end(addr, end);
137 if (vmap_pmd_range(pud, addr, next, prot, pages))
138 return -ENOMEM;
139 } while (pud++, addr = next, addr != end);
140 return 0;
141}
142
143int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
144{
145 pgd_t *pgd;
146 unsigned long next;
147 unsigned long addr = (unsigned long) area->addr;
148 unsigned long end = addr + area->size - PAGE_SIZE;
149 int err;
150
151 BUG_ON(addr >= end);
152 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 do {
154 next = pgd_addr_end(addr, end);
155 err = vmap_pud_range(pgd, addr, next, prot, pages);
156 if (err)
157 break;
158 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 flush_cache_vmap((unsigned long) area->addr, end);
160 return err;
161}
162
Christoph Lameter930fc452005-10-29 18:15:41 -0700163struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags,
164 unsigned long start, unsigned long end, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165{
166 struct vm_struct **p, *tmp, *area;
167 unsigned long align = 1;
168 unsigned long addr;
169
170 if (flags & VM_IOREMAP) {
171 int bit = fls(size);
172
173 if (bit > IOREMAP_MAX_ORDER)
174 bit = IOREMAP_MAX_ORDER;
175 else if (bit < PAGE_SHIFT)
176 bit = PAGE_SHIFT;
177
178 align = 1ul << bit;
179 }
180 addr = ALIGN(start, align);
181 size = PAGE_ALIGN(size);
182
Christoph Lameter930fc452005-10-29 18:15:41 -0700183 area = kmalloc_node(sizeof(*area), GFP_KERNEL, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 if (unlikely(!area))
185 return NULL;
186
187 if (unlikely(!size)) {
188 kfree (area);
189 return NULL;
190 }
191
192 /*
193 * We always allocate a guard page.
194 */
195 size += PAGE_SIZE;
196
197 write_lock(&vmlist_lock);
198 for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
199 if ((unsigned long)tmp->addr < addr) {
200 if((unsigned long)tmp->addr + tmp->size >= addr)
201 addr = ALIGN(tmp->size +
202 (unsigned long)tmp->addr, align);
203 continue;
204 }
205 if ((size + addr) < addr)
206 goto out;
207 if (size + addr <= (unsigned long)tmp->addr)
208 goto found;
209 addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
210 if (addr > end - size)
211 goto out;
212 }
213
214found:
215 area->next = *p;
216 *p = area;
217
218 area->flags = flags;
219 area->addr = (void *)addr;
220 area->size = size;
221 area->pages = NULL;
222 area->nr_pages = 0;
223 area->phys_addr = 0;
224 write_unlock(&vmlist_lock);
225
226 return area;
227
228out:
229 write_unlock(&vmlist_lock);
230 kfree(area);
231 if (printk_ratelimit())
232 printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
233 return NULL;
234}
235
Christoph Lameter930fc452005-10-29 18:15:41 -0700236struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
237 unsigned long start, unsigned long end)
238{
239 return __get_vm_area_node(size, flags, start, end, -1);
240}
241
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242/**
243 * get_vm_area - reserve a contingous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 * @size: size of the area
245 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
246 *
247 * Search an area of @size in the kernel virtual mapping area,
248 * and reserved it for out purposes. Returns the area descriptor
249 * on success or %NULL on failure.
250 */
251struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
252{
253 return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
254}
255
Christoph Lameter930fc452005-10-29 18:15:41 -0700256struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, int node)
257{
258 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node);
259}
260
Andi Kleen7856dfe2005-05-20 14:27:57 -0700261/* Caller must hold vmlist_lock */
Nick Piggin83342312006-06-23 02:03:20 -0700262static struct vm_struct *__find_vm_area(void *addr)
263{
264 struct vm_struct *tmp;
265
266 for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
267 if (tmp->addr == addr)
268 break;
269 }
270
271 return tmp;
272}
273
274/* Caller must hold vmlist_lock */
Rolf Eike Beerd24afc52006-09-27 01:50:13 -0700275static struct vm_struct *__remove_vm_area(void *addr)
Andi Kleen7856dfe2005-05-20 14:27:57 -0700276{
277 struct vm_struct **p, *tmp;
278
279 for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
280 if (tmp->addr == addr)
281 goto found;
282 }
283 return NULL;
284
285found:
286 unmap_vm_area(tmp);
287 *p = tmp->next;
288
289 /*
290 * Remove the guard page.
291 */
292 tmp->size -= PAGE_SIZE;
293 return tmp;
294}
295
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296/**
297 * remove_vm_area - find and remove a contingous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 * @addr: base address
299 *
300 * Search for the kernel VM area starting at @addr, and remove it.
301 * This function returns the found VM area, but using it is NOT safe
Andi Kleen7856dfe2005-05-20 14:27:57 -0700302 * on SMP machines, except for its size or flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 */
304struct vm_struct *remove_vm_area(void *addr)
305{
Andi Kleen7856dfe2005-05-20 14:27:57 -0700306 struct vm_struct *v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 write_lock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700308 v = __remove_vm_area(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 write_unlock(&vmlist_lock);
Andi Kleen7856dfe2005-05-20 14:27:57 -0700310 return v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311}
312
313void __vunmap(void *addr, int deallocate_pages)
314{
315 struct vm_struct *area;
316
317 if (!addr)
318 return;
319
320 if ((PAGE_SIZE-1) & (unsigned long)addr) {
321 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
322 WARN_ON(1);
323 return;
324 }
325
326 area = remove_vm_area(addr);
327 if (unlikely(!area)) {
328 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
329 addr);
330 WARN_ON(1);
331 return;
332 }
333
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700334 debug_check_no_locks_freed(addr, area->size);
335
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 if (deallocate_pages) {
337 int i;
338
339 for (i = 0; i < area->nr_pages; i++) {
Eric Sesterhenn5aae2772006-04-01 01:26:09 +0200340 BUG_ON(!area->pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 __free_page(area->pages[i]);
342 }
343
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700344 if (area->flags & VM_VPAGES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 vfree(area->pages);
346 else
347 kfree(area->pages);
348 }
349
350 kfree(area);
351 return;
352}
353
354/**
355 * vfree - release memory allocated by vmalloc()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 * @addr: memory base address
357 *
358 * Free the virtually contiguous memory area starting at @addr, as
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700359 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
360 * NULL, no operation is performed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700362 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 */
364void vfree(void *addr)
365{
366 BUG_ON(in_interrupt());
367 __vunmap(addr, 1);
368}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369EXPORT_SYMBOL(vfree);
370
371/**
372 * vunmap - release virtual mapping obtained by vmap()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 * @addr: memory base address
374 *
375 * Free the virtually contiguous memory area starting at @addr,
376 * which was created from the page array passed to vmap().
377 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -0700378 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 */
380void vunmap(void *addr)
381{
382 BUG_ON(in_interrupt());
383 __vunmap(addr, 0);
384}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385EXPORT_SYMBOL(vunmap);
386
387/**
388 * vmap - map an array of pages into virtually contiguous space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 * @pages: array of page pointers
390 * @count: number of pages to map
391 * @flags: vm_area->flags
392 * @prot: page protection for the mapping
393 *
394 * Maps @count pages from @pages into contiguous kernel virtual
395 * space.
396 */
397void *vmap(struct page **pages, unsigned int count,
398 unsigned long flags, pgprot_t prot)
399{
400 struct vm_struct *area;
401
402 if (count > num_physpages)
403 return NULL;
404
405 area = get_vm_area((count << PAGE_SHIFT), flags);
406 if (!area)
407 return NULL;
408 if (map_vm_area(area, prot, &pages)) {
409 vunmap(area->addr);
410 return NULL;
411 }
412
413 return area->addr;
414}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415EXPORT_SYMBOL(vmap);
416
Christoph Lameter930fc452005-10-29 18:15:41 -0700417void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
418 pgprot_t prot, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419{
420 struct page **pages;
421 unsigned int nr_pages, array_size, i;
422
423 nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
424 array_size = (nr_pages * sizeof(struct page *));
425
426 area->nr_pages = nr_pages;
427 /* Please note that the recursion is strictly bounded. */
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700428 if (array_size > PAGE_SIZE) {
Christoph Lameter930fc452005-10-29 18:15:41 -0700429 pages = __vmalloc_node(array_size, gfp_mask, PAGE_KERNEL, node);
Jan Kiszka8757d5f2006-07-14 00:23:56 -0700430 area->flags |= VM_VPAGES;
431 } else
Christoph Lameter930fc452005-10-29 18:15:41 -0700432 pages = kmalloc_node(array_size, (gfp_mask & ~__GFP_HIGHMEM), node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433 area->pages = pages;
434 if (!area->pages) {
435 remove_vm_area(area->addr);
436 kfree(area);
437 return NULL;
438 }
439 memset(area->pages, 0, array_size);
440
441 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameter930fc452005-10-29 18:15:41 -0700442 if (node < 0)
443 area->pages[i] = alloc_page(gfp_mask);
444 else
445 area->pages[i] = alloc_pages_node(node, gfp_mask, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 if (unlikely(!area->pages[i])) {
447 /* Successfully allocated i pages, free them in __vunmap() */
448 area->nr_pages = i;
449 goto fail;
450 }
451 }
452
453 if (map_vm_area(area, prot, &pages))
454 goto fail;
455 return area->addr;
456
457fail:
458 vfree(area->addr);
459 return NULL;
460}
461
Christoph Lameter930fc452005-10-29 18:15:41 -0700462void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
463{
464 return __vmalloc_area_node(area, gfp_mask, prot, -1);
465}
466
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700468 * __vmalloc_node - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 * @size: allocation size
470 * @gfp_mask: flags for the page level allocator
471 * @prot: protection mask for the allocated pages
Randy Dunlapd44e0782005-11-07 01:01:10 -0800472 * @node: node to use for allocation or -1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 *
474 * Allocate enough pages to cover @size from the page level
475 * allocator with @gfp_mask flags. Map them into contiguous
476 * kernel virtual space, using a pagetable protection of @prot.
477 */
Adrian Bunkb2213852006-09-25 23:31:02 -0700478static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
479 int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480{
481 struct vm_struct *area;
482
483 size = PAGE_ALIGN(size);
484 if (!size || (size >> PAGE_SHIFT) > num_physpages)
485 return NULL;
486
Christoph Lameter930fc452005-10-29 18:15:41 -0700487 area = get_vm_area_node(size, VM_ALLOC, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 if (!area)
489 return NULL;
490
Christoph Lameter930fc452005-10-29 18:15:41 -0700491 return __vmalloc_area_node(area, gfp_mask, prot, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492}
493
Christoph Lameter930fc452005-10-29 18:15:41 -0700494void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
495{
496 return __vmalloc_node(size, gfp_mask, prot, -1);
497}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498EXPORT_SYMBOL(__vmalloc);
499
500/**
501 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 * @size: allocation size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 * Allocate enough pages to cover @size from the page level
504 * allocator and map them into contiguous kernel virtual space.
505 *
506 * For tight cotrol over page level allocator and protection flags
507 * use __vmalloc() instead.
508 */
509void *vmalloc(unsigned long size)
510{
Nick Piggin83342312006-06-23 02:03:20 -0700511 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513EXPORT_SYMBOL(vmalloc);
514
Christoph Lameter930fc452005-10-29 18:15:41 -0700515/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700516 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
517 * @size: allocation size
Nick Piggin83342312006-06-23 02:03:20 -0700518 *
Rolf Eike Beeread04082006-09-27 01:50:13 -0700519 * The resulting memory area is zeroed so it can be mapped to userspace
520 * without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700521 */
522void *vmalloc_user(unsigned long size)
523{
524 struct vm_struct *area;
525 void *ret;
526
527 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
528 write_lock(&vmlist_lock);
529 area = __find_vm_area(ret);
530 area->flags |= VM_USERMAP;
531 write_unlock(&vmlist_lock);
532
533 return ret;
534}
535EXPORT_SYMBOL(vmalloc_user);
536
537/**
Christoph Lameter930fc452005-10-29 18:15:41 -0700538 * vmalloc_node - allocate memory on a specific node
Christoph Lameter930fc452005-10-29 18:15:41 -0700539 * @size: allocation size
Randy Dunlapd44e0782005-11-07 01:01:10 -0800540 * @node: numa node
Christoph Lameter930fc452005-10-29 18:15:41 -0700541 *
542 * Allocate enough pages to cover @size from the page level
543 * allocator and map them into contiguous kernel virtual space.
544 *
545 * For tight cotrol over page level allocator and protection flags
546 * use __vmalloc() instead.
547 */
548void *vmalloc_node(unsigned long size, int node)
549{
Nick Piggin83342312006-06-23 02:03:20 -0700550 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, node);
Christoph Lameter930fc452005-10-29 18:15:41 -0700551}
552EXPORT_SYMBOL(vmalloc_node);
553
Pavel Pisa4dc3b162005-05-01 08:59:25 -0700554#ifndef PAGE_KERNEL_EXEC
555# define PAGE_KERNEL_EXEC PAGE_KERNEL
556#endif
557
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558/**
559 * vmalloc_exec - allocate virtually contiguous, executable memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 * @size: allocation size
561 *
562 * Kernel-internal function to allocate enough pages to cover @size
563 * the page level allocator and map them into contiguous and
564 * executable kernel virtual space.
565 *
566 * For tight cotrol over page level allocator and protection flags
567 * use __vmalloc() instead.
568 */
569
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570void *vmalloc_exec(unsigned long size)
571{
572 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
573}
574
575/**
576 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577 * @size: allocation size
578 *
579 * Allocate enough 32bit PA addressable pages to cover @size from the
580 * page level allocator and map them into contiguous kernel virtual space.
581 */
582void *vmalloc_32(unsigned long size)
583{
584 return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
585}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586EXPORT_SYMBOL(vmalloc_32);
587
Nick Piggin83342312006-06-23 02:03:20 -0700588/**
Rolf Eike Beeread04082006-09-27 01:50:13 -0700589 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
Nick Piggin83342312006-06-23 02:03:20 -0700590 * @size: allocation size
Rolf Eike Beeread04082006-09-27 01:50:13 -0700591 *
592 * The resulting memory area is 32bit addressable and zeroed so it can be
593 * mapped to userspace without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -0700594 */
595void *vmalloc_32_user(unsigned long size)
596{
597 struct vm_struct *area;
598 void *ret;
599
600 ret = __vmalloc(size, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL);
601 write_lock(&vmlist_lock);
602 area = __find_vm_area(ret);
603 area->flags |= VM_USERMAP;
604 write_unlock(&vmlist_lock);
605
606 return ret;
607}
608EXPORT_SYMBOL(vmalloc_32_user);
609
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610long vread(char *buf, char *addr, unsigned long count)
611{
612 struct vm_struct *tmp;
613 char *vaddr, *buf_start = buf;
614 unsigned long n;
615
616 /* Don't allow overflow */
617 if ((unsigned long) addr + count < count)
618 count = -(unsigned long) addr;
619
620 read_lock(&vmlist_lock);
621 for (tmp = vmlist; tmp; tmp = tmp->next) {
622 vaddr = (char *) tmp->addr;
623 if (addr >= vaddr + tmp->size - PAGE_SIZE)
624 continue;
625 while (addr < vaddr) {
626 if (count == 0)
627 goto finished;
628 *buf = '\0';
629 buf++;
630 addr++;
631 count--;
632 }
633 n = vaddr + tmp->size - PAGE_SIZE - addr;
634 do {
635 if (count == 0)
636 goto finished;
637 *buf = *addr;
638 buf++;
639 addr++;
640 count--;
641 } while (--n > 0);
642 }
643finished:
644 read_unlock(&vmlist_lock);
645 return buf - buf_start;
646}
647
648long vwrite(char *buf, char *addr, unsigned long count)
649{
650 struct vm_struct *tmp;
651 char *vaddr, *buf_start = buf;
652 unsigned long n;
653
654 /* Don't allow overflow */
655 if ((unsigned long) addr + count < count)
656 count = -(unsigned long) addr;
657
658 read_lock(&vmlist_lock);
659 for (tmp = vmlist; tmp; tmp = tmp->next) {
660 vaddr = (char *) tmp->addr;
661 if (addr >= vaddr + tmp->size - PAGE_SIZE)
662 continue;
663 while (addr < vaddr) {
664 if (count == 0)
665 goto finished;
666 buf++;
667 addr++;
668 count--;
669 }
670 n = vaddr + tmp->size - PAGE_SIZE - addr;
671 do {
672 if (count == 0)
673 goto finished;
674 *addr = *buf;
675 buf++;
676 addr++;
677 count--;
678 } while (--n > 0);
679 }
680finished:
681 read_unlock(&vmlist_lock);
682 return buf - buf_start;
683}
Nick Piggin83342312006-06-23 02:03:20 -0700684
685/**
686 * remap_vmalloc_range - map vmalloc pages to userspace
Nick Piggin83342312006-06-23 02:03:20 -0700687 * @vma: vma to cover (map full range of vma)
688 * @addr: vmalloc memory
689 * @pgoff: number of pages into addr before first page to map
690 * @returns: 0 for success, -Exxx on failure
691 *
692 * This function checks that addr is a valid vmalloc'ed area, and
693 * that it is big enough to cover the vma. Will return failure if
694 * that criteria isn't met.
695 *
696 * Similar to remap_pfn_range (see mm/memory.c)
697 */
698int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
699 unsigned long pgoff)
700{
701 struct vm_struct *area;
702 unsigned long uaddr = vma->vm_start;
703 unsigned long usize = vma->vm_end - vma->vm_start;
704 int ret;
705
706 if ((PAGE_SIZE-1) & (unsigned long)addr)
707 return -EINVAL;
708
709 read_lock(&vmlist_lock);
710 area = __find_vm_area(addr);
711 if (!area)
712 goto out_einval_locked;
713
714 if (!(area->flags & VM_USERMAP))
715 goto out_einval_locked;
716
717 if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
718 goto out_einval_locked;
719 read_unlock(&vmlist_lock);
720
721 addr += pgoff << PAGE_SHIFT;
722 do {
723 struct page *page = vmalloc_to_page(addr);
724 ret = vm_insert_page(vma, uaddr, page);
725 if (ret)
726 return ret;
727
728 uaddr += PAGE_SIZE;
729 addr += PAGE_SIZE;
730 usize -= PAGE_SIZE;
731 } while (usize > 0);
732
733 /* Prevent "things" like memory migration? VM_flags need a cleanup... */
734 vma->vm_flags |= VM_RESERVED;
735
736 return ret;
737
738out_einval_locked:
739 read_unlock(&vmlist_lock);
740 return -EINVAL;
741}
742EXPORT_SYMBOL(remap_vmalloc_range);
743