blob: a3d66b3dc5cb0c9136b13764958d6aedf5b10526 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
Christoph Lameter930fc452005-10-29 18:15:41 -07008 * Numa awareness, Christoph Lameter, SGI, June 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
Nick Piggindb64fe02008-10-18 20:27:03 -070011#include <linux/vmalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/highmem.h>
Alexey Dobriyand43c36d2009-10-07 17:09:06 +040015#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/slab.h>
17#include <linux/spinlock.h>
18#include <linux/interrupt.h>
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +040019#include <linux/proc_fs.h>
Christoph Lametera10aa572008-04-28 02:12:40 -070020#include <linux/seq_file.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070021#include <linux/debugobjects.h>
Christoph Lameter23016962008-04-28 02:12:42 -070022#include <linux/kallsyms.h>
Nick Piggindb64fe02008-10-18 20:27:03 -070023#include <linux/list.h>
24#include <linux/rbtree.h>
25#include <linux/radix-tree.h>
26#include <linux/rcupdate.h>
Tejun Heof0aa6612009-02-20 16:29:08 +090027#include <linux/pfn.h>
Catalin Marinas89219d32009-06-11 13:23:19 +010028#include <linux/kmemleak.h>
Nick Piggindb64fe02008-10-18 20:27:03 -070029#include <asm/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <asm/uaccess.h>
31#include <asm/tlbflush.h>
David Miller2dca6992009-09-21 12:22:34 -070032#include <asm/shmparam.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
Jeremy Fitzhardingea0d40c82010-03-26 15:28:51 -070034bool vmap_lazy_unmap __read_mostly = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
Nick Piggindb64fe02008-10-18 20:27:03 -070036/*** Page table manipulation functions ***/
Adrian Bunkb2213852006-09-25 23:31:02 -070037
Linus Torvalds1da177e2005-04-16 15:20:36 -070038static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
39{
40 pte_t *pte;
41
42 pte = pte_offset_kernel(pmd, addr);
43 do {
44 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
45 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
46 } while (pte++, addr += PAGE_SIZE, addr != end);
47}
48
Nick Piggindb64fe02008-10-18 20:27:03 -070049static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -070050{
51 pmd_t *pmd;
52 unsigned long next;
53
54 pmd = pmd_offset(pud, addr);
55 do {
56 next = pmd_addr_end(addr, end);
57 if (pmd_none_or_clear_bad(pmd))
58 continue;
59 vunmap_pte_range(pmd, addr, next);
60 } while (pmd++, addr = next, addr != end);
61}
62
Nick Piggindb64fe02008-10-18 20:27:03 -070063static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -070064{
65 pud_t *pud;
66 unsigned long next;
67
68 pud = pud_offset(pgd, addr);
69 do {
70 next = pud_addr_end(addr, end);
71 if (pud_none_or_clear_bad(pud))
72 continue;
73 vunmap_pmd_range(pud, addr, next);
74 } while (pud++, addr = next, addr != end);
75}
76
Nick Piggindb64fe02008-10-18 20:27:03 -070077static void vunmap_page_range(unsigned long addr, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -070078{
79 pgd_t *pgd;
80 unsigned long next;
Linus Torvalds1da177e2005-04-16 15:20:36 -070081
82 BUG_ON(addr >= end);
83 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 do {
85 next = pgd_addr_end(addr, end);
86 if (pgd_none_or_clear_bad(pgd))
87 continue;
88 vunmap_pud_range(pgd, addr, next);
89 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -070090}
91
92static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
Nick Piggindb64fe02008-10-18 20:27:03 -070093 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -070094{
95 pte_t *pte;
96
Nick Piggindb64fe02008-10-18 20:27:03 -070097 /*
98 * nr is a running index into the array which helps higher level
99 * callers keep track of where we're up to.
100 */
101
Hugh Dickins872fec12005-10-29 18:16:21 -0700102 pte = pte_alloc_kernel(pmd, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 if (!pte)
104 return -ENOMEM;
105 do {
Nick Piggindb64fe02008-10-18 20:27:03 -0700106 struct page *page = pages[*nr];
107
108 if (WARN_ON(!pte_none(*pte)))
109 return -EBUSY;
110 if (WARN_ON(!page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 return -ENOMEM;
112 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
Nick Piggindb64fe02008-10-18 20:27:03 -0700113 (*nr)++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 } while (pte++, addr += PAGE_SIZE, addr != end);
115 return 0;
116}
117
Nick Piggindb64fe02008-10-18 20:27:03 -0700118static int vmap_pmd_range(pud_t *pud, unsigned long addr,
119 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120{
121 pmd_t *pmd;
122 unsigned long next;
123
124 pmd = pmd_alloc(&init_mm, pud, addr);
125 if (!pmd)
126 return -ENOMEM;
127 do {
128 next = pmd_addr_end(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700129 if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 return -ENOMEM;
131 } while (pmd++, addr = next, addr != end);
132 return 0;
133}
134
Nick Piggindb64fe02008-10-18 20:27:03 -0700135static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
136 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137{
138 pud_t *pud;
139 unsigned long next;
140
141 pud = pud_alloc(&init_mm, pgd, addr);
142 if (!pud)
143 return -ENOMEM;
144 do {
145 next = pud_addr_end(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700146 if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 return -ENOMEM;
148 } while (pud++, addr = next, addr != end);
149 return 0;
150}
151
Nick Piggindb64fe02008-10-18 20:27:03 -0700152/*
153 * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
154 * will have pfns corresponding to the "pages" array.
155 *
156 * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
157 */
Tejun Heo8fc48982009-02-20 16:29:08 +0900158static int vmap_page_range_noflush(unsigned long start, unsigned long end,
159 pgprot_t prot, struct page **pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160{
161 pgd_t *pgd;
162 unsigned long next;
Adam Lackorzynski2e4e27c2009-01-04 12:00:46 -0800163 unsigned long addr = start;
Nick Piggindb64fe02008-10-18 20:27:03 -0700164 int err = 0;
165 int nr = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166
167 BUG_ON(addr >= end);
168 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 do {
170 next = pgd_addr_end(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700171 err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 if (err)
Figo.zhangbf88c8c2009-09-21 17:01:47 -0700173 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 } while (pgd++, addr = next, addr != end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700175
Nick Piggindb64fe02008-10-18 20:27:03 -0700176 return nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177}
178
Tejun Heo8fc48982009-02-20 16:29:08 +0900179static int vmap_page_range(unsigned long start, unsigned long end,
180 pgprot_t prot, struct page **pages)
181{
182 int ret;
183
184 ret = vmap_page_range_noflush(start, end, prot, pages);
185 flush_cache_vmap(start, end);
186 return ret;
187}
188
KAMEZAWA Hiroyuki81ac3ad2009-09-22 16:45:49 -0700189int is_vmalloc_or_module_addr(const void *x)
Linus Torvalds73bdf0a2008-10-15 08:35:12 -0700190{
191 /*
Russell Kingab4f2ee2008-11-06 17:11:07 +0000192 * ARM, x86-64 and sparc64 put modules in a special place,
Linus Torvalds73bdf0a2008-10-15 08:35:12 -0700193 * and fall back on vmalloc() if that fails. Others
194 * just put it in the vmalloc space.
195 */
196#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
197 unsigned long addr = (unsigned long)x;
198 if (addr >= MODULES_VADDR && addr < MODULES_END)
199 return 1;
200#endif
201 return is_vmalloc_addr(x);
202}
203
Christoph Lameter48667e72008-02-04 22:28:31 -0800204/*
Nick Piggindb64fe02008-10-18 20:27:03 -0700205 * Walk a vmap address to the struct page it maps.
Christoph Lameter48667e72008-02-04 22:28:31 -0800206 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800207struct page *vmalloc_to_page(const void *vmalloc_addr)
Christoph Lameter48667e72008-02-04 22:28:31 -0800208{
209 unsigned long addr = (unsigned long) vmalloc_addr;
210 struct page *page = NULL;
211 pgd_t *pgd = pgd_offset_k(addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800212
Ingo Molnar7aa413d2008-06-19 13:28:11 +0200213 /*
214 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
215 * architectures that do not vmalloc module space
216 */
Linus Torvalds73bdf0a2008-10-15 08:35:12 -0700217 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
Jiri Slaby59ea7462008-06-12 13:56:40 +0200218
Christoph Lameter48667e72008-02-04 22:28:31 -0800219 if (!pgd_none(*pgd)) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700220 pud_t *pud = pud_offset(pgd, addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800221 if (!pud_none(*pud)) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700222 pmd_t *pmd = pmd_offset(pud, addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800223 if (!pmd_none(*pmd)) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700224 pte_t *ptep, pte;
225
Christoph Lameter48667e72008-02-04 22:28:31 -0800226 ptep = pte_offset_map(pmd, addr);
227 pte = *ptep;
228 if (pte_present(pte))
229 page = pte_page(pte);
230 pte_unmap(ptep);
231 }
232 }
233 }
234 return page;
235}
236EXPORT_SYMBOL(vmalloc_to_page);
237
238/*
239 * Map a vmalloc()-space virtual address to the physical page frame number.
240 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800241unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
Christoph Lameter48667e72008-02-04 22:28:31 -0800242{
243 return page_to_pfn(vmalloc_to_page(vmalloc_addr));
244}
245EXPORT_SYMBOL(vmalloc_to_pfn);
246
Nick Piggindb64fe02008-10-18 20:27:03 -0700247
248/*** Global kva allocator ***/
249
250#define VM_LAZY_FREE 0x01
251#define VM_LAZY_FREEING 0x02
252#define VM_VM_AREA 0x04
253
254struct vmap_area {
255 unsigned long va_start;
256 unsigned long va_end;
257 unsigned long flags;
258 struct rb_node rb_node; /* address sorted rbtree */
259 struct list_head list; /* address sorted list */
260 struct list_head purge_list; /* "lazy purge" list */
261 void *private;
262 struct rcu_head rcu_head;
263};
264
265static DEFINE_SPINLOCK(vmap_area_lock);
266static struct rb_root vmap_area_root = RB_ROOT;
267static LIST_HEAD(vmap_area_list);
Tejun Heoca23e402009-08-14 15:00:52 +0900268static unsigned long vmap_area_pcpu_hole;
Nick Piggindb64fe02008-10-18 20:27:03 -0700269
270static struct vmap_area *__find_vmap_area(unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271{
Nick Piggindb64fe02008-10-18 20:27:03 -0700272 struct rb_node *n = vmap_area_root.rb_node;
273
274 while (n) {
275 struct vmap_area *va;
276
277 va = rb_entry(n, struct vmap_area, rb_node);
278 if (addr < va->va_start)
279 n = n->rb_left;
280 else if (addr > va->va_start)
281 n = n->rb_right;
282 else
283 return va;
284 }
285
286 return NULL;
287}
288
289static void __insert_vmap_area(struct vmap_area *va)
290{
291 struct rb_node **p = &vmap_area_root.rb_node;
292 struct rb_node *parent = NULL;
293 struct rb_node *tmp;
294
295 while (*p) {
Namhyung Kim170168d2010-10-26 14:22:02 -0700296 struct vmap_area *tmp_va;
Nick Piggindb64fe02008-10-18 20:27:03 -0700297
298 parent = *p;
Namhyung Kim170168d2010-10-26 14:22:02 -0700299 tmp_va = rb_entry(parent, struct vmap_area, rb_node);
300 if (va->va_start < tmp_va->va_end)
Nick Piggindb64fe02008-10-18 20:27:03 -0700301 p = &(*p)->rb_left;
Namhyung Kim170168d2010-10-26 14:22:02 -0700302 else if (va->va_end > tmp_va->va_start)
Nick Piggindb64fe02008-10-18 20:27:03 -0700303 p = &(*p)->rb_right;
304 else
305 BUG();
306 }
307
308 rb_link_node(&va->rb_node, parent, p);
309 rb_insert_color(&va->rb_node, &vmap_area_root);
310
311 /* address-sort this list so it is usable like the vmlist */
312 tmp = rb_prev(&va->rb_node);
313 if (tmp) {
314 struct vmap_area *prev;
315 prev = rb_entry(tmp, struct vmap_area, rb_node);
316 list_add_rcu(&va->list, &prev->list);
317 } else
318 list_add_rcu(&va->list, &vmap_area_list);
319}
320
321static void purge_vmap_area_lazy(void);
322
323/*
324 * Allocate a region of KVA of the specified size and alignment, within the
325 * vstart and vend.
326 */
327static struct vmap_area *alloc_vmap_area(unsigned long size,
328 unsigned long align,
329 unsigned long vstart, unsigned long vend,
330 int node, gfp_t gfp_mask)
331{
332 struct vmap_area *va;
333 struct rb_node *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334 unsigned long addr;
Nick Piggindb64fe02008-10-18 20:27:03 -0700335 int purged = 0;
336
Nick Piggin77669702009-02-27 14:03:03 -0800337 BUG_ON(!size);
Nick Piggindb64fe02008-10-18 20:27:03 -0700338 BUG_ON(size & ~PAGE_MASK);
339
Nick Piggindb64fe02008-10-18 20:27:03 -0700340 va = kmalloc_node(sizeof(struct vmap_area),
341 gfp_mask & GFP_RECLAIM_MASK, node);
342 if (unlikely(!va))
343 return ERR_PTR(-ENOMEM);
344
345retry:
Glauber Costa0ae15132008-11-19 15:36:33 -0800346 addr = ALIGN(vstart, align);
347
Nick Piggindb64fe02008-10-18 20:27:03 -0700348 spin_lock(&vmap_area_lock);
Nick Piggin77669702009-02-27 14:03:03 -0800349 if (addr + size - 1 < addr)
350 goto overflow;
351
Nick Piggindb64fe02008-10-18 20:27:03 -0700352 /* XXX: could have a last_hole cache */
353 n = vmap_area_root.rb_node;
354 if (n) {
355 struct vmap_area *first = NULL;
356
357 do {
358 struct vmap_area *tmp;
359 tmp = rb_entry(n, struct vmap_area, rb_node);
360 if (tmp->va_end >= addr) {
361 if (!first && tmp->va_start < addr + size)
362 first = tmp;
363 n = n->rb_left;
364 } else {
365 first = tmp;
366 n = n->rb_right;
367 }
368 } while (n);
369
370 if (!first)
371 goto found;
372
373 if (first->va_end < addr) {
374 n = rb_next(&first->rb_node);
375 if (n)
376 first = rb_entry(n, struct vmap_area, rb_node);
377 else
378 goto found;
379 }
380
Nick Pigginf011c2d2008-11-19 15:36:32 -0800381 while (addr + size > first->va_start && addr + size <= vend) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700382 addr = ALIGN(first->va_end + PAGE_SIZE, align);
Nick Piggin77669702009-02-27 14:03:03 -0800383 if (addr + size - 1 < addr)
384 goto overflow;
Nick Piggindb64fe02008-10-18 20:27:03 -0700385
386 n = rb_next(&first->rb_node);
387 if (n)
388 first = rb_entry(n, struct vmap_area, rb_node);
389 else
390 goto found;
391 }
392 }
393found:
394 if (addr + size > vend) {
Nick Piggin77669702009-02-27 14:03:03 -0800395overflow:
Nick Piggindb64fe02008-10-18 20:27:03 -0700396 spin_unlock(&vmap_area_lock);
397 if (!purged) {
398 purge_vmap_area_lazy();
399 purged = 1;
400 goto retry;
401 }
402 if (printk_ratelimit())
Glauber Costac1279c42009-01-06 14:39:18 -0800403 printk(KERN_WARNING
404 "vmap allocation for size %lu failed: "
405 "use vmalloc=<size> to increase size.\n", size);
Ralph Wuerthner2498ce42009-05-06 16:02:59 -0700406 kfree(va);
Nick Piggindb64fe02008-10-18 20:27:03 -0700407 return ERR_PTR(-EBUSY);
408 }
409
410 BUG_ON(addr & (align-1));
411
412 va->va_start = addr;
413 va->va_end = addr + size;
414 va->flags = 0;
415 __insert_vmap_area(va);
416 spin_unlock(&vmap_area_lock);
417
418 return va;
419}
420
421static void rcu_free_va(struct rcu_head *head)
422{
423 struct vmap_area *va = container_of(head, struct vmap_area, rcu_head);
424
425 kfree(va);
426}
427
428static void __free_vmap_area(struct vmap_area *va)
429{
430 BUG_ON(RB_EMPTY_NODE(&va->rb_node));
431 rb_erase(&va->rb_node, &vmap_area_root);
432 RB_CLEAR_NODE(&va->rb_node);
433 list_del_rcu(&va->list);
434
Tejun Heoca23e402009-08-14 15:00:52 +0900435 /*
436 * Track the highest possible candidate for pcpu area
437 * allocation. Areas outside of vmalloc area can be returned
438 * here too, consider only end addresses which fall inside
439 * vmalloc area proper.
440 */
441 if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
442 vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
443
Nick Piggindb64fe02008-10-18 20:27:03 -0700444 call_rcu(&va->rcu_head, rcu_free_va);
445}
446
447/*
448 * Free a region of KVA allocated by alloc_vmap_area
449 */
450static void free_vmap_area(struct vmap_area *va)
451{
452 spin_lock(&vmap_area_lock);
453 __free_vmap_area(va);
454 spin_unlock(&vmap_area_lock);
455}
456
457/*
458 * Clear the pagetable entries of a given vmap_area
459 */
460static void unmap_vmap_area(struct vmap_area *va)
461{
462 vunmap_page_range(va->va_start, va->va_end);
463}
464
Nick Piggincd528582009-01-06 14:39:20 -0800465static void vmap_debug_free_range(unsigned long start, unsigned long end)
466{
467 /*
468 * Unmap page tables and force a TLB flush immediately if
469 * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
470 * bugs similarly to those in linear kernel virtual address
471 * space after a page has been freed.
472 *
473 * All the lazy freeing logic is still retained, in order to
474 * minimise intrusiveness of this debugging feature.
475 *
476 * This is going to be *slow* (linear kernel virtual address
477 * debugging doesn't do a broadcast TLB flush so it is a lot
478 * faster).
479 */
480#ifdef CONFIG_DEBUG_PAGEALLOC
481 vunmap_page_range(start, end);
482 flush_tlb_kernel_range(start, end);
483#endif
484}
485
Nick Piggindb64fe02008-10-18 20:27:03 -0700486/*
487 * lazy_max_pages is the maximum amount of virtual address space we gather up
488 * before attempting to purge with a TLB flush.
489 *
490 * There is a tradeoff here: a larger number will cover more kernel page tables
491 * and take slightly longer to purge, but it will linearly reduce the number of
492 * global TLB flushes that must be performed. It would seem natural to scale
493 * this number up linearly with the number of CPUs (because vmapping activity
494 * could also scale linearly with the number of CPUs), however it is likely
495 * that in practice, workloads might be constrained in other ways that mean
496 * vmap activity will not scale linearly with CPUs. Also, I want to be
497 * conservative and not introduce a big latency on huge systems, so go with
498 * a less aggressive log scale. It will still be an improvement over the old
499 * code, and it will be simple to change the scale factor if we find that it
500 * becomes a problem on bigger systems.
501 */
502static unsigned long lazy_max_pages(void)
503{
504 unsigned int log;
505
Jeremy Fitzhardingea0d40c82010-03-26 15:28:51 -0700506 if (!vmap_lazy_unmap)
507 return 0;
508
Nick Piggindb64fe02008-10-18 20:27:03 -0700509 log = fls(num_online_cpus());
510
511 return log * (32UL * 1024 * 1024 / PAGE_SIZE);
512}
513
514static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
515
Nick Piggin02b709d2010-02-01 22:25:57 +1100516/* for per-CPU blocks */
517static void purge_fragmented_blocks_allcpus(void);
518
Nick Piggindb64fe02008-10-18 20:27:03 -0700519/*
Cliff Wickman3ee48b62010-09-16 11:44:02 -0500520 * called before a call to iounmap() if the caller wants vm_area_struct's
521 * immediately freed.
522 */
523void set_iounmap_nonlazy(void)
524{
525 atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
526}
527
528/*
Nick Piggindb64fe02008-10-18 20:27:03 -0700529 * Purges all lazily-freed vmap areas.
530 *
531 * If sync is 0 then don't purge if there is already a purge in progress.
532 * If force_flush is 1, then flush kernel TLBs between *start and *end even
533 * if we found no lazy vmap areas to unmap (callers can use this to optimise
534 * their own TLB flushing).
535 * Returns with *start = min(*start, lowest purged address)
536 * *end = max(*end, highest purged address)
537 */
538static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
539 int sync, int force_flush)
540{
Andrew Morton46666d82009-01-15 13:51:15 -0800541 static DEFINE_SPINLOCK(purge_lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700542 LIST_HEAD(valist);
543 struct vmap_area *va;
Vegard Nossumcbb76672009-02-27 14:03:04 -0800544 struct vmap_area *n_va;
Nick Piggindb64fe02008-10-18 20:27:03 -0700545 int nr = 0;
546
547 /*
548 * If sync is 0 but force_flush is 1, we'll go sync anyway but callers
549 * should not expect such behaviour. This just simplifies locking for
550 * the case that isn't actually used at the moment anyway.
551 */
552 if (!sync && !force_flush) {
Andrew Morton46666d82009-01-15 13:51:15 -0800553 if (!spin_trylock(&purge_lock))
Nick Piggindb64fe02008-10-18 20:27:03 -0700554 return;
555 } else
Andrew Morton46666d82009-01-15 13:51:15 -0800556 spin_lock(&purge_lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700557
Nick Piggin02b709d2010-02-01 22:25:57 +1100558 if (sync)
559 purge_fragmented_blocks_allcpus();
560
Nick Piggindb64fe02008-10-18 20:27:03 -0700561 rcu_read_lock();
562 list_for_each_entry_rcu(va, &vmap_area_list, list) {
563 if (va->flags & VM_LAZY_FREE) {
564 if (va->va_start < *start)
565 *start = va->va_start;
566 if (va->va_end > *end)
567 *end = va->va_end;
568 nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
569 unmap_vmap_area(va);
570 list_add_tail(&va->purge_list, &valist);
571 va->flags |= VM_LAZY_FREEING;
572 va->flags &= ~VM_LAZY_FREE;
573 }
574 }
575 rcu_read_unlock();
576
Yongseok Koh88f50042010-01-19 17:33:49 +0900577 if (nr)
Nick Piggindb64fe02008-10-18 20:27:03 -0700578 atomic_sub(nr, &vmap_lazy_nr);
Nick Piggindb64fe02008-10-18 20:27:03 -0700579
580 if (nr || force_flush)
581 flush_tlb_kernel_range(*start, *end);
582
583 if (nr) {
584 spin_lock(&vmap_area_lock);
Vegard Nossumcbb76672009-02-27 14:03:04 -0800585 list_for_each_entry_safe(va, n_va, &valist, purge_list)
Nick Piggindb64fe02008-10-18 20:27:03 -0700586 __free_vmap_area(va);
587 spin_unlock(&vmap_area_lock);
588 }
Andrew Morton46666d82009-01-15 13:51:15 -0800589 spin_unlock(&purge_lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700590}
591
592/*
Nick Piggin496850e2008-11-19 15:36:33 -0800593 * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
594 * is already purging.
595 */
596static void try_purge_vmap_area_lazy(void)
597{
598 unsigned long start = ULONG_MAX, end = 0;
599
600 __purge_vmap_area_lazy(&start, &end, 0, 0);
601}
602
603/*
Nick Piggindb64fe02008-10-18 20:27:03 -0700604 * Kick off a purge of the outstanding lazy areas.
605 */
606static void purge_vmap_area_lazy(void)
607{
608 unsigned long start = ULONG_MAX, end = 0;
609
Nick Piggin496850e2008-11-19 15:36:33 -0800610 __purge_vmap_area_lazy(&start, &end, 1, 0);
Nick Piggindb64fe02008-10-18 20:27:03 -0700611}
612
613/*
Nick Pigginb29acbd2008-12-01 13:13:47 -0800614 * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
615 * called for the correct range previously.
Nick Piggindb64fe02008-10-18 20:27:03 -0700616 */
Nick Pigginb29acbd2008-12-01 13:13:47 -0800617static void free_unmap_vmap_area_noflush(struct vmap_area *va)
Nick Piggindb64fe02008-10-18 20:27:03 -0700618{
619 va->flags |= VM_LAZY_FREE;
620 atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
621 if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
Nick Piggin496850e2008-11-19 15:36:33 -0800622 try_purge_vmap_area_lazy();
Nick Piggindb64fe02008-10-18 20:27:03 -0700623}
624
Nick Pigginb29acbd2008-12-01 13:13:47 -0800625/*
626 * Free and unmap a vmap area
627 */
628static void free_unmap_vmap_area(struct vmap_area *va)
629{
630 flush_cache_vunmap(va->va_start, va->va_end);
631 free_unmap_vmap_area_noflush(va);
632}
633
Nick Piggindb64fe02008-10-18 20:27:03 -0700634static struct vmap_area *find_vmap_area(unsigned long addr)
635{
636 struct vmap_area *va;
637
638 spin_lock(&vmap_area_lock);
639 va = __find_vmap_area(addr);
640 spin_unlock(&vmap_area_lock);
641
642 return va;
643}
644
645static void free_unmap_vmap_area_addr(unsigned long addr)
646{
647 struct vmap_area *va;
648
649 va = find_vmap_area(addr);
650 BUG_ON(!va);
651 free_unmap_vmap_area(va);
652}
653
654
655/*** Per cpu kva allocator ***/
656
657/*
658 * vmap space is limited especially on 32 bit architectures. Ensure there is
659 * room for at least 16 percpu vmap blocks per CPU.
660 */
661/*
662 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
663 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
664 * instead (we just need a rough idea)
665 */
666#if BITS_PER_LONG == 32
667#define VMALLOC_SPACE (128UL*1024*1024)
668#else
669#define VMALLOC_SPACE (128UL*1024*1024*1024)
670#endif
671
672#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
673#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
674#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
675#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
676#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
677#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
678#define VMAP_BBMAP_BITS VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
679 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
680 VMALLOC_PAGES / NR_CPUS / 16))
681
682#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
683
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +1100684static bool vmap_initialized __read_mostly = false;
685
Nick Piggindb64fe02008-10-18 20:27:03 -0700686struct vmap_block_queue {
687 spinlock_t lock;
688 struct list_head free;
Nick Piggindb64fe02008-10-18 20:27:03 -0700689};
690
691struct vmap_block {
692 spinlock_t lock;
693 struct vmap_area *va;
694 struct vmap_block_queue *vbq;
695 unsigned long free, dirty;
696 DECLARE_BITMAP(alloc_map, VMAP_BBMAP_BITS);
697 DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS);
Nick Pigginde560422010-02-01 22:24:18 +1100698 struct list_head free_list;
699 struct rcu_head rcu_head;
Nick Piggin02b709d2010-02-01 22:25:57 +1100700 struct list_head purge;
Nick Piggindb64fe02008-10-18 20:27:03 -0700701};
702
703/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
704static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
705
706/*
707 * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
708 * in the free path. Could get rid of this if we change the API to return a
709 * "cookie" from alloc, to be passed to free. But no big deal yet.
710 */
711static DEFINE_SPINLOCK(vmap_block_tree_lock);
712static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
713
714/*
715 * We should probably have a fallback mechanism to allocate virtual memory
716 * out of partially filled vmap blocks. However vmap block sizing should be
717 * fairly reasonable according to the vmalloc size, so it shouldn't be a
718 * big problem.
719 */
720
721static unsigned long addr_to_vb_idx(unsigned long addr)
722{
723 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
724 addr /= VMAP_BLOCK_SIZE;
725 return addr;
726}
727
728static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
729{
730 struct vmap_block_queue *vbq;
731 struct vmap_block *vb;
732 struct vmap_area *va;
733 unsigned long vb_idx;
734 int node, err;
735
736 node = numa_node_id();
737
738 vb = kmalloc_node(sizeof(struct vmap_block),
739 gfp_mask & GFP_RECLAIM_MASK, node);
740 if (unlikely(!vb))
741 return ERR_PTR(-ENOMEM);
742
743 va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
744 VMALLOC_START, VMALLOC_END,
745 node, gfp_mask);
746 if (unlikely(IS_ERR(va))) {
747 kfree(vb);
Julia Lawalle7d86342010-08-09 17:18:28 -0700748 return ERR_CAST(va);
Nick Piggindb64fe02008-10-18 20:27:03 -0700749 }
750
751 err = radix_tree_preload(gfp_mask);
752 if (unlikely(err)) {
753 kfree(vb);
754 free_vmap_area(va);
755 return ERR_PTR(err);
756 }
757
758 spin_lock_init(&vb->lock);
759 vb->va = va;
760 vb->free = VMAP_BBMAP_BITS;
761 vb->dirty = 0;
762 bitmap_zero(vb->alloc_map, VMAP_BBMAP_BITS);
763 bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS);
764 INIT_LIST_HEAD(&vb->free_list);
Nick Piggindb64fe02008-10-18 20:27:03 -0700765
766 vb_idx = addr_to_vb_idx(va->va_start);
767 spin_lock(&vmap_block_tree_lock);
768 err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
769 spin_unlock(&vmap_block_tree_lock);
770 BUG_ON(err);
771 radix_tree_preload_end();
772
773 vbq = &get_cpu_var(vmap_block_queue);
774 vb->vbq = vbq;
775 spin_lock(&vbq->lock);
Nick Pigginde560422010-02-01 22:24:18 +1100776 list_add_rcu(&vb->free_list, &vbq->free);
Nick Piggindb64fe02008-10-18 20:27:03 -0700777 spin_unlock(&vbq->lock);
Tejun Heo3f04ba82009-10-29 22:34:12 +0900778 put_cpu_var(vmap_block_queue);
Nick Piggindb64fe02008-10-18 20:27:03 -0700779
780 return vb;
781}
782
783static void rcu_free_vb(struct rcu_head *head)
784{
785 struct vmap_block *vb = container_of(head, struct vmap_block, rcu_head);
786
787 kfree(vb);
788}
789
790static void free_vmap_block(struct vmap_block *vb)
791{
792 struct vmap_block *tmp;
793 unsigned long vb_idx;
794
Nick Piggindb64fe02008-10-18 20:27:03 -0700795 vb_idx = addr_to_vb_idx(vb->va->va_start);
796 spin_lock(&vmap_block_tree_lock);
797 tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
798 spin_unlock(&vmap_block_tree_lock);
799 BUG_ON(tmp != vb);
800
Nick Pigginb29acbd2008-12-01 13:13:47 -0800801 free_unmap_vmap_area_noflush(vb->va);
Nick Piggindb64fe02008-10-18 20:27:03 -0700802 call_rcu(&vb->rcu_head, rcu_free_vb);
803}
804
Nick Piggin02b709d2010-02-01 22:25:57 +1100805static void purge_fragmented_blocks(int cpu)
806{
807 LIST_HEAD(purge);
808 struct vmap_block *vb;
809 struct vmap_block *n_vb;
810 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
811
812 rcu_read_lock();
813 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
814
815 if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
816 continue;
817
818 spin_lock(&vb->lock);
819 if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
820 vb->free = 0; /* prevent further allocs after releasing lock */
821 vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
822 bitmap_fill(vb->alloc_map, VMAP_BBMAP_BITS);
823 bitmap_fill(vb->dirty_map, VMAP_BBMAP_BITS);
824 spin_lock(&vbq->lock);
825 list_del_rcu(&vb->free_list);
826 spin_unlock(&vbq->lock);
827 spin_unlock(&vb->lock);
828 list_add_tail(&vb->purge, &purge);
829 } else
830 spin_unlock(&vb->lock);
831 }
832 rcu_read_unlock();
833
834 list_for_each_entry_safe(vb, n_vb, &purge, purge) {
835 list_del(&vb->purge);
836 free_vmap_block(vb);
837 }
838}
839
840static void purge_fragmented_blocks_thiscpu(void)
841{
842 purge_fragmented_blocks(smp_processor_id());
843}
844
845static void purge_fragmented_blocks_allcpus(void)
846{
847 int cpu;
848
849 for_each_possible_cpu(cpu)
850 purge_fragmented_blocks(cpu);
851}
852
Nick Piggindb64fe02008-10-18 20:27:03 -0700853static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
854{
855 struct vmap_block_queue *vbq;
856 struct vmap_block *vb;
857 unsigned long addr = 0;
858 unsigned int order;
Nick Piggin02b709d2010-02-01 22:25:57 +1100859 int purge = 0;
Nick Piggindb64fe02008-10-18 20:27:03 -0700860
861 BUG_ON(size & ~PAGE_MASK);
862 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
863 order = get_order(size);
864
865again:
866 rcu_read_lock();
867 vbq = &get_cpu_var(vmap_block_queue);
868 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
869 int i;
870
871 spin_lock(&vb->lock);
Nick Piggin02b709d2010-02-01 22:25:57 +1100872 if (vb->free < 1UL << order)
873 goto next;
874
Nick Piggindb64fe02008-10-18 20:27:03 -0700875 i = bitmap_find_free_region(vb->alloc_map,
876 VMAP_BBMAP_BITS, order);
877
Nick Piggin02b709d2010-02-01 22:25:57 +1100878 if (i < 0) {
879 if (vb->free + vb->dirty == VMAP_BBMAP_BITS) {
880 /* fragmented and no outstanding allocations */
881 BUG_ON(vb->dirty != VMAP_BBMAP_BITS);
882 purge = 1;
Nick Piggindb64fe02008-10-18 20:27:03 -0700883 }
Nick Piggin02b709d2010-02-01 22:25:57 +1100884 goto next;
885 }
886 addr = vb->va->va_start + (i << PAGE_SHIFT);
887 BUG_ON(addr_to_vb_idx(addr) !=
888 addr_to_vb_idx(vb->va->va_start));
889 vb->free -= 1UL << order;
890 if (vb->free == 0) {
891 spin_lock(&vbq->lock);
892 list_del_rcu(&vb->free_list);
893 spin_unlock(&vbq->lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700894 }
895 spin_unlock(&vb->lock);
Nick Piggin02b709d2010-02-01 22:25:57 +1100896 break;
897next:
898 spin_unlock(&vb->lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700899 }
Nick Piggin02b709d2010-02-01 22:25:57 +1100900
901 if (purge)
902 purge_fragmented_blocks_thiscpu();
903
Tejun Heo3f04ba82009-10-29 22:34:12 +0900904 put_cpu_var(vmap_block_queue);
Nick Piggindb64fe02008-10-18 20:27:03 -0700905 rcu_read_unlock();
906
907 if (!addr) {
908 vb = new_vmap_block(gfp_mask);
909 if (IS_ERR(vb))
910 return vb;
911 goto again;
912 }
913
914 return (void *)addr;
915}
916
917static void vb_free(const void *addr, unsigned long size)
918{
919 unsigned long offset;
920 unsigned long vb_idx;
921 unsigned int order;
922 struct vmap_block *vb;
923
924 BUG_ON(size & ~PAGE_MASK);
925 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
Nick Pigginb29acbd2008-12-01 13:13:47 -0800926
927 flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
928
Nick Piggindb64fe02008-10-18 20:27:03 -0700929 order = get_order(size);
930
931 offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
932
933 vb_idx = addr_to_vb_idx((unsigned long)addr);
934 rcu_read_lock();
935 vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
936 rcu_read_unlock();
937 BUG_ON(!vb);
938
939 spin_lock(&vb->lock);
Nick Pigginde560422010-02-01 22:24:18 +1100940 BUG_ON(bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order));
MinChan Kimd0868172009-03-31 15:19:26 -0700941
Nick Piggindb64fe02008-10-18 20:27:03 -0700942 vb->dirty += 1UL << order;
943 if (vb->dirty == VMAP_BBMAP_BITS) {
Nick Pigginde560422010-02-01 22:24:18 +1100944 BUG_ON(vb->free);
Nick Piggindb64fe02008-10-18 20:27:03 -0700945 spin_unlock(&vb->lock);
946 free_vmap_block(vb);
947 } else
948 spin_unlock(&vb->lock);
949}
950
951/**
952 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
953 *
954 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
955 * to amortize TLB flushing overheads. What this means is that any page you
956 * have now, may, in a former life, have been mapped into kernel virtual
957 * address by the vmap layer and so there might be some CPUs with TLB entries
958 * still referencing that page (additional to the regular 1:1 kernel mapping).
959 *
960 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
961 * be sure that none of the pages we have control over will have any aliases
962 * from the vmap layer.
963 */
964void vm_unmap_aliases(void)
965{
966 unsigned long start = ULONG_MAX, end = 0;
967 int cpu;
968 int flush = 0;
969
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +1100970 if (unlikely(!vmap_initialized))
971 return;
972
Nick Piggindb64fe02008-10-18 20:27:03 -0700973 for_each_possible_cpu(cpu) {
974 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
975 struct vmap_block *vb;
976
977 rcu_read_lock();
978 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
979 int i;
980
981 spin_lock(&vb->lock);
982 i = find_first_bit(vb->dirty_map, VMAP_BBMAP_BITS);
983 while (i < VMAP_BBMAP_BITS) {
984 unsigned long s, e;
985 int j;
986 j = find_next_zero_bit(vb->dirty_map,
987 VMAP_BBMAP_BITS, i);
988
989 s = vb->va->va_start + (i << PAGE_SHIFT);
990 e = vb->va->va_start + (j << PAGE_SHIFT);
991 vunmap_page_range(s, e);
992 flush = 1;
993
994 if (s < start)
995 start = s;
996 if (e > end)
997 end = e;
998
999 i = j;
1000 i = find_next_bit(vb->dirty_map,
1001 VMAP_BBMAP_BITS, i);
1002 }
1003 spin_unlock(&vb->lock);
1004 }
1005 rcu_read_unlock();
1006 }
1007
1008 __purge_vmap_area_lazy(&start, &end, 1, flush);
1009}
1010EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1011
1012/**
1013 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
1014 * @mem: the pointer returned by vm_map_ram
1015 * @count: the count passed to that vm_map_ram call (cannot unmap partial)
1016 */
1017void vm_unmap_ram(const void *mem, unsigned int count)
1018{
1019 unsigned long size = count << PAGE_SHIFT;
1020 unsigned long addr = (unsigned long)mem;
1021
1022 BUG_ON(!addr);
1023 BUG_ON(addr < VMALLOC_START);
1024 BUG_ON(addr > VMALLOC_END);
1025 BUG_ON(addr & (PAGE_SIZE-1));
1026
1027 debug_check_no_locks_freed(mem, size);
Nick Piggincd528582009-01-06 14:39:20 -08001028 vmap_debug_free_range(addr, addr+size);
Nick Piggindb64fe02008-10-18 20:27:03 -07001029
1030 if (likely(count <= VMAP_MAX_ALLOC))
1031 vb_free(mem, size);
1032 else
1033 free_unmap_vmap_area_addr(addr);
1034}
1035EXPORT_SYMBOL(vm_unmap_ram);
1036
1037/**
1038 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
1039 * @pages: an array of pointers to the pages to be mapped
1040 * @count: number of pages
1041 * @node: prefer to allocate data structures on this node
1042 * @prot: memory protection to use. PAGE_KERNEL for regular RAM
Randy Dunlape99c97a2008-10-29 14:01:09 -07001043 *
1044 * Returns: a pointer to the address that has been mapped, or %NULL on failure
Nick Piggindb64fe02008-10-18 20:27:03 -07001045 */
1046void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
1047{
1048 unsigned long size = count << PAGE_SHIFT;
1049 unsigned long addr;
1050 void *mem;
1051
1052 if (likely(count <= VMAP_MAX_ALLOC)) {
1053 mem = vb_alloc(size, GFP_KERNEL);
1054 if (IS_ERR(mem))
1055 return NULL;
1056 addr = (unsigned long)mem;
1057 } else {
1058 struct vmap_area *va;
1059 va = alloc_vmap_area(size, PAGE_SIZE,
1060 VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
1061 if (IS_ERR(va))
1062 return NULL;
1063
1064 addr = va->va_start;
1065 mem = (void *)addr;
1066 }
1067 if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
1068 vm_unmap_ram(mem, count);
1069 return NULL;
1070 }
1071 return mem;
1072}
1073EXPORT_SYMBOL(vm_map_ram);
1074
Tejun Heof0aa6612009-02-20 16:29:08 +09001075/**
1076 * vm_area_register_early - register vmap area early during boot
1077 * @vm: vm_struct to register
Tejun Heoc0c0a292009-02-24 11:57:21 +09001078 * @align: requested alignment
Tejun Heof0aa6612009-02-20 16:29:08 +09001079 *
1080 * This function is used to register kernel vm area before
1081 * vmalloc_init() is called. @vm->size and @vm->flags should contain
1082 * proper values on entry and other fields should be zero. On return,
1083 * vm->addr contains the allocated address.
1084 *
1085 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1086 */
Tejun Heoc0c0a292009-02-24 11:57:21 +09001087void __init vm_area_register_early(struct vm_struct *vm, size_t align)
Tejun Heof0aa6612009-02-20 16:29:08 +09001088{
1089 static size_t vm_init_off __initdata;
Tejun Heoc0c0a292009-02-24 11:57:21 +09001090 unsigned long addr;
Tejun Heof0aa6612009-02-20 16:29:08 +09001091
Tejun Heoc0c0a292009-02-24 11:57:21 +09001092 addr = ALIGN(VMALLOC_START + vm_init_off, align);
1093 vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1094
1095 vm->addr = (void *)addr;
Tejun Heof0aa6612009-02-20 16:29:08 +09001096
1097 vm->next = vmlist;
1098 vmlist = vm;
1099}
1100
Nick Piggindb64fe02008-10-18 20:27:03 -07001101void __init vmalloc_init(void)
1102{
Ivan Kokshaysky822c18f2009-01-15 13:50:48 -08001103 struct vmap_area *va;
1104 struct vm_struct *tmp;
Nick Piggindb64fe02008-10-18 20:27:03 -07001105 int i;
1106
1107 for_each_possible_cpu(i) {
1108 struct vmap_block_queue *vbq;
1109
1110 vbq = &per_cpu(vmap_block_queue, i);
1111 spin_lock_init(&vbq->lock);
1112 INIT_LIST_HEAD(&vbq->free);
Nick Piggindb64fe02008-10-18 20:27:03 -07001113 }
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +11001114
Ivan Kokshaysky822c18f2009-01-15 13:50:48 -08001115 /* Import existing vmlist entries. */
1116 for (tmp = vmlist; tmp; tmp = tmp->next) {
Pekka Enberg43ebdac2009-05-25 15:01:35 +03001117 va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
Ivan Kokshaysky822c18f2009-01-15 13:50:48 -08001118 va->flags = tmp->flags | VM_VM_AREA;
1119 va->va_start = (unsigned long)tmp->addr;
1120 va->va_end = va->va_start + tmp->size;
1121 __insert_vmap_area(va);
1122 }
Tejun Heoca23e402009-08-14 15:00:52 +09001123
1124 vmap_area_pcpu_hole = VMALLOC_END;
1125
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +11001126 vmap_initialized = true;
Nick Piggindb64fe02008-10-18 20:27:03 -07001127}
1128
Tejun Heo8fc48982009-02-20 16:29:08 +09001129/**
1130 * map_kernel_range_noflush - map kernel VM area with the specified pages
1131 * @addr: start of the VM area to map
1132 * @size: size of the VM area to map
1133 * @prot: page protection flags to use
1134 * @pages: pages to map
1135 *
1136 * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size
1137 * specify should have been allocated using get_vm_area() and its
1138 * friends.
1139 *
1140 * NOTE:
1141 * This function does NOT do any cache flushing. The caller is
1142 * responsible for calling flush_cache_vmap() on to-be-mapped areas
1143 * before calling this function.
1144 *
1145 * RETURNS:
1146 * The number of pages mapped on success, -errno on failure.
1147 */
1148int map_kernel_range_noflush(unsigned long addr, unsigned long size,
1149 pgprot_t prot, struct page **pages)
1150{
1151 return vmap_page_range_noflush(addr, addr + size, prot, pages);
1152}
1153
1154/**
1155 * unmap_kernel_range_noflush - unmap kernel VM area
1156 * @addr: start of the VM area to unmap
1157 * @size: size of the VM area to unmap
1158 *
1159 * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size
1160 * specify should have been allocated using get_vm_area() and its
1161 * friends.
1162 *
1163 * NOTE:
1164 * This function does NOT do any cache flushing. The caller is
1165 * responsible for calling flush_cache_vunmap() on to-be-mapped areas
1166 * before calling this function and flush_tlb_kernel_range() after.
1167 */
1168void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
1169{
1170 vunmap_page_range(addr, addr + size);
1171}
1172
1173/**
1174 * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
1175 * @addr: start of the VM area to unmap
1176 * @size: size of the VM area to unmap
1177 *
1178 * Similar to unmap_kernel_range_noflush() but flushes vcache before
1179 * the unmapping and tlb after.
1180 */
Nick Piggindb64fe02008-10-18 20:27:03 -07001181void unmap_kernel_range(unsigned long addr, unsigned long size)
1182{
1183 unsigned long end = addr + size;
Tejun Heof6fcba72009-02-20 15:38:48 -08001184
1185 flush_cache_vunmap(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -07001186 vunmap_page_range(addr, end);
1187 flush_tlb_kernel_range(addr, end);
1188}
1189
1190int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
1191{
1192 unsigned long addr = (unsigned long)area->addr;
1193 unsigned long end = addr + area->size - PAGE_SIZE;
1194 int err;
1195
1196 err = vmap_page_range(addr, end, prot, *pages);
1197 if (err > 0) {
1198 *pages += err;
1199 err = 0;
1200 }
1201
1202 return err;
1203}
1204EXPORT_SYMBOL_GPL(map_vm_area);
1205
1206/*** Old vmalloc interfaces ***/
1207DEFINE_RWLOCK(vmlist_lock);
1208struct vm_struct *vmlist;
1209
Tejun Heocf88c792009-08-14 15:00:52 +09001210static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
1211 unsigned long flags, void *caller)
1212{
1213 struct vm_struct *tmp, **p;
1214
1215 vm->flags = flags;
1216 vm->addr = (void *)va->va_start;
1217 vm->size = va->va_end - va->va_start;
1218 vm->caller = caller;
1219 va->private = vm;
1220 va->flags |= VM_VM_AREA;
1221
1222 write_lock(&vmlist_lock);
1223 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1224 if (tmp->addr >= vm->addr)
1225 break;
1226 }
1227 vm->next = *p;
1228 *p = vm;
1229 write_unlock(&vmlist_lock);
1230}
1231
Nick Piggindb64fe02008-10-18 20:27:03 -07001232static struct vm_struct *__get_vm_area_node(unsigned long size,
David Miller2dca6992009-09-21 12:22:34 -07001233 unsigned long align, unsigned long flags, unsigned long start,
1234 unsigned long end, int node, gfp_t gfp_mask, void *caller)
Nick Piggindb64fe02008-10-18 20:27:03 -07001235{
1236 static struct vmap_area *va;
1237 struct vm_struct *area;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -07001239 BUG_ON(in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 if (flags & VM_IOREMAP) {
1241 int bit = fls(size);
1242
1243 if (bit > IOREMAP_MAX_ORDER)
1244 bit = IOREMAP_MAX_ORDER;
1245 else if (bit < PAGE_SHIFT)
1246 bit = PAGE_SHIFT;
1247
1248 align = 1ul << bit;
1249 }
Nick Piggindb64fe02008-10-18 20:27:03 -07001250
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251 size = PAGE_ALIGN(size);
OGAWA Hirofumi31be8302006-11-16 01:19:29 -08001252 if (unlikely(!size))
1253 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254
Tejun Heocf88c792009-08-14 15:00:52 +09001255 area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 if (unlikely(!area))
1257 return NULL;
1258
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259 /*
1260 * We always allocate a guard page.
1261 */
1262 size += PAGE_SIZE;
1263
Nick Piggindb64fe02008-10-18 20:27:03 -07001264 va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
1265 if (IS_ERR(va)) {
1266 kfree(area);
1267 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269
Tejun Heocf88c792009-08-14 15:00:52 +09001270 insert_vmalloc_vm(area, va, flags, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271 return area;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272}
1273
Christoph Lameter930fc452005-10-29 18:15:41 -07001274struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
1275 unsigned long start, unsigned long end)
1276{
David Miller2dca6992009-09-21 12:22:34 -07001277 return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL,
Christoph Lameter23016962008-04-28 02:12:42 -07001278 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -07001279}
Rusty Russell5992b6d2007-07-19 01:49:21 -07001280EXPORT_SYMBOL_GPL(__get_vm_area);
Christoph Lameter930fc452005-10-29 18:15:41 -07001281
Benjamin Herrenschmidtc2968612009-02-18 14:48:12 -08001282struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
1283 unsigned long start, unsigned long end,
1284 void *caller)
1285{
David Miller2dca6992009-09-21 12:22:34 -07001286 return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL,
Benjamin Herrenschmidtc2968612009-02-18 14:48:12 -08001287 caller);
1288}
1289
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290/**
Simon Arlott183ff222007-10-20 01:27:18 +02001291 * get_vm_area - reserve a contiguous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 * @size: size of the area
1293 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
1294 *
1295 * Search an area of @size in the kernel virtual mapping area,
1296 * and reserved it for out purposes. Returns the area descriptor
1297 * on success or %NULL on failure.
1298 */
1299struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
1300{
David Miller2dca6992009-09-21 12:22:34 -07001301 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
Christoph Lameter23016962008-04-28 02:12:42 -07001302 -1, GFP_KERNEL, __builtin_return_address(0));
1303}
1304
1305struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
1306 void *caller)
1307{
David Miller2dca6992009-09-21 12:22:34 -07001308 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
Christoph Lameter23016962008-04-28 02:12:42 -07001309 -1, GFP_KERNEL, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310}
1311
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -07001312struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
1313 int node, gfp_t gfp_mask)
Christoph Lameter930fc452005-10-29 18:15:41 -07001314{
David Miller2dca6992009-09-21 12:22:34 -07001315 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
1316 node, gfp_mask, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -07001317}
1318
Nick Piggindb64fe02008-10-18 20:27:03 -07001319static struct vm_struct *find_vm_area(const void *addr)
Nick Piggin83342312006-06-23 02:03:20 -07001320{
Nick Piggindb64fe02008-10-18 20:27:03 -07001321 struct vmap_area *va;
Nick Piggin83342312006-06-23 02:03:20 -07001322
Nick Piggindb64fe02008-10-18 20:27:03 -07001323 va = find_vmap_area((unsigned long)addr);
1324 if (va && va->flags & VM_VM_AREA)
1325 return va->private;
Nick Piggin83342312006-06-23 02:03:20 -07001326
Andi Kleen7856dfe2005-05-20 14:27:57 -07001327 return NULL;
Andi Kleen7856dfe2005-05-20 14:27:57 -07001328}
1329
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330/**
Simon Arlott183ff222007-10-20 01:27:18 +02001331 * remove_vm_area - find and remove a continuous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -07001332 * @addr: base address
1333 *
1334 * Search for the kernel VM area starting at @addr, and remove it.
1335 * This function returns the found VM area, but using it is NOT safe
Andi Kleen7856dfe2005-05-20 14:27:57 -07001336 * on SMP machines, except for its size or flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001338struct vm_struct *remove_vm_area(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339{
Nick Piggindb64fe02008-10-18 20:27:03 -07001340 struct vmap_area *va;
1341
1342 va = find_vmap_area((unsigned long)addr);
1343 if (va && va->flags & VM_VM_AREA) {
1344 struct vm_struct *vm = va->private;
1345 struct vm_struct *tmp, **p;
KAMEZAWA Hiroyukidd32c272009-09-21 17:02:32 -07001346 /*
1347 * remove from list and disallow access to this vm_struct
1348 * before unmap. (address range confliction is maintained by
1349 * vmap.)
1350 */
Nick Piggindb64fe02008-10-18 20:27:03 -07001351 write_lock(&vmlist_lock);
1352 for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next)
1353 ;
1354 *p = tmp->next;
1355 write_unlock(&vmlist_lock);
1356
KAMEZAWA Hiroyukidd32c272009-09-21 17:02:32 -07001357 vmap_debug_free_range(va->va_start, va->va_end);
1358 free_unmap_vmap_area(va);
1359 vm->size -= PAGE_SIZE;
1360
Nick Piggindb64fe02008-10-18 20:27:03 -07001361 return vm;
1362 }
1363 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364}
1365
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001366static void __vunmap(const void *addr, int deallocate_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367{
1368 struct vm_struct *area;
1369
1370 if (!addr)
1371 return;
1372
1373 if ((PAGE_SIZE-1) & (unsigned long)addr) {
Arjan van de Ven4c8573e2008-07-25 19:45:37 -07001374 WARN(1, KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375 return;
1376 }
1377
1378 area = remove_vm_area(addr);
1379 if (unlikely(!area)) {
Arjan van de Ven4c8573e2008-07-25 19:45:37 -07001380 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381 addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 return;
1383 }
1384
Ingo Molnar9a11b49a2006-07-03 00:24:33 -07001385 debug_check_no_locks_freed(addr, area->size);
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -07001386 debug_check_no_obj_freed(addr, area->size);
Ingo Molnar9a11b49a2006-07-03 00:24:33 -07001387
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388 if (deallocate_pages) {
1389 int i;
1390
1391 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001392 struct page *page = area->pages[i];
1393
1394 BUG_ON(!page);
1395 __free_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 }
1397
Jan Kiszka8757d5f2006-07-14 00:23:56 -07001398 if (area->flags & VM_VPAGES)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399 vfree(area->pages);
1400 else
1401 kfree(area->pages);
1402 }
1403
1404 kfree(area);
1405 return;
1406}
1407
1408/**
1409 * vfree - release memory allocated by vmalloc()
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 * @addr: memory base address
1411 *
Simon Arlott183ff222007-10-20 01:27:18 +02001412 * Free the virtually continuous memory area starting at @addr, as
Pekka Enberg80e93ef2005-09-09 13:10:16 -07001413 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
1414 * NULL, no operation is performed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -07001416 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001418void vfree(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419{
1420 BUG_ON(in_interrupt());
Catalin Marinas89219d32009-06-11 13:23:19 +01001421
1422 kmemleak_free(addr);
1423
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424 __vunmap(addr, 1);
1425}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426EXPORT_SYMBOL(vfree);
1427
1428/**
1429 * vunmap - release virtual mapping obtained by vmap()
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430 * @addr: memory base address
1431 *
1432 * Free the virtually contiguous memory area starting at @addr,
1433 * which was created from the page array passed to vmap().
1434 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -07001435 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001437void vunmap(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438{
1439 BUG_ON(in_interrupt());
Peter Zijlstra34754b62009-02-25 16:04:03 +01001440 might_sleep();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441 __vunmap(addr, 0);
1442}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443EXPORT_SYMBOL(vunmap);
1444
1445/**
1446 * vmap - map an array of pages into virtually contiguous space
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447 * @pages: array of page pointers
1448 * @count: number of pages to map
1449 * @flags: vm_area->flags
1450 * @prot: page protection for the mapping
1451 *
1452 * Maps @count pages from @pages into contiguous kernel virtual
1453 * space.
1454 */
1455void *vmap(struct page **pages, unsigned int count,
1456 unsigned long flags, pgprot_t prot)
1457{
1458 struct vm_struct *area;
1459
Peter Zijlstra34754b62009-02-25 16:04:03 +01001460 might_sleep();
1461
Jan Beulich44813742009-09-21 17:03:05 -07001462 if (count > totalram_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001463 return NULL;
1464
Christoph Lameter23016962008-04-28 02:12:42 -07001465 area = get_vm_area_caller((count << PAGE_SHIFT), flags,
1466 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467 if (!area)
1468 return NULL;
Christoph Lameter23016962008-04-28 02:12:42 -07001469
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470 if (map_vm_area(area, prot, &pages)) {
1471 vunmap(area->addr);
1472 return NULL;
1473 }
1474
1475 return area->addr;
1476}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477EXPORT_SYMBOL(vmap);
1478
David Miller2dca6992009-09-21 12:22:34 -07001479static void *__vmalloc_node(unsigned long size, unsigned long align,
1480 gfp_t gfp_mask, pgprot_t prot,
Nick Piggindb64fe02008-10-18 20:27:03 -07001481 int node, void *caller);
Adrian Bunke31d9eb2008-02-04 22:29:09 -08001482static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
Christoph Lameter23016962008-04-28 02:12:42 -07001483 pgprot_t prot, int node, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484{
1485 struct page **pages;
1486 unsigned int nr_pages, array_size, i;
Jan Beulich976d6df2009-12-14 17:58:39 -08001487 gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488
1489 nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
1490 array_size = (nr_pages * sizeof(struct page *));
1491
1492 area->nr_pages = nr_pages;
1493 /* Please note that the recursion is strictly bounded. */
Jan Kiszka8757d5f2006-07-14 00:23:56 -07001494 if (array_size > PAGE_SIZE) {
Jan Beulich976d6df2009-12-14 17:58:39 -08001495 pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
Christoph Lameter23016962008-04-28 02:12:42 -07001496 PAGE_KERNEL, node, caller);
Jan Kiszka8757d5f2006-07-14 00:23:56 -07001497 area->flags |= VM_VPAGES;
Andrew Morton286e1ea2006-10-17 00:09:57 -07001498 } else {
Jan Beulich976d6df2009-12-14 17:58:39 -08001499 pages = kmalloc_node(array_size, nested_gfp, node);
Andrew Morton286e1ea2006-10-17 00:09:57 -07001500 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501 area->pages = pages;
Christoph Lameter23016962008-04-28 02:12:42 -07001502 area->caller = caller;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503 if (!area->pages) {
1504 remove_vm_area(area->addr);
1505 kfree(area);
1506 return NULL;
1507 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508
1509 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001510 struct page *page;
1511
Christoph Lameter930fc452005-10-29 18:15:41 -07001512 if (node < 0)
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001513 page = alloc_page(gfp_mask);
Christoph Lameter930fc452005-10-29 18:15:41 -07001514 else
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001515 page = alloc_pages_node(node, gfp_mask, 0);
1516
1517 if (unlikely(!page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518 /* Successfully allocated i pages, free them in __vunmap() */
1519 area->nr_pages = i;
1520 goto fail;
1521 }
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001522 area->pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523 }
1524
1525 if (map_vm_area(area, prot, &pages))
1526 goto fail;
1527 return area->addr;
1528
1529fail:
1530 vfree(area->addr);
1531 return NULL;
1532}
1533
Christoph Lameter930fc452005-10-29 18:15:41 -07001534void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
1535{
Catalin Marinas89219d32009-06-11 13:23:19 +01001536 void *addr = __vmalloc_area_node(area, gfp_mask, prot, -1,
1537 __builtin_return_address(0));
1538
1539 /*
1540 * A ref_count = 3 is needed because the vm_struct and vmap_area
1541 * structures allocated in the __get_vm_area_node() function contain
1542 * references to the virtual address of the vmalloc'ed block.
1543 */
1544 kmemleak_alloc(addr, area->size - PAGE_SIZE, 3, gfp_mask);
1545
1546 return addr;
Christoph Lameter930fc452005-10-29 18:15:41 -07001547}
1548
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549/**
Christoph Lameter930fc452005-10-29 18:15:41 -07001550 * __vmalloc_node - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 * @size: allocation size
David Miller2dca6992009-09-21 12:22:34 -07001552 * @align: desired alignment
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553 * @gfp_mask: flags for the page level allocator
1554 * @prot: protection mask for the allocated pages
Randy Dunlapd44e0782005-11-07 01:01:10 -08001555 * @node: node to use for allocation or -1
Randy Dunlapc85d1942008-05-01 04:34:48 -07001556 * @caller: caller's return address
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 *
1558 * Allocate enough pages to cover @size from the page level
1559 * allocator with @gfp_mask flags. Map them into contiguous
1560 * kernel virtual space, using a pagetable protection of @prot.
1561 */
David Miller2dca6992009-09-21 12:22:34 -07001562static void *__vmalloc_node(unsigned long size, unsigned long align,
1563 gfp_t gfp_mask, pgprot_t prot,
1564 int node, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565{
1566 struct vm_struct *area;
Catalin Marinas89219d32009-06-11 13:23:19 +01001567 void *addr;
1568 unsigned long real_size = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569
1570 size = PAGE_ALIGN(size);
Jan Beulich44813742009-09-21 17:03:05 -07001571 if (!size || (size >> PAGE_SHIFT) > totalram_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572 return NULL;
1573
David Miller2dca6992009-09-21 12:22:34 -07001574 area = __get_vm_area_node(size, align, VM_ALLOC, VMALLOC_START,
1575 VMALLOC_END, node, gfp_mask, caller);
Christoph Lameter23016962008-04-28 02:12:42 -07001576
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577 if (!area)
1578 return NULL;
1579
Catalin Marinas89219d32009-06-11 13:23:19 +01001580 addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
1581
1582 /*
1583 * A ref_count = 3 is needed because the vm_struct and vmap_area
1584 * structures allocated in the __get_vm_area_node() function contain
1585 * references to the virtual address of the vmalloc'ed block.
1586 */
1587 kmemleak_alloc(addr, real_size, 3, gfp_mask);
1588
1589 return addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590}
1591
Christoph Lameter930fc452005-10-29 18:15:41 -07001592void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
1593{
David Miller2dca6992009-09-21 12:22:34 -07001594 return __vmalloc_node(size, 1, gfp_mask, prot, -1,
Christoph Lameter23016962008-04-28 02:12:42 -07001595 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -07001596}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597EXPORT_SYMBOL(__vmalloc);
1598
Dave Younge1ca7782010-10-26 14:22:06 -07001599static inline void *__vmalloc_node_flags(unsigned long size,
1600 int node, gfp_t flags)
1601{
1602 return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
1603 node, __builtin_return_address(0));
1604}
1605
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606/**
1607 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608 * @size: allocation size
Linus Torvalds1da177e2005-04-16 15:20:36 -07001609 * Allocate enough pages to cover @size from the page level
1610 * allocator and map them into contiguous kernel virtual space.
1611 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +02001612 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613 * use __vmalloc() instead.
1614 */
1615void *vmalloc(unsigned long size)
1616{
Dave Younge1ca7782010-10-26 14:22:06 -07001617 return __vmalloc_node_flags(size, -1, GFP_KERNEL | __GFP_HIGHMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619EXPORT_SYMBOL(vmalloc);
1620
Christoph Lameter930fc452005-10-29 18:15:41 -07001621/**
Dave Younge1ca7782010-10-26 14:22:06 -07001622 * vzalloc - allocate virtually contiguous memory with zero fill
1623 * @size: allocation size
1624 * Allocate enough pages to cover @size from the page level
1625 * allocator and map them into contiguous kernel virtual space.
1626 * The memory allocated is set to zero.
1627 *
1628 * For tight control over page level allocator and protection flags
1629 * use __vmalloc() instead.
1630 */
1631void *vzalloc(unsigned long size)
1632{
1633 return __vmalloc_node_flags(size, -1,
1634 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1635}
1636EXPORT_SYMBOL(vzalloc);
1637
1638/**
Rolf Eike Beeread04082006-09-27 01:50:13 -07001639 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
1640 * @size: allocation size
Nick Piggin83342312006-06-23 02:03:20 -07001641 *
Rolf Eike Beeread04082006-09-27 01:50:13 -07001642 * The resulting memory area is zeroed so it can be mapped to userspace
1643 * without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -07001644 */
1645void *vmalloc_user(unsigned long size)
1646{
1647 struct vm_struct *area;
1648 void *ret;
1649
David Miller2dca6992009-09-21 12:22:34 -07001650 ret = __vmalloc_node(size, SHMLBA,
1651 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
Glauber Costa84877842009-01-06 14:39:19 -08001652 PAGE_KERNEL, -1, __builtin_return_address(0));
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001653 if (ret) {
Nick Piggindb64fe02008-10-18 20:27:03 -07001654 area = find_vm_area(ret);
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001655 area->flags |= VM_USERMAP;
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001656 }
Nick Piggin83342312006-06-23 02:03:20 -07001657 return ret;
1658}
1659EXPORT_SYMBOL(vmalloc_user);
1660
1661/**
Christoph Lameter930fc452005-10-29 18:15:41 -07001662 * vmalloc_node - allocate memory on a specific node
Christoph Lameter930fc452005-10-29 18:15:41 -07001663 * @size: allocation size
Randy Dunlapd44e0782005-11-07 01:01:10 -08001664 * @node: numa node
Christoph Lameter930fc452005-10-29 18:15:41 -07001665 *
1666 * Allocate enough pages to cover @size from the page level
1667 * allocator and map them into contiguous kernel virtual space.
1668 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +02001669 * For tight control over page level allocator and protection flags
Christoph Lameter930fc452005-10-29 18:15:41 -07001670 * use __vmalloc() instead.
1671 */
1672void *vmalloc_node(unsigned long size, int node)
1673{
David Miller2dca6992009-09-21 12:22:34 -07001674 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
Christoph Lameter23016962008-04-28 02:12:42 -07001675 node, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -07001676}
1677EXPORT_SYMBOL(vmalloc_node);
1678
Dave Younge1ca7782010-10-26 14:22:06 -07001679/**
1680 * vzalloc_node - allocate memory on a specific node with zero fill
1681 * @size: allocation size
1682 * @node: numa node
1683 *
1684 * Allocate enough pages to cover @size from the page level
1685 * allocator and map them into contiguous kernel virtual space.
1686 * The memory allocated is set to zero.
1687 *
1688 * For tight control over page level allocator and protection flags
1689 * use __vmalloc_node() instead.
1690 */
1691void *vzalloc_node(unsigned long size, int node)
1692{
1693 return __vmalloc_node_flags(size, node,
1694 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1695}
1696EXPORT_SYMBOL(vzalloc_node);
1697
Pavel Pisa4dc3b162005-05-01 08:59:25 -07001698#ifndef PAGE_KERNEL_EXEC
1699# define PAGE_KERNEL_EXEC PAGE_KERNEL
1700#endif
1701
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702/**
1703 * vmalloc_exec - allocate virtually contiguous, executable memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 * @size: allocation size
1705 *
1706 * Kernel-internal function to allocate enough pages to cover @size
1707 * the page level allocator and map them into contiguous and
1708 * executable kernel virtual space.
1709 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +02001710 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711 * use __vmalloc() instead.
1712 */
1713
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714void *vmalloc_exec(unsigned long size)
1715{
David Miller2dca6992009-09-21 12:22:34 -07001716 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
Glauber Costa84877842009-01-06 14:39:19 -08001717 -1, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718}
1719
Andi Kleen0d08e0d2007-05-02 19:27:12 +02001720#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -07001721#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +02001722#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -07001723#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +02001724#else
1725#define GFP_VMALLOC32 GFP_KERNEL
1726#endif
1727
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728/**
1729 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730 * @size: allocation size
1731 *
1732 * Allocate enough 32bit PA addressable pages to cover @size from the
1733 * page level allocator and map them into contiguous kernel virtual space.
1734 */
1735void *vmalloc_32(unsigned long size)
1736{
David Miller2dca6992009-09-21 12:22:34 -07001737 return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
Glauber Costa84877842009-01-06 14:39:19 -08001738 -1, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740EXPORT_SYMBOL(vmalloc_32);
1741
Nick Piggin83342312006-06-23 02:03:20 -07001742/**
Rolf Eike Beeread04082006-09-27 01:50:13 -07001743 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
Nick Piggin83342312006-06-23 02:03:20 -07001744 * @size: allocation size
Rolf Eike Beeread04082006-09-27 01:50:13 -07001745 *
1746 * The resulting memory area is 32bit addressable and zeroed so it can be
1747 * mapped to userspace without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -07001748 */
1749void *vmalloc_32_user(unsigned long size)
1750{
1751 struct vm_struct *area;
1752 void *ret;
1753
David Miller2dca6992009-09-21 12:22:34 -07001754 ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
Glauber Costa84877842009-01-06 14:39:19 -08001755 -1, __builtin_return_address(0));
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001756 if (ret) {
Nick Piggindb64fe02008-10-18 20:27:03 -07001757 area = find_vm_area(ret);
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001758 area->flags |= VM_USERMAP;
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001759 }
Nick Piggin83342312006-06-23 02:03:20 -07001760 return ret;
1761}
1762EXPORT_SYMBOL(vmalloc_32_user);
1763
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001764/*
1765 * small helper routine , copy contents to buf from addr.
1766 * If the page is not present, fill zero.
1767 */
1768
1769static int aligned_vread(char *buf, char *addr, unsigned long count)
1770{
1771 struct page *p;
1772 int copied = 0;
1773
1774 while (count) {
1775 unsigned long offset, length;
1776
1777 offset = (unsigned long)addr & ~PAGE_MASK;
1778 length = PAGE_SIZE - offset;
1779 if (length > count)
1780 length = count;
1781 p = vmalloc_to_page(addr);
1782 /*
1783 * To do safe access to this _mapped_ area, we need
1784 * lock. But adding lock here means that we need to add
1785 * overhead of vmalloc()/vfree() calles for this _debug_
1786 * interface, rarely used. Instead of that, we'll use
1787 * kmap() and get small overhead in this access function.
1788 */
1789 if (p) {
1790 /*
1791 * we can expect USER0 is not used (see vread/vwrite's
1792 * function description)
1793 */
1794 void *map = kmap_atomic(p, KM_USER0);
1795 memcpy(buf, map + offset, length);
1796 kunmap_atomic(map, KM_USER0);
1797 } else
1798 memset(buf, 0, length);
1799
1800 addr += length;
1801 buf += length;
1802 copied += length;
1803 count -= length;
1804 }
1805 return copied;
1806}
1807
1808static int aligned_vwrite(char *buf, char *addr, unsigned long count)
1809{
1810 struct page *p;
1811 int copied = 0;
1812
1813 while (count) {
1814 unsigned long offset, length;
1815
1816 offset = (unsigned long)addr & ~PAGE_MASK;
1817 length = PAGE_SIZE - offset;
1818 if (length > count)
1819 length = count;
1820 p = vmalloc_to_page(addr);
1821 /*
1822 * To do safe access to this _mapped_ area, we need
1823 * lock. But adding lock here means that we need to add
1824 * overhead of vmalloc()/vfree() calles for this _debug_
1825 * interface, rarely used. Instead of that, we'll use
1826 * kmap() and get small overhead in this access function.
1827 */
1828 if (p) {
1829 /*
1830 * we can expect USER0 is not used (see vread/vwrite's
1831 * function description)
1832 */
1833 void *map = kmap_atomic(p, KM_USER0);
1834 memcpy(map + offset, buf, length);
1835 kunmap_atomic(map, KM_USER0);
1836 }
1837 addr += length;
1838 buf += length;
1839 copied += length;
1840 count -= length;
1841 }
1842 return copied;
1843}
1844
1845/**
1846 * vread() - read vmalloc area in a safe way.
1847 * @buf: buffer for reading data
1848 * @addr: vm address.
1849 * @count: number of bytes to be read.
1850 *
1851 * Returns # of bytes which addr and buf should be increased.
1852 * (same number to @count). Returns 0 if [addr...addr+count) doesn't
1853 * includes any intersect with alive vmalloc area.
1854 *
1855 * This function checks that addr is a valid vmalloc'ed area, and
1856 * copy data from that area to a given buffer. If the given memory range
1857 * of [addr...addr+count) includes some valid address, data is copied to
1858 * proper area of @buf. If there are memory holes, they'll be zero-filled.
1859 * IOREMAP area is treated as memory hole and no copy is done.
1860 *
1861 * If [addr...addr+count) doesn't includes any intersects with alive
1862 * vm_struct area, returns 0.
1863 * @buf should be kernel's buffer. Because this function uses KM_USER0,
1864 * the caller should guarantee KM_USER0 is not used.
1865 *
1866 * Note: In usual ops, vread() is never necessary because the caller
1867 * should know vmalloc() area is valid and can use memcpy().
1868 * This is for routines which have to access vmalloc area without
1869 * any informaion, as /dev/kmem.
1870 *
1871 */
1872
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873long vread(char *buf, char *addr, unsigned long count)
1874{
1875 struct vm_struct *tmp;
1876 char *vaddr, *buf_start = buf;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001877 unsigned long buflen = count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001878 unsigned long n;
1879
1880 /* Don't allow overflow */
1881 if ((unsigned long) addr + count < count)
1882 count = -(unsigned long) addr;
1883
1884 read_lock(&vmlist_lock);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001885 for (tmp = vmlist; count && tmp; tmp = tmp->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001886 vaddr = (char *) tmp->addr;
1887 if (addr >= vaddr + tmp->size - PAGE_SIZE)
1888 continue;
1889 while (addr < vaddr) {
1890 if (count == 0)
1891 goto finished;
1892 *buf = '\0';
1893 buf++;
1894 addr++;
1895 count--;
1896 }
1897 n = vaddr + tmp->size - PAGE_SIZE - addr;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001898 if (n > count)
1899 n = count;
1900 if (!(tmp->flags & VM_IOREMAP))
1901 aligned_vread(buf, addr, n);
1902 else /* IOREMAP area is treated as memory hole */
1903 memset(buf, 0, n);
1904 buf += n;
1905 addr += n;
1906 count -= n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001907 }
1908finished:
1909 read_unlock(&vmlist_lock);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001910
1911 if (buf == buf_start)
1912 return 0;
1913 /* zero-fill memory holes */
1914 if (buf != buf_start + buflen)
1915 memset(buf, 0, buflen - (buf - buf_start));
1916
1917 return buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918}
1919
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001920/**
1921 * vwrite() - write vmalloc area in a safe way.
1922 * @buf: buffer for source data
1923 * @addr: vm address.
1924 * @count: number of bytes to be read.
1925 *
1926 * Returns # of bytes which addr and buf should be incresed.
1927 * (same number to @count).
1928 * If [addr...addr+count) doesn't includes any intersect with valid
1929 * vmalloc area, returns 0.
1930 *
1931 * This function checks that addr is a valid vmalloc'ed area, and
1932 * copy data from a buffer to the given addr. If specified range of
1933 * [addr...addr+count) includes some valid address, data is copied from
1934 * proper area of @buf. If there are memory holes, no copy to hole.
1935 * IOREMAP area is treated as memory hole and no copy is done.
1936 *
1937 * If [addr...addr+count) doesn't includes any intersects with alive
1938 * vm_struct area, returns 0.
1939 * @buf should be kernel's buffer. Because this function uses KM_USER0,
1940 * the caller should guarantee KM_USER0 is not used.
1941 *
1942 * Note: In usual ops, vwrite() is never necessary because the caller
1943 * should know vmalloc() area is valid and can use memcpy().
1944 * This is for routines which have to access vmalloc area without
1945 * any informaion, as /dev/kmem.
1946 *
1947 * The caller should guarantee KM_USER1 is not used.
1948 */
1949
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950long vwrite(char *buf, char *addr, unsigned long count)
1951{
1952 struct vm_struct *tmp;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001953 char *vaddr;
1954 unsigned long n, buflen;
1955 int copied = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956
1957 /* Don't allow overflow */
1958 if ((unsigned long) addr + count < count)
1959 count = -(unsigned long) addr;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001960 buflen = count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961
1962 read_lock(&vmlist_lock);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001963 for (tmp = vmlist; count && tmp; tmp = tmp->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964 vaddr = (char *) tmp->addr;
1965 if (addr >= vaddr + tmp->size - PAGE_SIZE)
1966 continue;
1967 while (addr < vaddr) {
1968 if (count == 0)
1969 goto finished;
1970 buf++;
1971 addr++;
1972 count--;
1973 }
1974 n = vaddr + tmp->size - PAGE_SIZE - addr;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001975 if (n > count)
1976 n = count;
1977 if (!(tmp->flags & VM_IOREMAP)) {
1978 aligned_vwrite(buf, addr, n);
1979 copied++;
1980 }
1981 buf += n;
1982 addr += n;
1983 count -= n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984 }
1985finished:
1986 read_unlock(&vmlist_lock);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001987 if (!copied)
1988 return 0;
1989 return buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001990}
Nick Piggin83342312006-06-23 02:03:20 -07001991
1992/**
1993 * remap_vmalloc_range - map vmalloc pages to userspace
Nick Piggin83342312006-06-23 02:03:20 -07001994 * @vma: vma to cover (map full range of vma)
1995 * @addr: vmalloc memory
1996 * @pgoff: number of pages into addr before first page to map
Randy Dunlap76824862008-03-19 17:00:40 -07001997 *
1998 * Returns: 0 for success, -Exxx on failure
Nick Piggin83342312006-06-23 02:03:20 -07001999 *
2000 * This function checks that addr is a valid vmalloc'ed area, and
2001 * that it is big enough to cover the vma. Will return failure if
2002 * that criteria isn't met.
2003 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -08002004 * Similar to remap_pfn_range() (see mm/memory.c)
Nick Piggin83342312006-06-23 02:03:20 -07002005 */
2006int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
2007 unsigned long pgoff)
2008{
2009 struct vm_struct *area;
2010 unsigned long uaddr = vma->vm_start;
2011 unsigned long usize = vma->vm_end - vma->vm_start;
Nick Piggin83342312006-06-23 02:03:20 -07002012
2013 if ((PAGE_SIZE-1) & (unsigned long)addr)
2014 return -EINVAL;
2015
Nick Piggindb64fe02008-10-18 20:27:03 -07002016 area = find_vm_area(addr);
Nick Piggin83342312006-06-23 02:03:20 -07002017 if (!area)
Nick Piggindb64fe02008-10-18 20:27:03 -07002018 return -EINVAL;
Nick Piggin83342312006-06-23 02:03:20 -07002019
2020 if (!(area->flags & VM_USERMAP))
Nick Piggindb64fe02008-10-18 20:27:03 -07002021 return -EINVAL;
Nick Piggin83342312006-06-23 02:03:20 -07002022
2023 if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
Nick Piggindb64fe02008-10-18 20:27:03 -07002024 return -EINVAL;
Nick Piggin83342312006-06-23 02:03:20 -07002025
2026 addr += pgoff << PAGE_SHIFT;
2027 do {
2028 struct page *page = vmalloc_to_page(addr);
Nick Piggindb64fe02008-10-18 20:27:03 -07002029 int ret;
2030
Nick Piggin83342312006-06-23 02:03:20 -07002031 ret = vm_insert_page(vma, uaddr, page);
2032 if (ret)
2033 return ret;
2034
2035 uaddr += PAGE_SIZE;
2036 addr += PAGE_SIZE;
2037 usize -= PAGE_SIZE;
2038 } while (usize > 0);
2039
2040 /* Prevent "things" like memory migration? VM_flags need a cleanup... */
2041 vma->vm_flags |= VM_RESERVED;
2042
Nick Piggindb64fe02008-10-18 20:27:03 -07002043 return 0;
Nick Piggin83342312006-06-23 02:03:20 -07002044}
2045EXPORT_SYMBOL(remap_vmalloc_range);
2046
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -07002047/*
2048 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
2049 * have one.
2050 */
2051void __attribute__((weak)) vmalloc_sync_all(void)
2052{
2053}
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002054
2055
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08002056static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002057{
2058 /* apply_to_page_range() does all the hard work. */
2059 return 0;
2060}
2061
2062/**
2063 * alloc_vm_area - allocate a range of kernel address space
2064 * @size: size of the area
Randy Dunlap76824862008-03-19 17:00:40 -07002065 *
2066 * Returns: NULL on failure, vm_struct on success
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002067 *
2068 * This function reserves a range of kernel address space, and
2069 * allocates pagetables to map that range. No actual mappings
2070 * are created. If the kernel address space is not shared
2071 * between processes, it syncs the pagetable across all
2072 * processes.
2073 */
2074struct vm_struct *alloc_vm_area(size_t size)
2075{
2076 struct vm_struct *area;
2077
Christoph Lameter23016962008-04-28 02:12:42 -07002078 area = get_vm_area_caller(size, VM_IOREMAP,
2079 __builtin_return_address(0));
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002080 if (area == NULL)
2081 return NULL;
2082
2083 /*
2084 * This ensures that page tables are constructed for this region
2085 * of kernel virtual address space and mapped into init_mm.
2086 */
2087 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
2088 area->size, f, NULL)) {
2089 free_vm_area(area);
2090 return NULL;
2091 }
2092
2093 /* Make sure the pagetables are constructed in process kernel
2094 mappings */
2095 vmalloc_sync_all();
2096
2097 return area;
2098}
2099EXPORT_SYMBOL_GPL(alloc_vm_area);
2100
2101void free_vm_area(struct vm_struct *area)
2102{
2103 struct vm_struct *ret;
2104 ret = remove_vm_area(area->addr);
2105 BUG_ON(ret != area);
2106 kfree(area);
2107}
2108EXPORT_SYMBOL_GPL(free_vm_area);
Christoph Lametera10aa572008-04-28 02:12:40 -07002109
Tejun Heo4f8b02b2010-09-03 18:22:47 +02002110#ifdef CONFIG_SMP
Tejun Heoca23e402009-08-14 15:00:52 +09002111static struct vmap_area *node_to_va(struct rb_node *n)
2112{
2113 return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
2114}
2115
2116/**
2117 * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
2118 * @end: target address
2119 * @pnext: out arg for the next vmap_area
2120 * @pprev: out arg for the previous vmap_area
2121 *
2122 * Returns: %true if either or both of next and prev are found,
2123 * %false if no vmap_area exists
2124 *
2125 * Find vmap_areas end addresses of which enclose @end. ie. if not
2126 * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
2127 */
2128static bool pvm_find_next_prev(unsigned long end,
2129 struct vmap_area **pnext,
2130 struct vmap_area **pprev)
2131{
2132 struct rb_node *n = vmap_area_root.rb_node;
2133 struct vmap_area *va = NULL;
2134
2135 while (n) {
2136 va = rb_entry(n, struct vmap_area, rb_node);
2137 if (end < va->va_end)
2138 n = n->rb_left;
2139 else if (end > va->va_end)
2140 n = n->rb_right;
2141 else
2142 break;
2143 }
2144
2145 if (!va)
2146 return false;
2147
2148 if (va->va_end > end) {
2149 *pnext = va;
2150 *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2151 } else {
2152 *pprev = va;
2153 *pnext = node_to_va(rb_next(&(*pprev)->rb_node));
2154 }
2155 return true;
2156}
2157
2158/**
2159 * pvm_determine_end - find the highest aligned address between two vmap_areas
2160 * @pnext: in/out arg for the next vmap_area
2161 * @pprev: in/out arg for the previous vmap_area
2162 * @align: alignment
2163 *
2164 * Returns: determined end address
2165 *
2166 * Find the highest aligned address between *@pnext and *@pprev below
2167 * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned
2168 * down address is between the end addresses of the two vmap_areas.
2169 *
2170 * Please note that the address returned by this function may fall
2171 * inside *@pnext vmap_area. The caller is responsible for checking
2172 * that.
2173 */
2174static unsigned long pvm_determine_end(struct vmap_area **pnext,
2175 struct vmap_area **pprev,
2176 unsigned long align)
2177{
2178 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2179 unsigned long addr;
2180
2181 if (*pnext)
2182 addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
2183 else
2184 addr = vmalloc_end;
2185
2186 while (*pprev && (*pprev)->va_end > addr) {
2187 *pnext = *pprev;
2188 *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2189 }
2190
2191 return addr;
2192}
2193
2194/**
2195 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
2196 * @offsets: array containing offset of each area
2197 * @sizes: array containing size of each area
2198 * @nr_vms: the number of areas to allocate
2199 * @align: alignment, all entries in @offsets and @sizes must be aligned to this
2200 * @gfp_mask: allocation mask
2201 *
2202 * Returns: kmalloc'd vm_struct pointer array pointing to allocated
2203 * vm_structs on success, %NULL on failure
2204 *
2205 * Percpu allocator wants to use congruent vm areas so that it can
2206 * maintain the offsets among percpu areas. This function allocates
2207 * congruent vmalloc areas for it. These areas tend to be scattered
2208 * pretty far, distance between two areas easily going up to
2209 * gigabytes. To avoid interacting with regular vmallocs, these areas
2210 * are allocated from top.
2211 *
2212 * Despite its complicated look, this allocator is rather simple. It
2213 * does everything top-down and scans areas from the end looking for
2214 * matching slot. While scanning, if any of the areas overlaps with
2215 * existing vmap_area, the base address is pulled down to fit the
2216 * area. Scanning is repeated till all the areas fit and then all
2217 * necessary data structres are inserted and the result is returned.
2218 */
2219struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
2220 const size_t *sizes, int nr_vms,
2221 size_t align, gfp_t gfp_mask)
2222{
2223 const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
2224 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2225 struct vmap_area **vas, *prev, *next;
2226 struct vm_struct **vms;
2227 int area, area2, last_area, term_area;
2228 unsigned long base, start, end, last_end;
2229 bool purged = false;
2230
2231 gfp_mask &= GFP_RECLAIM_MASK;
2232
2233 /* verify parameters and allocate data structures */
2234 BUG_ON(align & ~PAGE_MASK || !is_power_of_2(align));
2235 for (last_area = 0, area = 0; area < nr_vms; area++) {
2236 start = offsets[area];
2237 end = start + sizes[area];
2238
2239 /* is everything aligned properly? */
2240 BUG_ON(!IS_ALIGNED(offsets[area], align));
2241 BUG_ON(!IS_ALIGNED(sizes[area], align));
2242
2243 /* detect the area with the highest address */
2244 if (start > offsets[last_area])
2245 last_area = area;
2246
2247 for (area2 = 0; area2 < nr_vms; area2++) {
2248 unsigned long start2 = offsets[area2];
2249 unsigned long end2 = start2 + sizes[area2];
2250
2251 if (area2 == area)
2252 continue;
2253
2254 BUG_ON(start2 >= start && start2 < end);
2255 BUG_ON(end2 <= end && end2 > start);
2256 }
2257 }
2258 last_end = offsets[last_area] + sizes[last_area];
2259
2260 if (vmalloc_end - vmalloc_start < last_end) {
2261 WARN_ON(true);
2262 return NULL;
2263 }
2264
2265 vms = kzalloc(sizeof(vms[0]) * nr_vms, gfp_mask);
2266 vas = kzalloc(sizeof(vas[0]) * nr_vms, gfp_mask);
2267 if (!vas || !vms)
2268 goto err_free;
2269
2270 for (area = 0; area < nr_vms; area++) {
2271 vas[area] = kzalloc(sizeof(struct vmap_area), gfp_mask);
2272 vms[area] = kzalloc(sizeof(struct vm_struct), gfp_mask);
2273 if (!vas[area] || !vms[area])
2274 goto err_free;
2275 }
2276retry:
2277 spin_lock(&vmap_area_lock);
2278
2279 /* start scanning - we scan from the top, begin with the last area */
2280 area = term_area = last_area;
2281 start = offsets[area];
2282 end = start + sizes[area];
2283
2284 if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
2285 base = vmalloc_end - last_end;
2286 goto found;
2287 }
2288 base = pvm_determine_end(&next, &prev, align) - end;
2289
2290 while (true) {
2291 BUG_ON(next && next->va_end <= base + end);
2292 BUG_ON(prev && prev->va_end > base + end);
2293
2294 /*
2295 * base might have underflowed, add last_end before
2296 * comparing.
2297 */
2298 if (base + last_end < vmalloc_start + last_end) {
2299 spin_unlock(&vmap_area_lock);
2300 if (!purged) {
2301 purge_vmap_area_lazy();
2302 purged = true;
2303 goto retry;
2304 }
2305 goto err_free;
2306 }
2307
2308 /*
2309 * If next overlaps, move base downwards so that it's
2310 * right below next and then recheck.
2311 */
2312 if (next && next->va_start < base + end) {
2313 base = pvm_determine_end(&next, &prev, align) - end;
2314 term_area = area;
2315 continue;
2316 }
2317
2318 /*
2319 * If prev overlaps, shift down next and prev and move
2320 * base so that it's right below new next and then
2321 * recheck.
2322 */
2323 if (prev && prev->va_end > base + start) {
2324 next = prev;
2325 prev = node_to_va(rb_prev(&next->rb_node));
2326 base = pvm_determine_end(&next, &prev, align) - end;
2327 term_area = area;
2328 continue;
2329 }
2330
2331 /*
2332 * This area fits, move on to the previous one. If
2333 * the previous one is the terminal one, we're done.
2334 */
2335 area = (area + nr_vms - 1) % nr_vms;
2336 if (area == term_area)
2337 break;
2338 start = offsets[area];
2339 end = start + sizes[area];
2340 pvm_find_next_prev(base + end, &next, &prev);
2341 }
2342found:
2343 /* we've found a fitting base, insert all va's */
2344 for (area = 0; area < nr_vms; area++) {
2345 struct vmap_area *va = vas[area];
2346
2347 va->va_start = base + offsets[area];
2348 va->va_end = va->va_start + sizes[area];
2349 __insert_vmap_area(va);
2350 }
2351
2352 vmap_area_pcpu_hole = base + offsets[last_area];
2353
2354 spin_unlock(&vmap_area_lock);
2355
2356 /* insert all vm's */
2357 for (area = 0; area < nr_vms; area++)
2358 insert_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
2359 pcpu_get_vm_areas);
2360
2361 kfree(vas);
2362 return vms;
2363
2364err_free:
2365 for (area = 0; area < nr_vms; area++) {
2366 if (vas)
2367 kfree(vas[area]);
2368 if (vms)
2369 kfree(vms[area]);
2370 }
2371 kfree(vas);
2372 kfree(vms);
2373 return NULL;
2374}
2375
2376/**
2377 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
2378 * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
2379 * @nr_vms: the number of allocated areas
2380 *
2381 * Free vm_structs and the array allocated by pcpu_get_vm_areas().
2382 */
2383void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
2384{
2385 int i;
2386
2387 for (i = 0; i < nr_vms; i++)
2388 free_vm_area(vms[i]);
2389 kfree(vms);
2390}
Tejun Heo4f8b02b2010-09-03 18:22:47 +02002391#endif /* CONFIG_SMP */
Christoph Lametera10aa572008-04-28 02:12:40 -07002392
2393#ifdef CONFIG_PROC_FS
2394static void *s_start(struct seq_file *m, loff_t *pos)
Namhyung Kime199b5d2010-10-26 14:22:03 -07002395 __acquires(&vmlist_lock)
Christoph Lametera10aa572008-04-28 02:12:40 -07002396{
2397 loff_t n = *pos;
2398 struct vm_struct *v;
2399
2400 read_lock(&vmlist_lock);
2401 v = vmlist;
2402 while (n > 0 && v) {
2403 n--;
2404 v = v->next;
2405 }
2406 if (!n)
2407 return v;
2408
2409 return NULL;
2410
2411}
2412
2413static void *s_next(struct seq_file *m, void *p, loff_t *pos)
2414{
2415 struct vm_struct *v = p;
2416
2417 ++*pos;
2418 return v->next;
2419}
2420
2421static void s_stop(struct seq_file *m, void *p)
Namhyung Kime199b5d2010-10-26 14:22:03 -07002422 __releases(&vmlist_lock)
Christoph Lametera10aa572008-04-28 02:12:40 -07002423{
2424 read_unlock(&vmlist_lock);
2425}
2426
Eric Dumazeta47a1262008-07-23 21:27:38 -07002427static void show_numa_info(struct seq_file *m, struct vm_struct *v)
2428{
2429 if (NUMA_BUILD) {
2430 unsigned int nr, *counters = m->private;
2431
2432 if (!counters)
2433 return;
2434
2435 memset(counters, 0, nr_node_ids * sizeof(unsigned int));
2436
2437 for (nr = 0; nr < v->nr_pages; nr++)
2438 counters[page_to_nid(v->pages[nr])]++;
2439
2440 for_each_node_state(nr, N_HIGH_MEMORY)
2441 if (counters[nr])
2442 seq_printf(m, " N%u=%u", nr, counters[nr]);
2443 }
2444}
2445
Christoph Lametera10aa572008-04-28 02:12:40 -07002446static int s_show(struct seq_file *m, void *p)
2447{
2448 struct vm_struct *v = p;
2449
2450 seq_printf(m, "0x%p-0x%p %7ld",
2451 v->addr, v->addr + v->size, v->size);
2452
Christoph Lameter23016962008-04-28 02:12:42 -07002453 if (v->caller) {
Hugh Dickins9c246242008-12-09 13:14:27 -08002454 char buff[KSYM_SYMBOL_LEN];
Christoph Lameter23016962008-04-28 02:12:42 -07002455
2456 seq_putc(m, ' ');
2457 sprint_symbol(buff, (unsigned long)v->caller);
2458 seq_puts(m, buff);
2459 }
2460
Christoph Lametera10aa572008-04-28 02:12:40 -07002461 if (v->nr_pages)
2462 seq_printf(m, " pages=%d", v->nr_pages);
2463
2464 if (v->phys_addr)
Kenji Kaneshigeffa71f32010-06-18 12:22:40 +09002465 seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
Christoph Lametera10aa572008-04-28 02:12:40 -07002466
2467 if (v->flags & VM_IOREMAP)
2468 seq_printf(m, " ioremap");
2469
2470 if (v->flags & VM_ALLOC)
2471 seq_printf(m, " vmalloc");
2472
2473 if (v->flags & VM_MAP)
2474 seq_printf(m, " vmap");
2475
2476 if (v->flags & VM_USERMAP)
2477 seq_printf(m, " user");
2478
2479 if (v->flags & VM_VPAGES)
2480 seq_printf(m, " vpages");
2481
Eric Dumazeta47a1262008-07-23 21:27:38 -07002482 show_numa_info(m, v);
Christoph Lametera10aa572008-04-28 02:12:40 -07002483 seq_putc(m, '\n');
2484 return 0;
2485}
2486
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +04002487static const struct seq_operations vmalloc_op = {
Christoph Lametera10aa572008-04-28 02:12:40 -07002488 .start = s_start,
2489 .next = s_next,
2490 .stop = s_stop,
2491 .show = s_show,
2492};
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +04002493
2494static int vmalloc_open(struct inode *inode, struct file *file)
2495{
2496 unsigned int *ptr = NULL;
2497 int ret;
2498
Kulikov Vasiliy51980ac2010-08-09 17:19:58 -07002499 if (NUMA_BUILD) {
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +04002500 ptr = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL);
Kulikov Vasiliy51980ac2010-08-09 17:19:58 -07002501 if (ptr == NULL)
2502 return -ENOMEM;
2503 }
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +04002504 ret = seq_open(file, &vmalloc_op);
2505 if (!ret) {
2506 struct seq_file *m = file->private_data;
2507 m->private = ptr;
2508 } else
2509 kfree(ptr);
2510 return ret;
2511}
2512
2513static const struct file_operations proc_vmalloc_operations = {
2514 .open = vmalloc_open,
2515 .read = seq_read,
2516 .llseek = seq_lseek,
2517 .release = seq_release_private,
2518};
2519
2520static int __init proc_vmalloc_init(void)
2521{
2522 proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
2523 return 0;
2524}
2525module_init(proc_vmalloc_init);
Christoph Lametera10aa572008-04-28 02:12:40 -07002526#endif
2527