blob: 8e3c9c5a3042b6149ff783bb1dfc0c0e08dc4046 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
Christoph Lameter930fc452005-10-29 18:15:41 -07008 * Numa awareness, Christoph Lameter, SGI, June 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
Nick Piggindb64fe02008-10-18 20:27:03 -070011#include <linux/vmalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/highmem.h>
Alexey Dobriyand43c36d2009-10-07 17:09:06 +040015#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/slab.h>
17#include <linux/spinlock.h>
18#include <linux/interrupt.h>
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +040019#include <linux/proc_fs.h>
Christoph Lametera10aa572008-04-28 02:12:40 -070020#include <linux/seq_file.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070021#include <linux/debugobjects.h>
Christoph Lameter23016962008-04-28 02:12:42 -070022#include <linux/kallsyms.h>
Nick Piggindb64fe02008-10-18 20:27:03 -070023#include <linux/list.h>
24#include <linux/rbtree.h>
25#include <linux/radix-tree.h>
26#include <linux/rcupdate.h>
Tejun Heof0aa6612009-02-20 16:29:08 +090027#include <linux/pfn.h>
Catalin Marinas89219d32009-06-11 13:23:19 +010028#include <linux/kmemleak.h>
Arun Sharma600634972011-07-26 16:09:06 -070029#include <linux/atomic.h>
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -070030#include <linux/compiler.h>
Al Viro32fcfd42013-03-10 20:14:08 -040031#include <linux/llist.h>
Toshi Kani0f616be2015-04-14 15:47:17 -070032#include <linux/bitops.h>
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -070033
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <asm/uaccess.h>
35#include <asm/tlbflush.h>
David Miller2dca6992009-09-21 12:22:34 -070036#include <asm/shmparam.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Mel Gormandd56b042015-11-06 16:28:43 -080038#include "internal.h"
39
Al Viro32fcfd42013-03-10 20:14:08 -040040struct vfree_deferred {
41 struct llist_head list;
42 struct work_struct wq;
43};
44static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
45
46static void __vunmap(const void *, int);
47
48static void free_work(struct work_struct *w)
49{
50 struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
51 struct llist_node *llnode = llist_del_all(&p->list);
52 while (llnode) {
53 void *p = llnode;
54 llnode = llist_next(llnode);
55 __vunmap(p, 1);
56 }
57}
58
Nick Piggindb64fe02008-10-18 20:27:03 -070059/*** Page table manipulation functions ***/
Adrian Bunkb2213852006-09-25 23:31:02 -070060
Linus Torvalds1da177e2005-04-16 15:20:36 -070061static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
62{
63 pte_t *pte;
64
65 pte = pte_offset_kernel(pmd, addr);
66 do {
67 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
68 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
69 } while (pte++, addr += PAGE_SIZE, addr != end);
70}
71
Nick Piggindb64fe02008-10-18 20:27:03 -070072static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -070073{
74 pmd_t *pmd;
75 unsigned long next;
76
77 pmd = pmd_offset(pud, addr);
78 do {
79 next = pmd_addr_end(addr, end);
Toshi Kanib9820d82015-04-14 15:47:26 -070080 if (pmd_clear_huge(pmd))
81 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070082 if (pmd_none_or_clear_bad(pmd))
83 continue;
84 vunmap_pte_range(pmd, addr, next);
85 } while (pmd++, addr = next, addr != end);
86}
87
Nick Piggindb64fe02008-10-18 20:27:03 -070088static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -070089{
90 pud_t *pud;
91 unsigned long next;
92
93 pud = pud_offset(pgd, addr);
94 do {
95 next = pud_addr_end(addr, end);
Toshi Kanib9820d82015-04-14 15:47:26 -070096 if (pud_clear_huge(pud))
97 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 if (pud_none_or_clear_bad(pud))
99 continue;
100 vunmap_pmd_range(pud, addr, next);
101 } while (pud++, addr = next, addr != end);
102}
103
Nick Piggindb64fe02008-10-18 20:27:03 -0700104static void vunmap_page_range(unsigned long addr, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105{
106 pgd_t *pgd;
107 unsigned long next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108
109 BUG_ON(addr >= end);
110 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 do {
112 next = pgd_addr_end(addr, end);
113 if (pgd_none_or_clear_bad(pgd))
114 continue;
115 vunmap_pud_range(pgd, addr, next);
116 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117}
118
119static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
Nick Piggindb64fe02008-10-18 20:27:03 -0700120 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121{
122 pte_t *pte;
123
Nick Piggindb64fe02008-10-18 20:27:03 -0700124 /*
125 * nr is a running index into the array which helps higher level
126 * callers keep track of where we're up to.
127 */
128
Hugh Dickins872fec12005-10-29 18:16:21 -0700129 pte = pte_alloc_kernel(pmd, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 if (!pte)
131 return -ENOMEM;
132 do {
Nick Piggindb64fe02008-10-18 20:27:03 -0700133 struct page *page = pages[*nr];
134
135 if (WARN_ON(!pte_none(*pte)))
136 return -EBUSY;
137 if (WARN_ON(!page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 return -ENOMEM;
139 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
Nick Piggindb64fe02008-10-18 20:27:03 -0700140 (*nr)++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141 } while (pte++, addr += PAGE_SIZE, addr != end);
142 return 0;
143}
144
Nick Piggindb64fe02008-10-18 20:27:03 -0700145static int vmap_pmd_range(pud_t *pud, unsigned long addr,
146 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147{
148 pmd_t *pmd;
149 unsigned long next;
150
151 pmd = pmd_alloc(&init_mm, pud, addr);
152 if (!pmd)
153 return -ENOMEM;
154 do {
155 next = pmd_addr_end(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700156 if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 return -ENOMEM;
158 } while (pmd++, addr = next, addr != end);
159 return 0;
160}
161
Nick Piggindb64fe02008-10-18 20:27:03 -0700162static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
163 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164{
165 pud_t *pud;
166 unsigned long next;
167
168 pud = pud_alloc(&init_mm, pgd, addr);
169 if (!pud)
170 return -ENOMEM;
171 do {
172 next = pud_addr_end(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700173 if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 return -ENOMEM;
175 } while (pud++, addr = next, addr != end);
176 return 0;
177}
178
Nick Piggindb64fe02008-10-18 20:27:03 -0700179/*
180 * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
181 * will have pfns corresponding to the "pages" array.
182 *
183 * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
184 */
Tejun Heo8fc48982009-02-20 16:29:08 +0900185static int vmap_page_range_noflush(unsigned long start, unsigned long end,
186 pgprot_t prot, struct page **pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187{
188 pgd_t *pgd;
189 unsigned long next;
Adam Lackorzynski2e4e27c2009-01-04 12:00:46 -0800190 unsigned long addr = start;
Nick Piggindb64fe02008-10-18 20:27:03 -0700191 int err = 0;
192 int nr = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194 BUG_ON(addr >= end);
195 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 do {
197 next = pgd_addr_end(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700198 err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 if (err)
Figo.zhangbf88c8c2009-09-21 17:01:47 -0700200 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 } while (pgd++, addr = next, addr != end);
Nick Piggindb64fe02008-10-18 20:27:03 -0700202
Nick Piggindb64fe02008-10-18 20:27:03 -0700203 return nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204}
205
Tejun Heo8fc48982009-02-20 16:29:08 +0900206static int vmap_page_range(unsigned long start, unsigned long end,
207 pgprot_t prot, struct page **pages)
208{
209 int ret;
210
211 ret = vmap_page_range_noflush(start, end, prot, pages);
212 flush_cache_vmap(start, end);
213 return ret;
214}
215
KAMEZAWA Hiroyuki81ac3ad2009-09-22 16:45:49 -0700216int is_vmalloc_or_module_addr(const void *x)
Linus Torvalds73bdf0a2008-10-15 08:35:12 -0700217{
218 /*
Russell Kingab4f2ee2008-11-06 17:11:07 +0000219 * ARM, x86-64 and sparc64 put modules in a special place,
Linus Torvalds73bdf0a2008-10-15 08:35:12 -0700220 * and fall back on vmalloc() if that fails. Others
221 * just put it in the vmalloc space.
222 */
223#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
224 unsigned long addr = (unsigned long)x;
225 if (addr >= MODULES_VADDR && addr < MODULES_END)
226 return 1;
227#endif
228 return is_vmalloc_addr(x);
229}
230
Christoph Lameter48667e72008-02-04 22:28:31 -0800231/*
malcadd688f2014-01-27 17:06:53 -0800232 * Walk a vmap address to the struct page it maps.
Christoph Lameter48667e72008-02-04 22:28:31 -0800233 */
malcadd688f2014-01-27 17:06:53 -0800234struct page *vmalloc_to_page(const void *vmalloc_addr)
Christoph Lameter48667e72008-02-04 22:28:31 -0800235{
236 unsigned long addr = (unsigned long) vmalloc_addr;
malcadd688f2014-01-27 17:06:53 -0800237 struct page *page = NULL;
Christoph Lameter48667e72008-02-04 22:28:31 -0800238 pgd_t *pgd = pgd_offset_k(addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800239
Ingo Molnar7aa413d2008-06-19 13:28:11 +0200240 /*
241 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
242 * architectures that do not vmalloc module space
243 */
Linus Torvalds73bdf0a2008-10-15 08:35:12 -0700244 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
Jiri Slaby59ea7462008-06-12 13:56:40 +0200245
Christoph Lameter48667e72008-02-04 22:28:31 -0800246 if (!pgd_none(*pgd)) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700247 pud_t *pud = pud_offset(pgd, addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800248 if (!pud_none(*pud)) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700249 pmd_t *pmd = pmd_offset(pud, addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800250 if (!pmd_none(*pmd)) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700251 pte_t *ptep, pte;
252
Christoph Lameter48667e72008-02-04 22:28:31 -0800253 ptep = pte_offset_map(pmd, addr);
254 pte = *ptep;
255 if (pte_present(pte))
malcadd688f2014-01-27 17:06:53 -0800256 page = pte_page(pte);
Christoph Lameter48667e72008-02-04 22:28:31 -0800257 pte_unmap(ptep);
258 }
259 }
260 }
malcadd688f2014-01-27 17:06:53 -0800261 return page;
Jianyu Zhanece86e222014-01-21 15:49:12 -0800262}
263EXPORT_SYMBOL(vmalloc_to_page);
264
malcadd688f2014-01-27 17:06:53 -0800265/*
266 * Map a vmalloc()-space virtual address to the physical page frame number.
267 */
268unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
269{
270 return page_to_pfn(vmalloc_to_page(vmalloc_addr));
271}
272EXPORT_SYMBOL(vmalloc_to_pfn);
273
Nick Piggindb64fe02008-10-18 20:27:03 -0700274
275/*** Global kva allocator ***/
276
277#define VM_LAZY_FREE 0x01
278#define VM_LAZY_FREEING 0x02
279#define VM_VM_AREA 0x04
280
Nick Piggindb64fe02008-10-18 20:27:03 -0700281static DEFINE_SPINLOCK(vmap_area_lock);
Joonsoo Kimf1c40692013-04-29 15:07:37 -0700282/* Export for kexec only */
283LIST_HEAD(vmap_area_list);
Nick Piggin89699602011-03-22 16:30:36 -0700284static struct rb_root vmap_area_root = RB_ROOT;
285
286/* The vmap cache globals are protected by vmap_area_lock */
287static struct rb_node *free_vmap_cache;
288static unsigned long cached_hole_size;
289static unsigned long cached_vstart;
290static unsigned long cached_align;
291
Tejun Heoca23e402009-08-14 15:00:52 +0900292static unsigned long vmap_area_pcpu_hole;
Nick Piggindb64fe02008-10-18 20:27:03 -0700293
294static struct vmap_area *__find_vmap_area(unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295{
Nick Piggindb64fe02008-10-18 20:27:03 -0700296 struct rb_node *n = vmap_area_root.rb_node;
297
298 while (n) {
299 struct vmap_area *va;
300
301 va = rb_entry(n, struct vmap_area, rb_node);
302 if (addr < va->va_start)
303 n = n->rb_left;
HATAYAMA Daisukecef2ac32013-07-03 15:02:17 -0700304 else if (addr >= va->va_end)
Nick Piggindb64fe02008-10-18 20:27:03 -0700305 n = n->rb_right;
306 else
307 return va;
308 }
309
310 return NULL;
311}
312
313static void __insert_vmap_area(struct vmap_area *va)
314{
315 struct rb_node **p = &vmap_area_root.rb_node;
316 struct rb_node *parent = NULL;
317 struct rb_node *tmp;
318
319 while (*p) {
Namhyung Kim170168d2010-10-26 14:22:02 -0700320 struct vmap_area *tmp_va;
Nick Piggindb64fe02008-10-18 20:27:03 -0700321
322 parent = *p;
Namhyung Kim170168d2010-10-26 14:22:02 -0700323 tmp_va = rb_entry(parent, struct vmap_area, rb_node);
324 if (va->va_start < tmp_va->va_end)
Nick Piggindb64fe02008-10-18 20:27:03 -0700325 p = &(*p)->rb_left;
Namhyung Kim170168d2010-10-26 14:22:02 -0700326 else if (va->va_end > tmp_va->va_start)
Nick Piggindb64fe02008-10-18 20:27:03 -0700327 p = &(*p)->rb_right;
328 else
329 BUG();
330 }
331
332 rb_link_node(&va->rb_node, parent, p);
333 rb_insert_color(&va->rb_node, &vmap_area_root);
334
Joonsoo Kim4341fa42013-04-29 15:07:39 -0700335 /* address-sort this list */
Nick Piggindb64fe02008-10-18 20:27:03 -0700336 tmp = rb_prev(&va->rb_node);
337 if (tmp) {
338 struct vmap_area *prev;
339 prev = rb_entry(tmp, struct vmap_area, rb_node);
340 list_add_rcu(&va->list, &prev->list);
341 } else
342 list_add_rcu(&va->list, &vmap_area_list);
343}
344
345static void purge_vmap_area_lazy(void);
346
347/*
348 * Allocate a region of KVA of the specified size and alignment, within the
349 * vstart and vend.
350 */
351static struct vmap_area *alloc_vmap_area(unsigned long size,
352 unsigned long align,
353 unsigned long vstart, unsigned long vend,
354 int node, gfp_t gfp_mask)
355{
356 struct vmap_area *va;
357 struct rb_node *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 unsigned long addr;
Nick Piggindb64fe02008-10-18 20:27:03 -0700359 int purged = 0;
Nick Piggin89699602011-03-22 16:30:36 -0700360 struct vmap_area *first;
Nick Piggindb64fe02008-10-18 20:27:03 -0700361
Nick Piggin77669702009-02-27 14:03:03 -0800362 BUG_ON(!size);
Alexander Kuleshov891c49a2015-11-05 18:46:51 -0800363 BUG_ON(offset_in_page(size));
Nick Piggin89699602011-03-22 16:30:36 -0700364 BUG_ON(!is_power_of_2(align));
Nick Piggindb64fe02008-10-18 20:27:03 -0700365
Nick Piggindb64fe02008-10-18 20:27:03 -0700366 va = kmalloc_node(sizeof(struct vmap_area),
367 gfp_mask & GFP_RECLAIM_MASK, node);
368 if (unlikely(!va))
369 return ERR_PTR(-ENOMEM);
370
Catalin Marinas7f88f882013-11-12 15:07:45 -0800371 /*
372 * Only scan the relevant parts containing pointers to other objects
373 * to avoid false negatives.
374 */
375 kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
376
Nick Piggindb64fe02008-10-18 20:27:03 -0700377retry:
378 spin_lock(&vmap_area_lock);
Nick Piggin89699602011-03-22 16:30:36 -0700379 /*
380 * Invalidate cache if we have more permissive parameters.
381 * cached_hole_size notes the largest hole noticed _below_
382 * the vmap_area cached in free_vmap_cache: if size fits
383 * into that hole, we want to scan from vstart to reuse
384 * the hole instead of allocating above free_vmap_cache.
385 * Note that __free_vmap_area may update free_vmap_cache
386 * without updating cached_hole_size or cached_align.
387 */
388 if (!free_vmap_cache ||
389 size < cached_hole_size ||
390 vstart < cached_vstart ||
391 align < cached_align) {
392nocache:
393 cached_hole_size = 0;
394 free_vmap_cache = NULL;
395 }
396 /* record if we encounter less permissive parameters */
397 cached_vstart = vstart;
398 cached_align = align;
Nick Piggin77669702009-02-27 14:03:03 -0800399
Nick Piggin89699602011-03-22 16:30:36 -0700400 /* find starting point for our search */
401 if (free_vmap_cache) {
402 first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
Johannes Weiner248ac0e2011-05-24 17:11:43 -0700403 addr = ALIGN(first->va_end, align);
Nick Piggin89699602011-03-22 16:30:36 -0700404 if (addr < vstart)
405 goto nocache;
Zhang Yanfeibcb615a2013-07-08 16:00:19 -0700406 if (addr + size < addr)
Nick Piggin89699602011-03-22 16:30:36 -0700407 goto overflow;
Nick Piggindb64fe02008-10-18 20:27:03 -0700408
Nick Piggin89699602011-03-22 16:30:36 -0700409 } else {
410 addr = ALIGN(vstart, align);
Zhang Yanfeibcb615a2013-07-08 16:00:19 -0700411 if (addr + size < addr)
Nick Piggin89699602011-03-22 16:30:36 -0700412 goto overflow;
413
414 n = vmap_area_root.rb_node;
415 first = NULL;
416
417 while (n) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700418 struct vmap_area *tmp;
419 tmp = rb_entry(n, struct vmap_area, rb_node);
420 if (tmp->va_end >= addr) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700421 first = tmp;
Nick Piggin89699602011-03-22 16:30:36 -0700422 if (tmp->va_start <= addr)
423 break;
424 n = n->rb_left;
425 } else
Nick Piggindb64fe02008-10-18 20:27:03 -0700426 n = n->rb_right;
Nick Piggin89699602011-03-22 16:30:36 -0700427 }
Nick Piggindb64fe02008-10-18 20:27:03 -0700428
429 if (!first)
430 goto found;
Nick Piggindb64fe02008-10-18 20:27:03 -0700431 }
Nick Piggin89699602011-03-22 16:30:36 -0700432
433 /* from the starting point, walk areas until a suitable hole is found */
Johannes Weiner248ac0e2011-05-24 17:11:43 -0700434 while (addr + size > first->va_start && addr + size <= vend) {
Nick Piggin89699602011-03-22 16:30:36 -0700435 if (addr + cached_hole_size < first->va_start)
436 cached_hole_size = first->va_start - addr;
Johannes Weiner248ac0e2011-05-24 17:11:43 -0700437 addr = ALIGN(first->va_end, align);
Zhang Yanfeibcb615a2013-07-08 16:00:19 -0700438 if (addr + size < addr)
Nick Piggin89699602011-03-22 16:30:36 -0700439 goto overflow;
440
Hong zhi guo92ca9222012-07-31 16:41:35 -0700441 if (list_is_last(&first->list, &vmap_area_list))
Nick Piggin89699602011-03-22 16:30:36 -0700442 goto found;
Hong zhi guo92ca9222012-07-31 16:41:35 -0700443
444 first = list_entry(first->list.next,
445 struct vmap_area, list);
Nick Piggin89699602011-03-22 16:30:36 -0700446 }
447
Nick Piggindb64fe02008-10-18 20:27:03 -0700448found:
Nick Piggin89699602011-03-22 16:30:36 -0700449 if (addr + size > vend)
450 goto overflow;
Nick Piggindb64fe02008-10-18 20:27:03 -0700451
452 va->va_start = addr;
453 va->va_end = addr + size;
454 va->flags = 0;
455 __insert_vmap_area(va);
Nick Piggin89699602011-03-22 16:30:36 -0700456 free_vmap_cache = &va->rb_node;
Nick Piggindb64fe02008-10-18 20:27:03 -0700457 spin_unlock(&vmap_area_lock);
458
Nick Piggin89699602011-03-22 16:30:36 -0700459 BUG_ON(va->va_start & (align-1));
460 BUG_ON(va->va_start < vstart);
461 BUG_ON(va->va_end > vend);
462
Nick Piggindb64fe02008-10-18 20:27:03 -0700463 return va;
Nick Piggin89699602011-03-22 16:30:36 -0700464
465overflow:
466 spin_unlock(&vmap_area_lock);
467 if (!purged) {
468 purge_vmap_area_lazy();
469 purged = 1;
470 goto retry;
471 }
472 if (printk_ratelimit())
Pintu Kumar0cbc8532014-12-10 15:42:56 -0800473 pr_warn("vmap allocation for size %lu failed: "
Nick Piggin89699602011-03-22 16:30:36 -0700474 "use vmalloc=<size> to increase size.\n", size);
475 kfree(va);
476 return ERR_PTR(-EBUSY);
Nick Piggindb64fe02008-10-18 20:27:03 -0700477}
478
Nick Piggindb64fe02008-10-18 20:27:03 -0700479static void __free_vmap_area(struct vmap_area *va)
480{
481 BUG_ON(RB_EMPTY_NODE(&va->rb_node));
Nick Piggin89699602011-03-22 16:30:36 -0700482
483 if (free_vmap_cache) {
484 if (va->va_end < cached_vstart) {
485 free_vmap_cache = NULL;
486 } else {
487 struct vmap_area *cache;
488 cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
489 if (va->va_start <= cache->va_start) {
490 free_vmap_cache = rb_prev(&va->rb_node);
491 /*
492 * We don't try to update cached_hole_size or
493 * cached_align, but it won't go very wrong.
494 */
495 }
496 }
497 }
Nick Piggindb64fe02008-10-18 20:27:03 -0700498 rb_erase(&va->rb_node, &vmap_area_root);
499 RB_CLEAR_NODE(&va->rb_node);
500 list_del_rcu(&va->list);
501
Tejun Heoca23e402009-08-14 15:00:52 +0900502 /*
503 * Track the highest possible candidate for pcpu area
504 * allocation. Areas outside of vmalloc area can be returned
505 * here too, consider only end addresses which fall inside
506 * vmalloc area proper.
507 */
508 if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
509 vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
510
Lai Jiangshan14769de2011-03-18 12:12:19 +0800511 kfree_rcu(va, rcu_head);
Nick Piggindb64fe02008-10-18 20:27:03 -0700512}
513
514/*
515 * Free a region of KVA allocated by alloc_vmap_area
516 */
517static void free_vmap_area(struct vmap_area *va)
518{
519 spin_lock(&vmap_area_lock);
520 __free_vmap_area(va);
521 spin_unlock(&vmap_area_lock);
522}
523
524/*
525 * Clear the pagetable entries of a given vmap_area
526 */
527static void unmap_vmap_area(struct vmap_area *va)
528{
529 vunmap_page_range(va->va_start, va->va_end);
530}
531
Nick Piggincd528582009-01-06 14:39:20 -0800532static void vmap_debug_free_range(unsigned long start, unsigned long end)
533{
534 /*
535 * Unmap page tables and force a TLB flush immediately if
536 * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
537 * bugs similarly to those in linear kernel virtual address
538 * space after a page has been freed.
539 *
540 * All the lazy freeing logic is still retained, in order to
541 * minimise intrusiveness of this debugging feature.
542 *
543 * This is going to be *slow* (linear kernel virtual address
544 * debugging doesn't do a broadcast TLB flush so it is a lot
545 * faster).
546 */
547#ifdef CONFIG_DEBUG_PAGEALLOC
548 vunmap_page_range(start, end);
549 flush_tlb_kernel_range(start, end);
550#endif
551}
552
Nick Piggindb64fe02008-10-18 20:27:03 -0700553/*
554 * lazy_max_pages is the maximum amount of virtual address space we gather up
555 * before attempting to purge with a TLB flush.
556 *
557 * There is a tradeoff here: a larger number will cover more kernel page tables
558 * and take slightly longer to purge, but it will linearly reduce the number of
559 * global TLB flushes that must be performed. It would seem natural to scale
560 * this number up linearly with the number of CPUs (because vmapping activity
561 * could also scale linearly with the number of CPUs), however it is likely
562 * that in practice, workloads might be constrained in other ways that mean
563 * vmap activity will not scale linearly with CPUs. Also, I want to be
564 * conservative and not introduce a big latency on huge systems, so go with
565 * a less aggressive log scale. It will still be an improvement over the old
566 * code, and it will be simple to change the scale factor if we find that it
567 * becomes a problem on bigger systems.
568 */
569static unsigned long lazy_max_pages(void)
570{
571 unsigned int log;
572
573 log = fls(num_online_cpus());
574
575 return log * (32UL * 1024 * 1024 / PAGE_SIZE);
576}
577
578static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
579
Nick Piggin02b709d2010-02-01 22:25:57 +1100580/* for per-CPU blocks */
581static void purge_fragmented_blocks_allcpus(void);
582
Nick Piggindb64fe02008-10-18 20:27:03 -0700583/*
Cliff Wickman3ee48b62010-09-16 11:44:02 -0500584 * called before a call to iounmap() if the caller wants vm_area_struct's
585 * immediately freed.
586 */
587void set_iounmap_nonlazy(void)
588{
589 atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
590}
591
592/*
Nick Piggindb64fe02008-10-18 20:27:03 -0700593 * Purges all lazily-freed vmap areas.
594 *
595 * If sync is 0 then don't purge if there is already a purge in progress.
596 * If force_flush is 1, then flush kernel TLBs between *start and *end even
597 * if we found no lazy vmap areas to unmap (callers can use this to optimise
598 * their own TLB flushing).
599 * Returns with *start = min(*start, lowest purged address)
600 * *end = max(*end, highest purged address)
601 */
602static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
603 int sync, int force_flush)
604{
Andrew Morton46666d82009-01-15 13:51:15 -0800605 static DEFINE_SPINLOCK(purge_lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700606 LIST_HEAD(valist);
607 struct vmap_area *va;
Vegard Nossumcbb76672009-02-27 14:03:04 -0800608 struct vmap_area *n_va;
Nick Piggindb64fe02008-10-18 20:27:03 -0700609 int nr = 0;
610
611 /*
612 * If sync is 0 but force_flush is 1, we'll go sync anyway but callers
613 * should not expect such behaviour. This just simplifies locking for
614 * the case that isn't actually used at the moment anyway.
615 */
616 if (!sync && !force_flush) {
Andrew Morton46666d82009-01-15 13:51:15 -0800617 if (!spin_trylock(&purge_lock))
Nick Piggindb64fe02008-10-18 20:27:03 -0700618 return;
619 } else
Andrew Morton46666d82009-01-15 13:51:15 -0800620 spin_lock(&purge_lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700621
Nick Piggin02b709d2010-02-01 22:25:57 +1100622 if (sync)
623 purge_fragmented_blocks_allcpus();
624
Nick Piggindb64fe02008-10-18 20:27:03 -0700625 rcu_read_lock();
626 list_for_each_entry_rcu(va, &vmap_area_list, list) {
627 if (va->flags & VM_LAZY_FREE) {
628 if (va->va_start < *start)
629 *start = va->va_start;
630 if (va->va_end > *end)
631 *end = va->va_end;
632 nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
Nick Piggindb64fe02008-10-18 20:27:03 -0700633 list_add_tail(&va->purge_list, &valist);
634 va->flags |= VM_LAZY_FREEING;
635 va->flags &= ~VM_LAZY_FREE;
636 }
637 }
638 rcu_read_unlock();
639
Yongseok Koh88f50042010-01-19 17:33:49 +0900640 if (nr)
Nick Piggindb64fe02008-10-18 20:27:03 -0700641 atomic_sub(nr, &vmap_lazy_nr);
Nick Piggindb64fe02008-10-18 20:27:03 -0700642
643 if (nr || force_flush)
644 flush_tlb_kernel_range(*start, *end);
645
646 if (nr) {
647 spin_lock(&vmap_area_lock);
Vegard Nossumcbb76672009-02-27 14:03:04 -0800648 list_for_each_entry_safe(va, n_va, &valist, purge_list)
Nick Piggindb64fe02008-10-18 20:27:03 -0700649 __free_vmap_area(va);
650 spin_unlock(&vmap_area_lock);
651 }
Andrew Morton46666d82009-01-15 13:51:15 -0800652 spin_unlock(&purge_lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700653}
654
655/*
Nick Piggin496850e2008-11-19 15:36:33 -0800656 * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
657 * is already purging.
658 */
659static void try_purge_vmap_area_lazy(void)
660{
661 unsigned long start = ULONG_MAX, end = 0;
662
663 __purge_vmap_area_lazy(&start, &end, 0, 0);
664}
665
666/*
Nick Piggindb64fe02008-10-18 20:27:03 -0700667 * Kick off a purge of the outstanding lazy areas.
668 */
669static void purge_vmap_area_lazy(void)
670{
671 unsigned long start = ULONG_MAX, end = 0;
672
Nick Piggin496850e2008-11-19 15:36:33 -0800673 __purge_vmap_area_lazy(&start, &end, 1, 0);
Nick Piggindb64fe02008-10-18 20:27:03 -0700674}
675
676/*
Jeremy Fitzhardinge64141da2010-12-02 14:31:18 -0800677 * Free a vmap area, caller ensuring that the area has been unmapped
678 * and flush_cache_vunmap had been called for the correct range
679 * previously.
Nick Piggindb64fe02008-10-18 20:27:03 -0700680 */
Jeremy Fitzhardinge64141da2010-12-02 14:31:18 -0800681static void free_vmap_area_noflush(struct vmap_area *va)
Nick Piggindb64fe02008-10-18 20:27:03 -0700682{
683 va->flags |= VM_LAZY_FREE;
684 atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
685 if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
Nick Piggin496850e2008-11-19 15:36:33 -0800686 try_purge_vmap_area_lazy();
Nick Piggindb64fe02008-10-18 20:27:03 -0700687}
688
Nick Pigginb29acbd2008-12-01 13:13:47 -0800689/*
Jeremy Fitzhardinge64141da2010-12-02 14:31:18 -0800690 * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
691 * called for the correct range previously.
692 */
693static void free_unmap_vmap_area_noflush(struct vmap_area *va)
694{
695 unmap_vmap_area(va);
696 free_vmap_area_noflush(va);
697}
698
699/*
Nick Pigginb29acbd2008-12-01 13:13:47 -0800700 * Free and unmap a vmap area
701 */
702static void free_unmap_vmap_area(struct vmap_area *va)
703{
704 flush_cache_vunmap(va->va_start, va->va_end);
705 free_unmap_vmap_area_noflush(va);
706}
707
Nick Piggindb64fe02008-10-18 20:27:03 -0700708static struct vmap_area *find_vmap_area(unsigned long addr)
709{
710 struct vmap_area *va;
711
712 spin_lock(&vmap_area_lock);
713 va = __find_vmap_area(addr);
714 spin_unlock(&vmap_area_lock);
715
716 return va;
717}
718
719static void free_unmap_vmap_area_addr(unsigned long addr)
720{
721 struct vmap_area *va;
722
723 va = find_vmap_area(addr);
724 BUG_ON(!va);
725 free_unmap_vmap_area(va);
726}
727
728
729/*** Per cpu kva allocator ***/
730
731/*
732 * vmap space is limited especially on 32 bit architectures. Ensure there is
733 * room for at least 16 percpu vmap blocks per CPU.
734 */
735/*
736 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
737 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
738 * instead (we just need a rough idea)
739 */
740#if BITS_PER_LONG == 32
741#define VMALLOC_SPACE (128UL*1024*1024)
742#else
743#define VMALLOC_SPACE (128UL*1024*1024*1024)
744#endif
745
746#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
747#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
748#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
749#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
750#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
751#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
Clemens Ladischf982f912011-06-21 22:09:50 +0200752#define VMAP_BBMAP_BITS \
753 VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
754 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
755 VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
Nick Piggindb64fe02008-10-18 20:27:03 -0700756
757#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
758
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +1100759static bool vmap_initialized __read_mostly = false;
760
Nick Piggindb64fe02008-10-18 20:27:03 -0700761struct vmap_block_queue {
762 spinlock_t lock;
763 struct list_head free;
Nick Piggindb64fe02008-10-18 20:27:03 -0700764};
765
766struct vmap_block {
767 spinlock_t lock;
768 struct vmap_area *va;
Nick Piggindb64fe02008-10-18 20:27:03 -0700769 unsigned long free, dirty;
Roman Pen7d61bfe2015-04-15 16:13:55 -0700770 unsigned long dirty_min, dirty_max; /*< dirty range */
Nick Pigginde560422010-02-01 22:24:18 +1100771 struct list_head free_list;
772 struct rcu_head rcu_head;
Nick Piggin02b709d2010-02-01 22:25:57 +1100773 struct list_head purge;
Nick Piggindb64fe02008-10-18 20:27:03 -0700774};
775
776/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
777static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
778
779/*
780 * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
781 * in the free path. Could get rid of this if we change the API to return a
782 * "cookie" from alloc, to be passed to free. But no big deal yet.
783 */
784static DEFINE_SPINLOCK(vmap_block_tree_lock);
785static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
786
787/*
788 * We should probably have a fallback mechanism to allocate virtual memory
789 * out of partially filled vmap blocks. However vmap block sizing should be
790 * fairly reasonable according to the vmalloc size, so it shouldn't be a
791 * big problem.
792 */
793
794static unsigned long addr_to_vb_idx(unsigned long addr)
795{
796 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
797 addr /= VMAP_BLOCK_SIZE;
798 return addr;
799}
800
Roman Pencf725ce2015-04-15 16:13:52 -0700801static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
802{
803 unsigned long addr;
804
805 addr = va_start + (pages_off << PAGE_SHIFT);
806 BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
807 return (void *)addr;
808}
809
810/**
811 * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
812 * block. Of course pages number can't exceed VMAP_BBMAP_BITS
813 * @order: how many 2^order pages should be occupied in newly allocated block
814 * @gfp_mask: flags for the page level allocator
815 *
816 * Returns: virtual address in a newly allocated block or ERR_PTR(-errno)
817 */
818static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
Nick Piggindb64fe02008-10-18 20:27:03 -0700819{
820 struct vmap_block_queue *vbq;
821 struct vmap_block *vb;
822 struct vmap_area *va;
823 unsigned long vb_idx;
824 int node, err;
Roman Pencf725ce2015-04-15 16:13:52 -0700825 void *vaddr;
Nick Piggindb64fe02008-10-18 20:27:03 -0700826
827 node = numa_node_id();
828
829 vb = kmalloc_node(sizeof(struct vmap_block),
830 gfp_mask & GFP_RECLAIM_MASK, node);
831 if (unlikely(!vb))
832 return ERR_PTR(-ENOMEM);
833
834 va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
835 VMALLOC_START, VMALLOC_END,
836 node, gfp_mask);
Tobias Klauserddf9c6d2011-01-13 15:46:15 -0800837 if (IS_ERR(va)) {
Nick Piggindb64fe02008-10-18 20:27:03 -0700838 kfree(vb);
Julia Lawalle7d86342010-08-09 17:18:28 -0700839 return ERR_CAST(va);
Nick Piggindb64fe02008-10-18 20:27:03 -0700840 }
841
842 err = radix_tree_preload(gfp_mask);
843 if (unlikely(err)) {
844 kfree(vb);
845 free_vmap_area(va);
846 return ERR_PTR(err);
847 }
848
Roman Pencf725ce2015-04-15 16:13:52 -0700849 vaddr = vmap_block_vaddr(va->va_start, 0);
Nick Piggindb64fe02008-10-18 20:27:03 -0700850 spin_lock_init(&vb->lock);
851 vb->va = va;
Roman Pencf725ce2015-04-15 16:13:52 -0700852 /* At least something should be left free */
853 BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
854 vb->free = VMAP_BBMAP_BITS - (1UL << order);
Nick Piggindb64fe02008-10-18 20:27:03 -0700855 vb->dirty = 0;
Roman Pen7d61bfe2015-04-15 16:13:55 -0700856 vb->dirty_min = VMAP_BBMAP_BITS;
857 vb->dirty_max = 0;
Nick Piggindb64fe02008-10-18 20:27:03 -0700858 INIT_LIST_HEAD(&vb->free_list);
Nick Piggindb64fe02008-10-18 20:27:03 -0700859
860 vb_idx = addr_to_vb_idx(va->va_start);
861 spin_lock(&vmap_block_tree_lock);
862 err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
863 spin_unlock(&vmap_block_tree_lock);
864 BUG_ON(err);
865 radix_tree_preload_end();
866
867 vbq = &get_cpu_var(vmap_block_queue);
Nick Piggindb64fe02008-10-18 20:27:03 -0700868 spin_lock(&vbq->lock);
Roman Pen68ac5462015-04-15 16:13:48 -0700869 list_add_tail_rcu(&vb->free_list, &vbq->free);
Nick Piggindb64fe02008-10-18 20:27:03 -0700870 spin_unlock(&vbq->lock);
Tejun Heo3f04ba82009-10-29 22:34:12 +0900871 put_cpu_var(vmap_block_queue);
Nick Piggindb64fe02008-10-18 20:27:03 -0700872
Roman Pencf725ce2015-04-15 16:13:52 -0700873 return vaddr;
Nick Piggindb64fe02008-10-18 20:27:03 -0700874}
875
Nick Piggindb64fe02008-10-18 20:27:03 -0700876static void free_vmap_block(struct vmap_block *vb)
877{
878 struct vmap_block *tmp;
879 unsigned long vb_idx;
880
Nick Piggindb64fe02008-10-18 20:27:03 -0700881 vb_idx = addr_to_vb_idx(vb->va->va_start);
882 spin_lock(&vmap_block_tree_lock);
883 tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
884 spin_unlock(&vmap_block_tree_lock);
885 BUG_ON(tmp != vb);
886
Jeremy Fitzhardinge64141da2010-12-02 14:31:18 -0800887 free_vmap_area_noflush(vb->va);
Lai Jiangshan22a3c7d2011-03-18 12:13:08 +0800888 kfree_rcu(vb, rcu_head);
Nick Piggindb64fe02008-10-18 20:27:03 -0700889}
890
Nick Piggin02b709d2010-02-01 22:25:57 +1100891static void purge_fragmented_blocks(int cpu)
892{
893 LIST_HEAD(purge);
894 struct vmap_block *vb;
895 struct vmap_block *n_vb;
896 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
897
898 rcu_read_lock();
899 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
900
901 if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
902 continue;
903
904 spin_lock(&vb->lock);
905 if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
906 vb->free = 0; /* prevent further allocs after releasing lock */
907 vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
Roman Pen7d61bfe2015-04-15 16:13:55 -0700908 vb->dirty_min = 0;
909 vb->dirty_max = VMAP_BBMAP_BITS;
Nick Piggin02b709d2010-02-01 22:25:57 +1100910 spin_lock(&vbq->lock);
911 list_del_rcu(&vb->free_list);
912 spin_unlock(&vbq->lock);
913 spin_unlock(&vb->lock);
914 list_add_tail(&vb->purge, &purge);
915 } else
916 spin_unlock(&vb->lock);
917 }
918 rcu_read_unlock();
919
920 list_for_each_entry_safe(vb, n_vb, &purge, purge) {
921 list_del(&vb->purge);
922 free_vmap_block(vb);
923 }
924}
925
Nick Piggin02b709d2010-02-01 22:25:57 +1100926static void purge_fragmented_blocks_allcpus(void)
927{
928 int cpu;
929
930 for_each_possible_cpu(cpu)
931 purge_fragmented_blocks(cpu);
932}
933
Nick Piggindb64fe02008-10-18 20:27:03 -0700934static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
935{
936 struct vmap_block_queue *vbq;
937 struct vmap_block *vb;
Roman Pencf725ce2015-04-15 16:13:52 -0700938 void *vaddr = NULL;
Nick Piggindb64fe02008-10-18 20:27:03 -0700939 unsigned int order;
940
Alexander Kuleshov891c49a2015-11-05 18:46:51 -0800941 BUG_ON(offset_in_page(size));
Nick Piggindb64fe02008-10-18 20:27:03 -0700942 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
Jan Karaaa91c4d2012-07-31 16:41:37 -0700943 if (WARN_ON(size == 0)) {
944 /*
945 * Allocating 0 bytes isn't what caller wants since
946 * get_order(0) returns funny result. Just warn and terminate
947 * early.
948 */
949 return NULL;
950 }
Nick Piggindb64fe02008-10-18 20:27:03 -0700951 order = get_order(size);
952
Nick Piggindb64fe02008-10-18 20:27:03 -0700953 rcu_read_lock();
954 vbq = &get_cpu_var(vmap_block_queue);
955 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
Roman Pencf725ce2015-04-15 16:13:52 -0700956 unsigned long pages_off;
Nick Piggindb64fe02008-10-18 20:27:03 -0700957
958 spin_lock(&vb->lock);
Roman Pencf725ce2015-04-15 16:13:52 -0700959 if (vb->free < (1UL << order)) {
960 spin_unlock(&vb->lock);
961 continue;
962 }
Nick Piggin02b709d2010-02-01 22:25:57 +1100963
Roman Pencf725ce2015-04-15 16:13:52 -0700964 pages_off = VMAP_BBMAP_BITS - vb->free;
965 vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
Nick Piggin02b709d2010-02-01 22:25:57 +1100966 vb->free -= 1UL << order;
967 if (vb->free == 0) {
968 spin_lock(&vbq->lock);
969 list_del_rcu(&vb->free_list);
970 spin_unlock(&vbq->lock);
Nick Piggindb64fe02008-10-18 20:27:03 -0700971 }
Roman Pencf725ce2015-04-15 16:13:52 -0700972
Nick Piggindb64fe02008-10-18 20:27:03 -0700973 spin_unlock(&vb->lock);
Nick Piggin02b709d2010-02-01 22:25:57 +1100974 break;
Nick Piggindb64fe02008-10-18 20:27:03 -0700975 }
Nick Piggin02b709d2010-02-01 22:25:57 +1100976
Tejun Heo3f04ba82009-10-29 22:34:12 +0900977 put_cpu_var(vmap_block_queue);
Nick Piggindb64fe02008-10-18 20:27:03 -0700978 rcu_read_unlock();
979
Roman Pencf725ce2015-04-15 16:13:52 -0700980 /* Allocate new block if nothing was found */
981 if (!vaddr)
982 vaddr = new_vmap_block(order, gfp_mask);
Nick Piggindb64fe02008-10-18 20:27:03 -0700983
Roman Pencf725ce2015-04-15 16:13:52 -0700984 return vaddr;
Nick Piggindb64fe02008-10-18 20:27:03 -0700985}
986
987static void vb_free(const void *addr, unsigned long size)
988{
989 unsigned long offset;
990 unsigned long vb_idx;
991 unsigned int order;
992 struct vmap_block *vb;
993
Alexander Kuleshov891c49a2015-11-05 18:46:51 -0800994 BUG_ON(offset_in_page(size));
Nick Piggindb64fe02008-10-18 20:27:03 -0700995 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
Nick Pigginb29acbd2008-12-01 13:13:47 -0800996
997 flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
998
Nick Piggindb64fe02008-10-18 20:27:03 -0700999 order = get_order(size);
1000
1001 offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
Roman Pen7d61bfe2015-04-15 16:13:55 -07001002 offset >>= PAGE_SHIFT;
Nick Piggindb64fe02008-10-18 20:27:03 -07001003
1004 vb_idx = addr_to_vb_idx((unsigned long)addr);
1005 rcu_read_lock();
1006 vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
1007 rcu_read_unlock();
1008 BUG_ON(!vb);
1009
Jeremy Fitzhardinge64141da2010-12-02 14:31:18 -08001010 vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
1011
Nick Piggindb64fe02008-10-18 20:27:03 -07001012 spin_lock(&vb->lock);
Roman Pen7d61bfe2015-04-15 16:13:55 -07001013
1014 /* Expand dirty range */
1015 vb->dirty_min = min(vb->dirty_min, offset);
1016 vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
MinChan Kimd0868172009-03-31 15:19:26 -07001017
Nick Piggindb64fe02008-10-18 20:27:03 -07001018 vb->dirty += 1UL << order;
1019 if (vb->dirty == VMAP_BBMAP_BITS) {
Nick Pigginde560422010-02-01 22:24:18 +11001020 BUG_ON(vb->free);
Nick Piggindb64fe02008-10-18 20:27:03 -07001021 spin_unlock(&vb->lock);
1022 free_vmap_block(vb);
1023 } else
1024 spin_unlock(&vb->lock);
1025}
1026
1027/**
1028 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
1029 *
1030 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
1031 * to amortize TLB flushing overheads. What this means is that any page you
1032 * have now, may, in a former life, have been mapped into kernel virtual
1033 * address by the vmap layer and so there might be some CPUs with TLB entries
1034 * still referencing that page (additional to the regular 1:1 kernel mapping).
1035 *
1036 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
1037 * be sure that none of the pages we have control over will have any aliases
1038 * from the vmap layer.
1039 */
1040void vm_unmap_aliases(void)
1041{
1042 unsigned long start = ULONG_MAX, end = 0;
1043 int cpu;
1044 int flush = 0;
1045
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +11001046 if (unlikely(!vmap_initialized))
1047 return;
1048
Nick Piggindb64fe02008-10-18 20:27:03 -07001049 for_each_possible_cpu(cpu) {
1050 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1051 struct vmap_block *vb;
1052
1053 rcu_read_lock();
1054 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
Nick Piggindb64fe02008-10-18 20:27:03 -07001055 spin_lock(&vb->lock);
Roman Pen7d61bfe2015-04-15 16:13:55 -07001056 if (vb->dirty) {
1057 unsigned long va_start = vb->va->va_start;
Nick Piggindb64fe02008-10-18 20:27:03 -07001058 unsigned long s, e;
Joonsoo Kimb136be5e2013-09-11 14:21:40 -07001059
Roman Pen7d61bfe2015-04-15 16:13:55 -07001060 s = va_start + (vb->dirty_min << PAGE_SHIFT);
1061 e = va_start + (vb->dirty_max << PAGE_SHIFT);
Nick Piggindb64fe02008-10-18 20:27:03 -07001062
Roman Pen7d61bfe2015-04-15 16:13:55 -07001063 start = min(s, start);
1064 end = max(e, end);
1065
Nick Piggindb64fe02008-10-18 20:27:03 -07001066 flush = 1;
Nick Piggindb64fe02008-10-18 20:27:03 -07001067 }
1068 spin_unlock(&vb->lock);
1069 }
1070 rcu_read_unlock();
1071 }
1072
1073 __purge_vmap_area_lazy(&start, &end, 1, flush);
1074}
1075EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1076
1077/**
1078 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
1079 * @mem: the pointer returned by vm_map_ram
1080 * @count: the count passed to that vm_map_ram call (cannot unmap partial)
1081 */
1082void vm_unmap_ram(const void *mem, unsigned int count)
1083{
1084 unsigned long size = count << PAGE_SHIFT;
1085 unsigned long addr = (unsigned long)mem;
1086
1087 BUG_ON(!addr);
1088 BUG_ON(addr < VMALLOC_START);
1089 BUG_ON(addr > VMALLOC_END);
1090 BUG_ON(addr & (PAGE_SIZE-1));
1091
1092 debug_check_no_locks_freed(mem, size);
Nick Piggincd528582009-01-06 14:39:20 -08001093 vmap_debug_free_range(addr, addr+size);
Nick Piggindb64fe02008-10-18 20:27:03 -07001094
1095 if (likely(count <= VMAP_MAX_ALLOC))
1096 vb_free(mem, size);
1097 else
1098 free_unmap_vmap_area_addr(addr);
1099}
1100EXPORT_SYMBOL(vm_unmap_ram);
1101
1102/**
1103 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
1104 * @pages: an array of pointers to the pages to be mapped
1105 * @count: number of pages
1106 * @node: prefer to allocate data structures on this node
1107 * @prot: memory protection to use. PAGE_KERNEL for regular RAM
Randy Dunlape99c97a2008-10-29 14:01:09 -07001108 *
Gioh Kim36437632014-04-07 15:37:37 -07001109 * If you use this function for less than VMAP_MAX_ALLOC pages, it could be
1110 * faster than vmap so it's good. But if you mix long-life and short-life
1111 * objects with vm_map_ram(), it could consume lots of address space through
1112 * fragmentation (especially on a 32bit machine). You could see failures in
1113 * the end. Please use this function for short-lived objects.
1114 *
Randy Dunlape99c97a2008-10-29 14:01:09 -07001115 * Returns: a pointer to the address that has been mapped, or %NULL on failure
Nick Piggindb64fe02008-10-18 20:27:03 -07001116 */
1117void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
1118{
1119 unsigned long size = count << PAGE_SHIFT;
1120 unsigned long addr;
1121 void *mem;
1122
1123 if (likely(count <= VMAP_MAX_ALLOC)) {
1124 mem = vb_alloc(size, GFP_KERNEL);
1125 if (IS_ERR(mem))
1126 return NULL;
1127 addr = (unsigned long)mem;
1128 } else {
1129 struct vmap_area *va;
1130 va = alloc_vmap_area(size, PAGE_SIZE,
1131 VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
1132 if (IS_ERR(va))
1133 return NULL;
1134
1135 addr = va->va_start;
1136 mem = (void *)addr;
1137 }
1138 if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
1139 vm_unmap_ram(mem, count);
1140 return NULL;
1141 }
1142 return mem;
1143}
1144EXPORT_SYMBOL(vm_map_ram);
1145
Joonsoo Kim4341fa42013-04-29 15:07:39 -07001146static struct vm_struct *vmlist __initdata;
Tejun Heof0aa6612009-02-20 16:29:08 +09001147/**
Nicolas Pitrebe9b7332011-08-25 00:24:21 -04001148 * vm_area_add_early - add vmap area early during boot
1149 * @vm: vm_struct to add
1150 *
1151 * This function is used to add fixed kernel vm area to vmlist before
1152 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
1153 * should contain proper values and the other fields should be zero.
1154 *
1155 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1156 */
1157void __init vm_area_add_early(struct vm_struct *vm)
1158{
1159 struct vm_struct *tmp, **p;
1160
1161 BUG_ON(vmap_initialized);
1162 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1163 if (tmp->addr >= vm->addr) {
1164 BUG_ON(tmp->addr < vm->addr + vm->size);
1165 break;
1166 } else
1167 BUG_ON(tmp->addr + tmp->size > vm->addr);
1168 }
1169 vm->next = *p;
1170 *p = vm;
1171}
1172
1173/**
Tejun Heof0aa6612009-02-20 16:29:08 +09001174 * vm_area_register_early - register vmap area early during boot
1175 * @vm: vm_struct to register
Tejun Heoc0c0a292009-02-24 11:57:21 +09001176 * @align: requested alignment
Tejun Heof0aa6612009-02-20 16:29:08 +09001177 *
1178 * This function is used to register kernel vm area before
1179 * vmalloc_init() is called. @vm->size and @vm->flags should contain
1180 * proper values on entry and other fields should be zero. On return,
1181 * vm->addr contains the allocated address.
1182 *
1183 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1184 */
Tejun Heoc0c0a292009-02-24 11:57:21 +09001185void __init vm_area_register_early(struct vm_struct *vm, size_t align)
Tejun Heof0aa6612009-02-20 16:29:08 +09001186{
1187 static size_t vm_init_off __initdata;
Tejun Heoc0c0a292009-02-24 11:57:21 +09001188 unsigned long addr;
Tejun Heof0aa6612009-02-20 16:29:08 +09001189
Tejun Heoc0c0a292009-02-24 11:57:21 +09001190 addr = ALIGN(VMALLOC_START + vm_init_off, align);
1191 vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1192
1193 vm->addr = (void *)addr;
Tejun Heof0aa6612009-02-20 16:29:08 +09001194
Nicolas Pitrebe9b7332011-08-25 00:24:21 -04001195 vm_area_add_early(vm);
Tejun Heof0aa6612009-02-20 16:29:08 +09001196}
1197
Nick Piggindb64fe02008-10-18 20:27:03 -07001198void __init vmalloc_init(void)
1199{
Ivan Kokshaysky822c18f2009-01-15 13:50:48 -08001200 struct vmap_area *va;
1201 struct vm_struct *tmp;
Nick Piggindb64fe02008-10-18 20:27:03 -07001202 int i;
1203
1204 for_each_possible_cpu(i) {
1205 struct vmap_block_queue *vbq;
Al Viro32fcfd42013-03-10 20:14:08 -04001206 struct vfree_deferred *p;
Nick Piggindb64fe02008-10-18 20:27:03 -07001207
1208 vbq = &per_cpu(vmap_block_queue, i);
1209 spin_lock_init(&vbq->lock);
1210 INIT_LIST_HEAD(&vbq->free);
Al Viro32fcfd42013-03-10 20:14:08 -04001211 p = &per_cpu(vfree_deferred, i);
1212 init_llist_head(&p->list);
1213 INIT_WORK(&p->wq, free_work);
Nick Piggindb64fe02008-10-18 20:27:03 -07001214 }
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +11001215
Ivan Kokshaysky822c18f2009-01-15 13:50:48 -08001216 /* Import existing vmlist entries. */
1217 for (tmp = vmlist; tmp; tmp = tmp->next) {
Pekka Enberg43ebdac2009-05-25 15:01:35 +03001218 va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
KyongHodbda5912012-05-29 15:06:49 -07001219 va->flags = VM_VM_AREA;
Ivan Kokshaysky822c18f2009-01-15 13:50:48 -08001220 va->va_start = (unsigned long)tmp->addr;
1221 va->va_end = va->va_start + tmp->size;
KyongHodbda5912012-05-29 15:06:49 -07001222 va->vm = tmp;
Ivan Kokshaysky822c18f2009-01-15 13:50:48 -08001223 __insert_vmap_area(va);
1224 }
Tejun Heoca23e402009-08-14 15:00:52 +09001225
1226 vmap_area_pcpu_hole = VMALLOC_END;
1227
Jeremy Fitzhardinge9b463332008-10-28 19:22:34 +11001228 vmap_initialized = true;
Nick Piggindb64fe02008-10-18 20:27:03 -07001229}
1230
Tejun Heo8fc48982009-02-20 16:29:08 +09001231/**
1232 * map_kernel_range_noflush - map kernel VM area with the specified pages
1233 * @addr: start of the VM area to map
1234 * @size: size of the VM area to map
1235 * @prot: page protection flags to use
1236 * @pages: pages to map
1237 *
1238 * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size
1239 * specify should have been allocated using get_vm_area() and its
1240 * friends.
1241 *
1242 * NOTE:
1243 * This function does NOT do any cache flushing. The caller is
1244 * responsible for calling flush_cache_vmap() on to-be-mapped areas
1245 * before calling this function.
1246 *
1247 * RETURNS:
1248 * The number of pages mapped on success, -errno on failure.
1249 */
1250int map_kernel_range_noflush(unsigned long addr, unsigned long size,
1251 pgprot_t prot, struct page **pages)
1252{
1253 return vmap_page_range_noflush(addr, addr + size, prot, pages);
1254}
1255
1256/**
1257 * unmap_kernel_range_noflush - unmap kernel VM area
1258 * @addr: start of the VM area to unmap
1259 * @size: size of the VM area to unmap
1260 *
1261 * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size
1262 * specify should have been allocated using get_vm_area() and its
1263 * friends.
1264 *
1265 * NOTE:
1266 * This function does NOT do any cache flushing. The caller is
1267 * responsible for calling flush_cache_vunmap() on to-be-mapped areas
1268 * before calling this function and flush_tlb_kernel_range() after.
1269 */
1270void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
1271{
1272 vunmap_page_range(addr, addr + size);
1273}
Huang Ying81e88fd2011-01-12 14:44:55 +08001274EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
Tejun Heo8fc48982009-02-20 16:29:08 +09001275
1276/**
1277 * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
1278 * @addr: start of the VM area to unmap
1279 * @size: size of the VM area to unmap
1280 *
1281 * Similar to unmap_kernel_range_noflush() but flushes vcache before
1282 * the unmapping and tlb after.
1283 */
Nick Piggindb64fe02008-10-18 20:27:03 -07001284void unmap_kernel_range(unsigned long addr, unsigned long size)
1285{
1286 unsigned long end = addr + size;
Tejun Heof6fcba72009-02-20 15:38:48 -08001287
1288 flush_cache_vunmap(addr, end);
Nick Piggindb64fe02008-10-18 20:27:03 -07001289 vunmap_page_range(addr, end);
1290 flush_tlb_kernel_range(addr, end);
1291}
Minchan Kim93ef6d62014-06-04 16:11:09 -07001292EXPORT_SYMBOL_GPL(unmap_kernel_range);
Nick Piggindb64fe02008-10-18 20:27:03 -07001293
WANG Chaof6f8ed42014-08-06 16:06:58 -07001294int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)
Nick Piggindb64fe02008-10-18 20:27:03 -07001295{
1296 unsigned long addr = (unsigned long)area->addr;
Wanpeng Li762216a2013-09-11 14:22:42 -07001297 unsigned long end = addr + get_vm_area_size(area);
Nick Piggindb64fe02008-10-18 20:27:03 -07001298 int err;
1299
WANG Chaof6f8ed42014-08-06 16:06:58 -07001300 err = vmap_page_range(addr, end, prot, pages);
Nick Piggindb64fe02008-10-18 20:27:03 -07001301
WANG Chaof6f8ed42014-08-06 16:06:58 -07001302 return err > 0 ? 0 : err;
Nick Piggindb64fe02008-10-18 20:27:03 -07001303}
1304EXPORT_SYMBOL_GPL(map_vm_area);
1305
Mitsuo Hayasakaf5252e02011-10-31 17:08:13 -07001306static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
Marek Szyprowski5e6cafc2012-04-13 12:32:09 +02001307 unsigned long flags, const void *caller)
Tejun Heocf88c792009-08-14 15:00:52 +09001308{
Joonsoo Kimc69480a2013-04-29 15:07:30 -07001309 spin_lock(&vmap_area_lock);
Tejun Heocf88c792009-08-14 15:00:52 +09001310 vm->flags = flags;
1311 vm->addr = (void *)va->va_start;
1312 vm->size = va->va_end - va->va_start;
1313 vm->caller = caller;
Minchan Kimdb1aeca2012-01-10 15:08:39 -08001314 va->vm = vm;
Tejun Heocf88c792009-08-14 15:00:52 +09001315 va->flags |= VM_VM_AREA;
Joonsoo Kimc69480a2013-04-29 15:07:30 -07001316 spin_unlock(&vmap_area_lock);
Mitsuo Hayasakaf5252e02011-10-31 17:08:13 -07001317}
Tejun Heocf88c792009-08-14 15:00:52 +09001318
Zhang Yanfei20fc02b2013-07-08 15:59:58 -07001319static void clear_vm_uninitialized_flag(struct vm_struct *vm)
Mitsuo Hayasakaf5252e02011-10-31 17:08:13 -07001320{
Joonsoo Kimd4033af2013-04-29 15:07:35 -07001321 /*
Zhang Yanfei20fc02b2013-07-08 15:59:58 -07001322 * Before removing VM_UNINITIALIZED,
Joonsoo Kimd4033af2013-04-29 15:07:35 -07001323 * we should make sure that vm has proper values.
1324 * Pair with smp_rmb() in show_numa_info().
1325 */
1326 smp_wmb();
Zhang Yanfei20fc02b2013-07-08 15:59:58 -07001327 vm->flags &= ~VM_UNINITIALIZED;
Tejun Heocf88c792009-08-14 15:00:52 +09001328}
1329
Nick Piggindb64fe02008-10-18 20:27:03 -07001330static struct vm_struct *__get_vm_area_node(unsigned long size,
David Miller2dca6992009-09-21 12:22:34 -07001331 unsigned long align, unsigned long flags, unsigned long start,
Marek Szyprowski5e6cafc2012-04-13 12:32:09 +02001332 unsigned long end, int node, gfp_t gfp_mask, const void *caller)
Nick Piggindb64fe02008-10-18 20:27:03 -07001333{
Kautuk Consul00065262011-12-19 17:12:04 -08001334 struct vmap_area *va;
Nick Piggindb64fe02008-10-18 20:27:03 -07001335 struct vm_struct *area;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336
Giridhar Pemmasani52fd24c2006-10-28 10:38:34 -07001337 BUG_ON(in_interrupt());
Zhang Yanfei0f2d4a82013-07-03 15:04:50 -07001338 if (flags & VM_IOREMAP)
Toshi Kani0f616be2015-04-14 15:47:17 -07001339 align = 1ul << clamp_t(int, fls_long(size),
1340 PAGE_SHIFT, IOREMAP_MAX_ORDER);
Nick Piggindb64fe02008-10-18 20:27:03 -07001341
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342 size = PAGE_ALIGN(size);
OGAWA Hirofumi31be8302006-11-16 01:19:29 -08001343 if (unlikely(!size))
1344 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345
Tejun Heocf88c792009-08-14 15:00:52 +09001346 area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 if (unlikely(!area))
1348 return NULL;
1349
Andrey Ryabinin71394fe2015-02-13 14:40:03 -08001350 if (!(flags & VM_NO_GUARD))
1351 size += PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352
Nick Piggindb64fe02008-10-18 20:27:03 -07001353 va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
1354 if (IS_ERR(va)) {
1355 kfree(area);
1356 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358
Zhang Yanfeid82b1d82013-07-03 15:04:47 -07001359 setup_vmalloc_vm(area, va, flags, caller);
Mitsuo Hayasakaf5252e02011-10-31 17:08:13 -07001360
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361 return area;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362}
1363
Christoph Lameter930fc452005-10-29 18:15:41 -07001364struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
1365 unsigned long start, unsigned long end)
1366{
David Rientjes00ef2d22013-02-22 16:35:36 -08001367 return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
1368 GFP_KERNEL, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -07001369}
Rusty Russell5992b6d2007-07-19 01:49:21 -07001370EXPORT_SYMBOL_GPL(__get_vm_area);
Christoph Lameter930fc452005-10-29 18:15:41 -07001371
Benjamin Herrenschmidtc2968612009-02-18 14:48:12 -08001372struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
1373 unsigned long start, unsigned long end,
Marek Szyprowski5e6cafc2012-04-13 12:32:09 +02001374 const void *caller)
Benjamin Herrenschmidtc2968612009-02-18 14:48:12 -08001375{
David Rientjes00ef2d22013-02-22 16:35:36 -08001376 return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
1377 GFP_KERNEL, caller);
Benjamin Herrenschmidtc2968612009-02-18 14:48:12 -08001378}
1379
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380/**
Simon Arlott183ff222007-10-20 01:27:18 +02001381 * get_vm_area - reserve a contiguous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 * @size: size of the area
1383 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
1384 *
1385 * Search an area of @size in the kernel virtual mapping area,
1386 * and reserved it for out purposes. Returns the area descriptor
1387 * on success or %NULL on failure.
1388 */
1389struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
1390{
David Miller2dca6992009-09-21 12:22:34 -07001391 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
David Rientjes00ef2d22013-02-22 16:35:36 -08001392 NUMA_NO_NODE, GFP_KERNEL,
1393 __builtin_return_address(0));
Christoph Lameter23016962008-04-28 02:12:42 -07001394}
1395
1396struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
Marek Szyprowski5e6cafc2012-04-13 12:32:09 +02001397 const void *caller)
Christoph Lameter23016962008-04-28 02:12:42 -07001398{
David Miller2dca6992009-09-21 12:22:34 -07001399 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
David Rientjes00ef2d22013-02-22 16:35:36 -08001400 NUMA_NO_NODE, GFP_KERNEL, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401}
1402
Marek Szyprowskie9da6e92012-07-30 09:11:33 +02001403/**
1404 * find_vm_area - find a continuous kernel virtual area
1405 * @addr: base address
1406 *
1407 * Search for the kernel VM area starting at @addr, and return it.
1408 * It is up to the caller to do all required locking to keep the returned
1409 * pointer valid.
1410 */
1411struct vm_struct *find_vm_area(const void *addr)
Nick Piggin83342312006-06-23 02:03:20 -07001412{
Nick Piggindb64fe02008-10-18 20:27:03 -07001413 struct vmap_area *va;
Nick Piggin83342312006-06-23 02:03:20 -07001414
Nick Piggindb64fe02008-10-18 20:27:03 -07001415 va = find_vmap_area((unsigned long)addr);
1416 if (va && va->flags & VM_VM_AREA)
Minchan Kimdb1aeca2012-01-10 15:08:39 -08001417 return va->vm;
Nick Piggin83342312006-06-23 02:03:20 -07001418
Andi Kleen7856dfe2005-05-20 14:27:57 -07001419 return NULL;
Andi Kleen7856dfe2005-05-20 14:27:57 -07001420}
1421
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422/**
Simon Arlott183ff222007-10-20 01:27:18 +02001423 * remove_vm_area - find and remove a continuous kernel virtual area
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424 * @addr: base address
1425 *
1426 * Search for the kernel VM area starting at @addr, and remove it.
1427 * This function returns the found VM area, but using it is NOT safe
Andi Kleen7856dfe2005-05-20 14:27:57 -07001428 * on SMP machines, except for its size or flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001430struct vm_struct *remove_vm_area(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431{
Nick Piggindb64fe02008-10-18 20:27:03 -07001432 struct vmap_area *va;
1433
1434 va = find_vmap_area((unsigned long)addr);
1435 if (va && va->flags & VM_VM_AREA) {
Minchan Kimdb1aeca2012-01-10 15:08:39 -08001436 struct vm_struct *vm = va->vm;
Mitsuo Hayasakaf5252e02011-10-31 17:08:13 -07001437
Joonsoo Kimc69480a2013-04-29 15:07:30 -07001438 spin_lock(&vmap_area_lock);
1439 va->vm = NULL;
1440 va->flags &= ~VM_VM_AREA;
1441 spin_unlock(&vmap_area_lock);
1442
KAMEZAWA Hiroyukidd32c272009-09-21 17:02:32 -07001443 vmap_debug_free_range(va->va_start, va->va_end);
Andrey Ryabinina5af5aa2015-03-12 16:26:11 -07001444 kasan_free_shadow(vm);
KAMEZAWA Hiroyukidd32c272009-09-21 17:02:32 -07001445 free_unmap_vmap_area(va);
KAMEZAWA Hiroyukidd32c272009-09-21 17:02:32 -07001446
Nick Piggindb64fe02008-10-18 20:27:03 -07001447 return vm;
1448 }
1449 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450}
1451
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001452static void __vunmap(const void *addr, int deallocate_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453{
1454 struct vm_struct *area;
1455
1456 if (!addr)
1457 return;
1458
HATAYAMA Daisukee69e9d4a2013-07-03 15:02:18 -07001459 if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
Dan Carpenterab15d9b2013-07-08 15:59:53 -07001460 addr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462
1463 area = remove_vm_area(addr);
1464 if (unlikely(!area)) {
Arjan van de Ven4c8573e2008-07-25 19:45:37 -07001465 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466 addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467 return;
1468 }
1469
Jerome Marchand7511c3e2015-11-20 15:57:02 -08001470 debug_check_no_locks_freed(addr, get_vm_area_size(area));
1471 debug_check_no_obj_freed(addr, get_vm_area_size(area));
Ingo Molnar9a11b49a2006-07-03 00:24:33 -07001472
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473 if (deallocate_pages) {
1474 int i;
1475
1476 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001477 struct page *page = area->pages[i];
1478
1479 BUG_ON(!page);
1480 __free_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481 }
1482
Jan Kiszka8757d5f2006-07-14 00:23:56 -07001483 if (area->flags & VM_VPAGES)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 vfree(area->pages);
1485 else
1486 kfree(area->pages);
1487 }
1488
1489 kfree(area);
1490 return;
1491}
Al Viro32fcfd42013-03-10 20:14:08 -04001492
Linus Torvalds1da177e2005-04-16 15:20:36 -07001493/**
1494 * vfree - release memory allocated by vmalloc()
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495 * @addr: memory base address
1496 *
Simon Arlott183ff222007-10-20 01:27:18 +02001497 * Free the virtually continuous memory area starting at @addr, as
Pekka Enberg80e93ef2005-09-09 13:10:16 -07001498 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
1499 * NULL, no operation is performed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 *
Al Viro32fcfd42013-03-10 20:14:08 -04001501 * Must not be called in NMI context (strictly speaking, only if we don't
1502 * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
1503 * conventions for vfree() arch-depenedent would be a really bad idea)
Andrew Mortonc9fcee52013-05-07 16:18:18 -07001504 *
1505 * NOTE: assumes that the object at *addr has a size >= sizeof(llist_node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001507void vfree(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508{
Al Viro32fcfd42013-03-10 20:14:08 -04001509 BUG_ON(in_nmi());
Catalin Marinas89219d32009-06-11 13:23:19 +01001510
1511 kmemleak_free(addr);
1512
Al Viro32fcfd42013-03-10 20:14:08 -04001513 if (!addr)
1514 return;
1515 if (unlikely(in_interrupt())) {
Christoph Lameter7c8e0182014-06-04 16:07:56 -07001516 struct vfree_deferred *p = this_cpu_ptr(&vfree_deferred);
Oleg Nesterov59d31322013-07-08 16:00:08 -07001517 if (llist_add((struct llist_node *)addr, &p->list))
1518 schedule_work(&p->wq);
Al Viro32fcfd42013-03-10 20:14:08 -04001519 } else
1520 __vunmap(addr, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522EXPORT_SYMBOL(vfree);
1523
1524/**
1525 * vunmap - release virtual mapping obtained by vmap()
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 * @addr: memory base address
1527 *
1528 * Free the virtually contiguous memory area starting at @addr,
1529 * which was created from the page array passed to vmap().
1530 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -07001531 * Must not be called in interrupt context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 */
Christoph Lameterb3bdda02008-02-04 22:28:32 -08001533void vunmap(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534{
1535 BUG_ON(in_interrupt());
Peter Zijlstra34754b62009-02-25 16:04:03 +01001536 might_sleep();
Al Viro32fcfd42013-03-10 20:14:08 -04001537 if (addr)
1538 __vunmap(addr, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540EXPORT_SYMBOL(vunmap);
1541
1542/**
1543 * vmap - map an array of pages into virtually contiguous space
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 * @pages: array of page pointers
1545 * @count: number of pages to map
1546 * @flags: vm_area->flags
1547 * @prot: page protection for the mapping
1548 *
1549 * Maps @count pages from @pages into contiguous kernel virtual
1550 * space.
1551 */
1552void *vmap(struct page **pages, unsigned int count,
1553 unsigned long flags, pgprot_t prot)
1554{
1555 struct vm_struct *area;
1556
Peter Zijlstra34754b62009-02-25 16:04:03 +01001557 might_sleep();
1558
Jan Beulich44813742009-09-21 17:03:05 -07001559 if (count > totalram_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560 return NULL;
1561
Christoph Lameter23016962008-04-28 02:12:42 -07001562 area = get_vm_area_caller((count << PAGE_SHIFT), flags,
1563 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564 if (!area)
1565 return NULL;
Christoph Lameter23016962008-04-28 02:12:42 -07001566
WANG Chaof6f8ed42014-08-06 16:06:58 -07001567 if (map_vm_area(area, prot, pages)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568 vunmap(area->addr);
1569 return NULL;
1570 }
1571
1572 return area->addr;
1573}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574EXPORT_SYMBOL(vmap);
1575
David Miller2dca6992009-09-21 12:22:34 -07001576static void *__vmalloc_node(unsigned long size, unsigned long align,
1577 gfp_t gfp_mask, pgprot_t prot,
Marek Szyprowski5e6cafc2012-04-13 12:32:09 +02001578 int node, const void *caller);
Adrian Bunke31d9eb2008-02-04 22:29:09 -08001579static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
Wanpeng Li3722e132013-11-12 15:07:29 -08001580 pgprot_t prot, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581{
Dave Hansen22943ab2011-05-24 17:12:18 -07001582 const int order = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 struct page **pages;
1584 unsigned int nr_pages, array_size, i;
David Rientjes930f0362014-08-06 16:06:28 -07001585 const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
1586 const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587
Wanpeng Li762216a2013-09-11 14:22:42 -07001588 nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589 array_size = (nr_pages * sizeof(struct page *));
1590
1591 area->nr_pages = nr_pages;
1592 /* Please note that the recursion is strictly bounded. */
Jan Kiszka8757d5f2006-07-14 00:23:56 -07001593 if (array_size > PAGE_SIZE) {
Jan Beulich976d6df2009-12-14 17:58:39 -08001594 pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
Wanpeng Li3722e132013-11-12 15:07:29 -08001595 PAGE_KERNEL, node, area->caller);
Jan Kiszka8757d5f2006-07-14 00:23:56 -07001596 area->flags |= VM_VPAGES;
Andrew Morton286e1ea2006-10-17 00:09:57 -07001597 } else {
Jan Beulich976d6df2009-12-14 17:58:39 -08001598 pages = kmalloc_node(array_size, nested_gfp, node);
Andrew Morton286e1ea2006-10-17 00:09:57 -07001599 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001600 area->pages = pages;
1601 if (!area->pages) {
1602 remove_vm_area(area->addr);
1603 kfree(area);
1604 return NULL;
1605 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606
1607 for (i = 0; i < area->nr_pages; i++) {
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001608 struct page *page;
1609
Jianguo Wu4b909512013-11-12 15:07:11 -08001610 if (node == NUMA_NO_NODE)
David Rientjes930f0362014-08-06 16:06:28 -07001611 page = alloc_page(alloc_mask);
Christoph Lameter930fc452005-10-29 18:15:41 -07001612 else
David Rientjes930f0362014-08-06 16:06:28 -07001613 page = alloc_pages_node(node, alloc_mask, order);
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001614
1615 if (unlikely(!page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616 /* Successfully allocated i pages, free them in __vunmap() */
1617 area->nr_pages = i;
1618 goto fail;
1619 }
Christoph Lameterbf53d6f2008-02-04 22:28:34 -08001620 area->pages[i] = page;
Mel Gormand0164ad2015-11-06 16:28:21 -08001621 if (gfpflags_allow_blocking(gfp_mask))
Eric Dumazet660654f2014-08-06 16:06:25 -07001622 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623 }
1624
WANG Chaof6f8ed42014-08-06 16:06:58 -07001625 if (map_vm_area(area, prot, pages))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626 goto fail;
1627 return area->addr;
1628
1629fail:
Joe Perches3ee9a4f2011-10-31 17:08:35 -07001630 warn_alloc_failed(gfp_mask, order,
1631 "vmalloc: allocation failure, allocated %ld of %ld bytes\n",
Dave Hansen22943ab2011-05-24 17:12:18 -07001632 (area->nr_pages*PAGE_SIZE), area->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 vfree(area->addr);
1634 return NULL;
1635}
1636
David Rientjesd0a21262011-01-13 15:46:02 -08001637/**
1638 * __vmalloc_node_range - allocate virtually contiguous memory
1639 * @size: allocation size
1640 * @align: desired alignment
1641 * @start: vm area range start
1642 * @end: vm area range end
1643 * @gfp_mask: flags for the page level allocator
1644 * @prot: protection mask for the allocated pages
Andrey Ryabinincb9e3c22015-02-13 14:40:07 -08001645 * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
David Rientjes00ef2d22013-02-22 16:35:36 -08001646 * @node: node to use for allocation or NUMA_NO_NODE
David Rientjesd0a21262011-01-13 15:46:02 -08001647 * @caller: caller's return address
1648 *
1649 * Allocate enough pages to cover @size from the page level
1650 * allocator with @gfp_mask flags. Map them into contiguous
1651 * kernel virtual space, using a pagetable protection of @prot.
1652 */
1653void *__vmalloc_node_range(unsigned long size, unsigned long align,
1654 unsigned long start, unsigned long end, gfp_t gfp_mask,
Andrey Ryabinincb9e3c22015-02-13 14:40:07 -08001655 pgprot_t prot, unsigned long vm_flags, int node,
1656 const void *caller)
Christoph Lameter930fc452005-10-29 18:15:41 -07001657{
David Rientjesd0a21262011-01-13 15:46:02 -08001658 struct vm_struct *area;
1659 void *addr;
1660 unsigned long real_size = size;
1661
1662 size = PAGE_ALIGN(size);
1663 if (!size || (size >> PAGE_SHIFT) > totalram_pages)
Joe Perchesde7d2b52011-10-31 17:08:48 -07001664 goto fail;
David Rientjesd0a21262011-01-13 15:46:02 -08001665
Andrey Ryabinincb9e3c22015-02-13 14:40:07 -08001666 area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |
1667 vm_flags, start, end, node, gfp_mask, caller);
David Rientjesd0a21262011-01-13 15:46:02 -08001668 if (!area)
Joe Perchesde7d2b52011-10-31 17:08:48 -07001669 goto fail;
David Rientjesd0a21262011-01-13 15:46:02 -08001670
Wanpeng Li3722e132013-11-12 15:07:29 -08001671 addr = __vmalloc_area_node(area, gfp_mask, prot, node);
Mel Gorman1368edf2011-12-08 14:34:30 -08001672 if (!addr)
Wanpeng Lib82225f32013-11-12 15:07:33 -08001673 return NULL;
Catalin Marinas89219d32009-06-11 13:23:19 +01001674
1675 /*
Zhang Yanfei20fc02b2013-07-08 15:59:58 -07001676 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
1677 * flag. It means that vm_struct is not fully initialized.
Joonsoo Kim4341fa42013-04-29 15:07:39 -07001678 * Now, it is fully initialized, so remove this flag here.
Mitsuo Hayasakaf5252e02011-10-31 17:08:13 -07001679 */
Zhang Yanfei20fc02b2013-07-08 15:59:58 -07001680 clear_vm_uninitialized_flag(area);
Mitsuo Hayasakaf5252e02011-10-31 17:08:13 -07001681
1682 /*
Catalin Marinas7f88f882013-11-12 15:07:45 -08001683 * A ref_count = 2 is needed because vm_struct allocated in
1684 * __get_vm_area_node() contains a reference to the virtual address of
1685 * the vmalloc'ed block.
Catalin Marinas89219d32009-06-11 13:23:19 +01001686 */
Catalin Marinas7f88f882013-11-12 15:07:45 -08001687 kmemleak_alloc(addr, real_size, 2, gfp_mask);
Catalin Marinas89219d32009-06-11 13:23:19 +01001688
1689 return addr;
Joe Perchesde7d2b52011-10-31 17:08:48 -07001690
1691fail:
1692 warn_alloc_failed(gfp_mask, 0,
1693 "vmalloc: allocation failure: %lu bytes\n",
1694 real_size);
1695 return NULL;
Christoph Lameter930fc452005-10-29 18:15:41 -07001696}
1697
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698/**
Christoph Lameter930fc452005-10-29 18:15:41 -07001699 * __vmalloc_node - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700 * @size: allocation size
David Miller2dca6992009-09-21 12:22:34 -07001701 * @align: desired alignment
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702 * @gfp_mask: flags for the page level allocator
1703 * @prot: protection mask for the allocated pages
David Rientjes00ef2d22013-02-22 16:35:36 -08001704 * @node: node to use for allocation or NUMA_NO_NODE
Randy Dunlapc85d1942008-05-01 04:34:48 -07001705 * @caller: caller's return address
Linus Torvalds1da177e2005-04-16 15:20:36 -07001706 *
1707 * Allocate enough pages to cover @size from the page level
1708 * allocator with @gfp_mask flags. Map them into contiguous
1709 * kernel virtual space, using a pagetable protection of @prot.
1710 */
David Miller2dca6992009-09-21 12:22:34 -07001711static void *__vmalloc_node(unsigned long size, unsigned long align,
1712 gfp_t gfp_mask, pgprot_t prot,
Marek Szyprowski5e6cafc2012-04-13 12:32:09 +02001713 int node, const void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714{
David Rientjesd0a21262011-01-13 15:46:02 -08001715 return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
Andrey Ryabinincb9e3c22015-02-13 14:40:07 -08001716 gfp_mask, prot, 0, node, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717}
1718
Christoph Lameter930fc452005-10-29 18:15:41 -07001719void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
1720{
David Rientjes00ef2d22013-02-22 16:35:36 -08001721 return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
Christoph Lameter23016962008-04-28 02:12:42 -07001722 __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -07001723}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001724EXPORT_SYMBOL(__vmalloc);
1725
Dave Younge1ca7782010-10-26 14:22:06 -07001726static inline void *__vmalloc_node_flags(unsigned long size,
1727 int node, gfp_t flags)
1728{
1729 return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
1730 node, __builtin_return_address(0));
1731}
1732
Linus Torvalds1da177e2005-04-16 15:20:36 -07001733/**
1734 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735 * @size: allocation size
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736 * Allocate enough pages to cover @size from the page level
1737 * allocator and map them into contiguous kernel virtual space.
1738 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +02001739 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 * use __vmalloc() instead.
1741 */
1742void *vmalloc(unsigned long size)
1743{
David Rientjes00ef2d22013-02-22 16:35:36 -08001744 return __vmalloc_node_flags(size, NUMA_NO_NODE,
1745 GFP_KERNEL | __GFP_HIGHMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001746}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001747EXPORT_SYMBOL(vmalloc);
1748
Christoph Lameter930fc452005-10-29 18:15:41 -07001749/**
Dave Younge1ca7782010-10-26 14:22:06 -07001750 * vzalloc - allocate virtually contiguous memory with zero fill
1751 * @size: allocation size
1752 * Allocate enough pages to cover @size from the page level
1753 * allocator and map them into contiguous kernel virtual space.
1754 * The memory allocated is set to zero.
1755 *
1756 * For tight control over page level allocator and protection flags
1757 * use __vmalloc() instead.
1758 */
1759void *vzalloc(unsigned long size)
1760{
David Rientjes00ef2d22013-02-22 16:35:36 -08001761 return __vmalloc_node_flags(size, NUMA_NO_NODE,
Dave Younge1ca7782010-10-26 14:22:06 -07001762 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1763}
1764EXPORT_SYMBOL(vzalloc);
1765
1766/**
Rolf Eike Beeread04082006-09-27 01:50:13 -07001767 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
1768 * @size: allocation size
Nick Piggin83342312006-06-23 02:03:20 -07001769 *
Rolf Eike Beeread04082006-09-27 01:50:13 -07001770 * The resulting memory area is zeroed so it can be mapped to userspace
1771 * without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -07001772 */
1773void *vmalloc_user(unsigned long size)
1774{
1775 struct vm_struct *area;
1776 void *ret;
1777
David Miller2dca6992009-09-21 12:22:34 -07001778 ret = __vmalloc_node(size, SHMLBA,
1779 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
David Rientjes00ef2d22013-02-22 16:35:36 -08001780 PAGE_KERNEL, NUMA_NO_NODE,
1781 __builtin_return_address(0));
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001782 if (ret) {
Nick Piggindb64fe02008-10-18 20:27:03 -07001783 area = find_vm_area(ret);
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001784 area->flags |= VM_USERMAP;
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001785 }
Nick Piggin83342312006-06-23 02:03:20 -07001786 return ret;
1787}
1788EXPORT_SYMBOL(vmalloc_user);
1789
1790/**
Christoph Lameter930fc452005-10-29 18:15:41 -07001791 * vmalloc_node - allocate memory on a specific node
Christoph Lameter930fc452005-10-29 18:15:41 -07001792 * @size: allocation size
Randy Dunlapd44e0782005-11-07 01:01:10 -08001793 * @node: numa node
Christoph Lameter930fc452005-10-29 18:15:41 -07001794 *
1795 * Allocate enough pages to cover @size from the page level
1796 * allocator and map them into contiguous kernel virtual space.
1797 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +02001798 * For tight control over page level allocator and protection flags
Christoph Lameter930fc452005-10-29 18:15:41 -07001799 * use __vmalloc() instead.
1800 */
1801void *vmalloc_node(unsigned long size, int node)
1802{
David Miller2dca6992009-09-21 12:22:34 -07001803 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
Christoph Lameter23016962008-04-28 02:12:42 -07001804 node, __builtin_return_address(0));
Christoph Lameter930fc452005-10-29 18:15:41 -07001805}
1806EXPORT_SYMBOL(vmalloc_node);
1807
Dave Younge1ca7782010-10-26 14:22:06 -07001808/**
1809 * vzalloc_node - allocate memory on a specific node with zero fill
1810 * @size: allocation size
1811 * @node: numa node
1812 *
1813 * Allocate enough pages to cover @size from the page level
1814 * allocator and map them into contiguous kernel virtual space.
1815 * The memory allocated is set to zero.
1816 *
1817 * For tight control over page level allocator and protection flags
1818 * use __vmalloc_node() instead.
1819 */
1820void *vzalloc_node(unsigned long size, int node)
1821{
1822 return __vmalloc_node_flags(size, node,
1823 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1824}
1825EXPORT_SYMBOL(vzalloc_node);
1826
Pavel Pisa4dc3b162005-05-01 08:59:25 -07001827#ifndef PAGE_KERNEL_EXEC
1828# define PAGE_KERNEL_EXEC PAGE_KERNEL
1829#endif
1830
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831/**
1832 * vmalloc_exec - allocate virtually contiguous, executable memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833 * @size: allocation size
1834 *
1835 * Kernel-internal function to allocate enough pages to cover @size
1836 * the page level allocator and map them into contiguous and
1837 * executable kernel virtual space.
1838 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +02001839 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840 * use __vmalloc() instead.
1841 */
1842
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843void *vmalloc_exec(unsigned long size)
1844{
David Miller2dca6992009-09-21 12:22:34 -07001845 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
David Rientjes00ef2d22013-02-22 16:35:36 -08001846 NUMA_NO_NODE, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847}
1848
Andi Kleen0d08e0d2007-05-02 19:27:12 +02001849#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -07001850#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +02001851#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
Benjamin Herrenschmidt7ac674f2007-07-19 01:49:10 -07001852#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
Andi Kleen0d08e0d2007-05-02 19:27:12 +02001853#else
1854#define GFP_VMALLOC32 GFP_KERNEL
1855#endif
1856
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857/**
1858 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859 * @size: allocation size
1860 *
1861 * Allocate enough 32bit PA addressable pages to cover @size from the
1862 * page level allocator and map them into contiguous kernel virtual space.
1863 */
1864void *vmalloc_32(unsigned long size)
1865{
David Miller2dca6992009-09-21 12:22:34 -07001866 return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
David Rientjes00ef2d22013-02-22 16:35:36 -08001867 NUMA_NO_NODE, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869EXPORT_SYMBOL(vmalloc_32);
1870
Nick Piggin83342312006-06-23 02:03:20 -07001871/**
Rolf Eike Beeread04082006-09-27 01:50:13 -07001872 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
Nick Piggin83342312006-06-23 02:03:20 -07001873 * @size: allocation size
Rolf Eike Beeread04082006-09-27 01:50:13 -07001874 *
1875 * The resulting memory area is 32bit addressable and zeroed so it can be
1876 * mapped to userspace without leaking data.
Nick Piggin83342312006-06-23 02:03:20 -07001877 */
1878void *vmalloc_32_user(unsigned long size)
1879{
1880 struct vm_struct *area;
1881 void *ret;
1882
David Miller2dca6992009-09-21 12:22:34 -07001883 ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
David Rientjes00ef2d22013-02-22 16:35:36 -08001884 NUMA_NO_NODE, __builtin_return_address(0));
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001885 if (ret) {
Nick Piggindb64fe02008-10-18 20:27:03 -07001886 area = find_vm_area(ret);
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001887 area->flags |= VM_USERMAP;
Eric Dumazet2b4ac442006-11-10 12:27:48 -08001888 }
Nick Piggin83342312006-06-23 02:03:20 -07001889 return ret;
1890}
1891EXPORT_SYMBOL(vmalloc_32_user);
1892
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001893/*
1894 * small helper routine , copy contents to buf from addr.
1895 * If the page is not present, fill zero.
1896 */
1897
1898static int aligned_vread(char *buf, char *addr, unsigned long count)
1899{
1900 struct page *p;
1901 int copied = 0;
1902
1903 while (count) {
1904 unsigned long offset, length;
1905
Alexander Kuleshov891c49a2015-11-05 18:46:51 -08001906 offset = offset_in_page(addr);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001907 length = PAGE_SIZE - offset;
1908 if (length > count)
1909 length = count;
1910 p = vmalloc_to_page(addr);
1911 /*
1912 * To do safe access to this _mapped_ area, we need
1913 * lock. But adding lock here means that we need to add
1914 * overhead of vmalloc()/vfree() calles for this _debug_
1915 * interface, rarely used. Instead of that, we'll use
1916 * kmap() and get small overhead in this access function.
1917 */
1918 if (p) {
1919 /*
1920 * we can expect USER0 is not used (see vread/vwrite's
1921 * function description)
1922 */
Cong Wang9b04c5f2011-11-25 23:14:39 +08001923 void *map = kmap_atomic(p);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001924 memcpy(buf, map + offset, length);
Cong Wang9b04c5f2011-11-25 23:14:39 +08001925 kunmap_atomic(map);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001926 } else
1927 memset(buf, 0, length);
1928
1929 addr += length;
1930 buf += length;
1931 copied += length;
1932 count -= length;
1933 }
1934 return copied;
1935}
1936
1937static int aligned_vwrite(char *buf, char *addr, unsigned long count)
1938{
1939 struct page *p;
1940 int copied = 0;
1941
1942 while (count) {
1943 unsigned long offset, length;
1944
Alexander Kuleshov891c49a2015-11-05 18:46:51 -08001945 offset = offset_in_page(addr);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001946 length = PAGE_SIZE - offset;
1947 if (length > count)
1948 length = count;
1949 p = vmalloc_to_page(addr);
1950 /*
1951 * To do safe access to this _mapped_ area, we need
1952 * lock. But adding lock here means that we need to add
1953 * overhead of vmalloc()/vfree() calles for this _debug_
1954 * interface, rarely used. Instead of that, we'll use
1955 * kmap() and get small overhead in this access function.
1956 */
1957 if (p) {
1958 /*
1959 * we can expect USER0 is not used (see vread/vwrite's
1960 * function description)
1961 */
Cong Wang9b04c5f2011-11-25 23:14:39 +08001962 void *map = kmap_atomic(p);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001963 memcpy(map + offset, buf, length);
Cong Wang9b04c5f2011-11-25 23:14:39 +08001964 kunmap_atomic(map);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001965 }
1966 addr += length;
1967 buf += length;
1968 copied += length;
1969 count -= length;
1970 }
1971 return copied;
1972}
1973
1974/**
1975 * vread() - read vmalloc area in a safe way.
1976 * @buf: buffer for reading data
1977 * @addr: vm address.
1978 * @count: number of bytes to be read.
1979 *
1980 * Returns # of bytes which addr and buf should be increased.
1981 * (same number to @count). Returns 0 if [addr...addr+count) doesn't
1982 * includes any intersect with alive vmalloc area.
1983 *
1984 * This function checks that addr is a valid vmalloc'ed area, and
1985 * copy data from that area to a given buffer. If the given memory range
1986 * of [addr...addr+count) includes some valid address, data is copied to
1987 * proper area of @buf. If there are memory holes, they'll be zero-filled.
1988 * IOREMAP area is treated as memory hole and no copy is done.
1989 *
1990 * If [addr...addr+count) doesn't includes any intersects with alive
Cong Wanga8e52022012-06-23 11:30:16 +08001991 * vm_struct area, returns 0. @buf should be kernel's buffer.
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07001992 *
1993 * Note: In usual ops, vread() is never necessary because the caller
1994 * should know vmalloc() area is valid and can use memcpy().
1995 * This is for routines which have to access vmalloc area without
1996 * any informaion, as /dev/kmem.
1997 *
1998 */
1999
Linus Torvalds1da177e2005-04-16 15:20:36 -07002000long vread(char *buf, char *addr, unsigned long count)
2001{
Joonsoo Kime81ce852013-04-29 15:07:32 -07002002 struct vmap_area *va;
2003 struct vm_struct *vm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002004 char *vaddr, *buf_start = buf;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002005 unsigned long buflen = count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002006 unsigned long n;
2007
2008 /* Don't allow overflow */
2009 if ((unsigned long) addr + count < count)
2010 count = -(unsigned long) addr;
2011
Joonsoo Kime81ce852013-04-29 15:07:32 -07002012 spin_lock(&vmap_area_lock);
2013 list_for_each_entry(va, &vmap_area_list, list) {
2014 if (!count)
2015 break;
2016
2017 if (!(va->flags & VM_VM_AREA))
2018 continue;
2019
2020 vm = va->vm;
2021 vaddr = (char *) vm->addr;
Wanpeng Li762216a2013-09-11 14:22:42 -07002022 if (addr >= vaddr + get_vm_area_size(vm))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023 continue;
2024 while (addr < vaddr) {
2025 if (count == 0)
2026 goto finished;
2027 *buf = '\0';
2028 buf++;
2029 addr++;
2030 count--;
2031 }
Wanpeng Li762216a2013-09-11 14:22:42 -07002032 n = vaddr + get_vm_area_size(vm) - addr;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002033 if (n > count)
2034 n = count;
Joonsoo Kime81ce852013-04-29 15:07:32 -07002035 if (!(vm->flags & VM_IOREMAP))
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002036 aligned_vread(buf, addr, n);
2037 else /* IOREMAP area is treated as memory hole */
2038 memset(buf, 0, n);
2039 buf += n;
2040 addr += n;
2041 count -= n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042 }
2043finished:
Joonsoo Kime81ce852013-04-29 15:07:32 -07002044 spin_unlock(&vmap_area_lock);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002045
2046 if (buf == buf_start)
2047 return 0;
2048 /* zero-fill memory holes */
2049 if (buf != buf_start + buflen)
2050 memset(buf, 0, buflen - (buf - buf_start));
2051
2052 return buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053}
2054
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002055/**
2056 * vwrite() - write vmalloc area in a safe way.
2057 * @buf: buffer for source data
2058 * @addr: vm address.
2059 * @count: number of bytes to be read.
2060 *
2061 * Returns # of bytes which addr and buf should be incresed.
2062 * (same number to @count).
2063 * If [addr...addr+count) doesn't includes any intersect with valid
2064 * vmalloc area, returns 0.
2065 *
2066 * This function checks that addr is a valid vmalloc'ed area, and
2067 * copy data from a buffer to the given addr. If specified range of
2068 * [addr...addr+count) includes some valid address, data is copied from
2069 * proper area of @buf. If there are memory holes, no copy to hole.
2070 * IOREMAP area is treated as memory hole and no copy is done.
2071 *
2072 * If [addr...addr+count) doesn't includes any intersects with alive
Cong Wanga8e52022012-06-23 11:30:16 +08002073 * vm_struct area, returns 0. @buf should be kernel's buffer.
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002074 *
2075 * Note: In usual ops, vwrite() is never necessary because the caller
2076 * should know vmalloc() area is valid and can use memcpy().
2077 * This is for routines which have to access vmalloc area without
2078 * any informaion, as /dev/kmem.
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002079 */
2080
Linus Torvalds1da177e2005-04-16 15:20:36 -07002081long vwrite(char *buf, char *addr, unsigned long count)
2082{
Joonsoo Kime81ce852013-04-29 15:07:32 -07002083 struct vmap_area *va;
2084 struct vm_struct *vm;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002085 char *vaddr;
2086 unsigned long n, buflen;
2087 int copied = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088
2089 /* Don't allow overflow */
2090 if ((unsigned long) addr + count < count)
2091 count = -(unsigned long) addr;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002092 buflen = count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002093
Joonsoo Kime81ce852013-04-29 15:07:32 -07002094 spin_lock(&vmap_area_lock);
2095 list_for_each_entry(va, &vmap_area_list, list) {
2096 if (!count)
2097 break;
2098
2099 if (!(va->flags & VM_VM_AREA))
2100 continue;
2101
2102 vm = va->vm;
2103 vaddr = (char *) vm->addr;
Wanpeng Li762216a2013-09-11 14:22:42 -07002104 if (addr >= vaddr + get_vm_area_size(vm))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002105 continue;
2106 while (addr < vaddr) {
2107 if (count == 0)
2108 goto finished;
2109 buf++;
2110 addr++;
2111 count--;
2112 }
Wanpeng Li762216a2013-09-11 14:22:42 -07002113 n = vaddr + get_vm_area_size(vm) - addr;
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002114 if (n > count)
2115 n = count;
Joonsoo Kime81ce852013-04-29 15:07:32 -07002116 if (!(vm->flags & VM_IOREMAP)) {
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002117 aligned_vwrite(buf, addr, n);
2118 copied++;
2119 }
2120 buf += n;
2121 addr += n;
2122 count -= n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123 }
2124finished:
Joonsoo Kime81ce852013-04-29 15:07:32 -07002125 spin_unlock(&vmap_area_lock);
KAMEZAWA Hiroyukid0107eb2009-09-21 17:02:34 -07002126 if (!copied)
2127 return 0;
2128 return buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002129}
Nick Piggin83342312006-06-23 02:03:20 -07002130
2131/**
HATAYAMA Daisukee69e9d4a2013-07-03 15:02:18 -07002132 * remap_vmalloc_range_partial - map vmalloc pages to userspace
2133 * @vma: vma to cover
2134 * @uaddr: target user address to start at
2135 * @kaddr: virtual address of vmalloc kernel memory
2136 * @size: size of map area
2137 *
2138 * Returns: 0 for success, -Exxx on failure
2139 *
2140 * This function checks that @kaddr is a valid vmalloc'ed area,
2141 * and that it is big enough to cover the range starting at
2142 * @uaddr in @vma. Will return failure if that criteria isn't
2143 * met.
2144 *
2145 * Similar to remap_pfn_range() (see mm/memory.c)
2146 */
2147int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
2148 void *kaddr, unsigned long size)
2149{
2150 struct vm_struct *area;
2151
2152 size = PAGE_ALIGN(size);
2153
2154 if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
2155 return -EINVAL;
2156
2157 area = find_vm_area(kaddr);
2158 if (!area)
2159 return -EINVAL;
2160
2161 if (!(area->flags & VM_USERMAP))
2162 return -EINVAL;
2163
2164 if (kaddr + size > area->addr + area->size)
2165 return -EINVAL;
2166
2167 do {
2168 struct page *page = vmalloc_to_page(kaddr);
2169 int ret;
2170
2171 ret = vm_insert_page(vma, uaddr, page);
2172 if (ret)
2173 return ret;
2174
2175 uaddr += PAGE_SIZE;
2176 kaddr += PAGE_SIZE;
2177 size -= PAGE_SIZE;
2178 } while (size > 0);
2179
2180 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
2181
2182 return 0;
2183}
2184EXPORT_SYMBOL(remap_vmalloc_range_partial);
2185
2186/**
Nick Piggin83342312006-06-23 02:03:20 -07002187 * remap_vmalloc_range - map vmalloc pages to userspace
Nick Piggin83342312006-06-23 02:03:20 -07002188 * @vma: vma to cover (map full range of vma)
2189 * @addr: vmalloc memory
2190 * @pgoff: number of pages into addr before first page to map
Randy Dunlap76824862008-03-19 17:00:40 -07002191 *
2192 * Returns: 0 for success, -Exxx on failure
Nick Piggin83342312006-06-23 02:03:20 -07002193 *
2194 * This function checks that addr is a valid vmalloc'ed area, and
2195 * that it is big enough to cover the vma. Will return failure if
2196 * that criteria isn't met.
2197 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -08002198 * Similar to remap_pfn_range() (see mm/memory.c)
Nick Piggin83342312006-06-23 02:03:20 -07002199 */
2200int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
2201 unsigned long pgoff)
2202{
HATAYAMA Daisukee69e9d4a2013-07-03 15:02:18 -07002203 return remap_vmalloc_range_partial(vma, vma->vm_start,
2204 addr + (pgoff << PAGE_SHIFT),
2205 vma->vm_end - vma->vm_start);
Nick Piggin83342312006-06-23 02:03:20 -07002206}
2207EXPORT_SYMBOL(remap_vmalloc_range);
2208
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -07002209/*
2210 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
2211 * have one.
2212 */
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -07002213void __weak vmalloc_sync_all(void)
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -07002214{
2215}
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002216
2217
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08002218static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002219{
David Vrabelcd129092011-09-29 16:53:32 +01002220 pte_t ***p = data;
2221
2222 if (p) {
2223 *(*p) = pte;
2224 (*p)++;
2225 }
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002226 return 0;
2227}
2228
2229/**
2230 * alloc_vm_area - allocate a range of kernel address space
2231 * @size: size of the area
David Vrabelcd129092011-09-29 16:53:32 +01002232 * @ptes: returns the PTEs for the address space
Randy Dunlap76824862008-03-19 17:00:40 -07002233 *
2234 * Returns: NULL on failure, vm_struct on success
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002235 *
2236 * This function reserves a range of kernel address space, and
2237 * allocates pagetables to map that range. No actual mappings
David Vrabelcd129092011-09-29 16:53:32 +01002238 * are created.
2239 *
2240 * If @ptes is non-NULL, pointers to the PTEs (in init_mm)
2241 * allocated for the VM area are returned.
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002242 */
David Vrabelcd129092011-09-29 16:53:32 +01002243struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002244{
2245 struct vm_struct *area;
2246
Christoph Lameter23016962008-04-28 02:12:42 -07002247 area = get_vm_area_caller(size, VM_IOREMAP,
2248 __builtin_return_address(0));
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002249 if (area == NULL)
2250 return NULL;
2251
2252 /*
2253 * This ensures that page tables are constructed for this region
2254 * of kernel virtual address space and mapped into init_mm.
2255 */
2256 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
David Vrabelcd129092011-09-29 16:53:32 +01002257 size, f, ptes ? &ptes : NULL)) {
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002258 free_vm_area(area);
2259 return NULL;
2260 }
2261
Jeremy Fitzhardinge5f4352f2007-07-17 18:37:04 -07002262 return area;
2263}
2264EXPORT_SYMBOL_GPL(alloc_vm_area);
2265
2266void free_vm_area(struct vm_struct *area)
2267{
2268 struct vm_struct *ret;
2269 ret = remove_vm_area(area->addr);
2270 BUG_ON(ret != area);
2271 kfree(area);
2272}
2273EXPORT_SYMBOL_GPL(free_vm_area);
Christoph Lametera10aa572008-04-28 02:12:40 -07002274
Tejun Heo4f8b02b2010-09-03 18:22:47 +02002275#ifdef CONFIG_SMP
Tejun Heoca23e402009-08-14 15:00:52 +09002276static struct vmap_area *node_to_va(struct rb_node *n)
2277{
2278 return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
2279}
2280
2281/**
2282 * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
2283 * @end: target address
2284 * @pnext: out arg for the next vmap_area
2285 * @pprev: out arg for the previous vmap_area
2286 *
2287 * Returns: %true if either or both of next and prev are found,
2288 * %false if no vmap_area exists
2289 *
2290 * Find vmap_areas end addresses of which enclose @end. ie. if not
2291 * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
2292 */
2293static bool pvm_find_next_prev(unsigned long end,
2294 struct vmap_area **pnext,
2295 struct vmap_area **pprev)
2296{
2297 struct rb_node *n = vmap_area_root.rb_node;
2298 struct vmap_area *va = NULL;
2299
2300 while (n) {
2301 va = rb_entry(n, struct vmap_area, rb_node);
2302 if (end < va->va_end)
2303 n = n->rb_left;
2304 else if (end > va->va_end)
2305 n = n->rb_right;
2306 else
2307 break;
2308 }
2309
2310 if (!va)
2311 return false;
2312
2313 if (va->va_end > end) {
2314 *pnext = va;
2315 *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2316 } else {
2317 *pprev = va;
2318 *pnext = node_to_va(rb_next(&(*pprev)->rb_node));
2319 }
2320 return true;
2321}
2322
2323/**
2324 * pvm_determine_end - find the highest aligned address between two vmap_areas
2325 * @pnext: in/out arg for the next vmap_area
2326 * @pprev: in/out arg for the previous vmap_area
2327 * @align: alignment
2328 *
2329 * Returns: determined end address
2330 *
2331 * Find the highest aligned address between *@pnext and *@pprev below
2332 * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned
2333 * down address is between the end addresses of the two vmap_areas.
2334 *
2335 * Please note that the address returned by this function may fall
2336 * inside *@pnext vmap_area. The caller is responsible for checking
2337 * that.
2338 */
2339static unsigned long pvm_determine_end(struct vmap_area **pnext,
2340 struct vmap_area **pprev,
2341 unsigned long align)
2342{
2343 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2344 unsigned long addr;
2345
2346 if (*pnext)
2347 addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
2348 else
2349 addr = vmalloc_end;
2350
2351 while (*pprev && (*pprev)->va_end > addr) {
2352 *pnext = *pprev;
2353 *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2354 }
2355
2356 return addr;
2357}
2358
2359/**
2360 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
2361 * @offsets: array containing offset of each area
2362 * @sizes: array containing size of each area
2363 * @nr_vms: the number of areas to allocate
2364 * @align: alignment, all entries in @offsets and @sizes must be aligned to this
Tejun Heoca23e402009-08-14 15:00:52 +09002365 *
2366 * Returns: kmalloc'd vm_struct pointer array pointing to allocated
2367 * vm_structs on success, %NULL on failure
2368 *
2369 * Percpu allocator wants to use congruent vm areas so that it can
2370 * maintain the offsets among percpu areas. This function allocates
David Rientjesec3f64f2011-01-13 15:46:01 -08002371 * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
2372 * be scattered pretty far, distance between two areas easily going up
2373 * to gigabytes. To avoid interacting with regular vmallocs, these
2374 * areas are allocated from top.
Tejun Heoca23e402009-08-14 15:00:52 +09002375 *
2376 * Despite its complicated look, this allocator is rather simple. It
2377 * does everything top-down and scans areas from the end looking for
2378 * matching slot. While scanning, if any of the areas overlaps with
2379 * existing vmap_area, the base address is pulled down to fit the
2380 * area. Scanning is repeated till all the areas fit and then all
2381 * necessary data structres are inserted and the result is returned.
2382 */
2383struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
2384 const size_t *sizes, int nr_vms,
David Rientjesec3f64f2011-01-13 15:46:01 -08002385 size_t align)
Tejun Heoca23e402009-08-14 15:00:52 +09002386{
2387 const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
2388 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2389 struct vmap_area **vas, *prev, *next;
2390 struct vm_struct **vms;
2391 int area, area2, last_area, term_area;
2392 unsigned long base, start, end, last_end;
2393 bool purged = false;
2394
Tejun Heoca23e402009-08-14 15:00:52 +09002395 /* verify parameters and allocate data structures */
Alexander Kuleshov891c49a2015-11-05 18:46:51 -08002396 BUG_ON(offset_in_page(align) || !is_power_of_2(align));
Tejun Heoca23e402009-08-14 15:00:52 +09002397 for (last_area = 0, area = 0; area < nr_vms; area++) {
2398 start = offsets[area];
2399 end = start + sizes[area];
2400
2401 /* is everything aligned properly? */
2402 BUG_ON(!IS_ALIGNED(offsets[area], align));
2403 BUG_ON(!IS_ALIGNED(sizes[area], align));
2404
2405 /* detect the area with the highest address */
2406 if (start > offsets[last_area])
2407 last_area = area;
2408
2409 for (area2 = 0; area2 < nr_vms; area2++) {
2410 unsigned long start2 = offsets[area2];
2411 unsigned long end2 = start2 + sizes[area2];
2412
2413 if (area2 == area)
2414 continue;
2415
2416 BUG_ON(start2 >= start && start2 < end);
2417 BUG_ON(end2 <= end && end2 > start);
2418 }
2419 }
2420 last_end = offsets[last_area] + sizes[last_area];
2421
2422 if (vmalloc_end - vmalloc_start < last_end) {
2423 WARN_ON(true);
2424 return NULL;
2425 }
2426
Thomas Meyer4d67d862012-05-29 15:06:21 -07002427 vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
2428 vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
Tejun Heoca23e402009-08-14 15:00:52 +09002429 if (!vas || !vms)
Kautuk Consulf1db7af2012-01-12 17:20:08 -08002430 goto err_free2;
Tejun Heoca23e402009-08-14 15:00:52 +09002431
2432 for (area = 0; area < nr_vms; area++) {
David Rientjesec3f64f2011-01-13 15:46:01 -08002433 vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);
2434 vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
Tejun Heoca23e402009-08-14 15:00:52 +09002435 if (!vas[area] || !vms[area])
2436 goto err_free;
2437 }
2438retry:
2439 spin_lock(&vmap_area_lock);
2440
2441 /* start scanning - we scan from the top, begin with the last area */
2442 area = term_area = last_area;
2443 start = offsets[area];
2444 end = start + sizes[area];
2445
2446 if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
2447 base = vmalloc_end - last_end;
2448 goto found;
2449 }
2450 base = pvm_determine_end(&next, &prev, align) - end;
2451
2452 while (true) {
2453 BUG_ON(next && next->va_end <= base + end);
2454 BUG_ON(prev && prev->va_end > base + end);
2455
2456 /*
2457 * base might have underflowed, add last_end before
2458 * comparing.
2459 */
2460 if (base + last_end < vmalloc_start + last_end) {
2461 spin_unlock(&vmap_area_lock);
2462 if (!purged) {
2463 purge_vmap_area_lazy();
2464 purged = true;
2465 goto retry;
2466 }
2467 goto err_free;
2468 }
2469
2470 /*
2471 * If next overlaps, move base downwards so that it's
2472 * right below next and then recheck.
2473 */
2474 if (next && next->va_start < base + end) {
2475 base = pvm_determine_end(&next, &prev, align) - end;
2476 term_area = area;
2477 continue;
2478 }
2479
2480 /*
2481 * If prev overlaps, shift down next and prev and move
2482 * base so that it's right below new next and then
2483 * recheck.
2484 */
2485 if (prev && prev->va_end > base + start) {
2486 next = prev;
2487 prev = node_to_va(rb_prev(&next->rb_node));
2488 base = pvm_determine_end(&next, &prev, align) - end;
2489 term_area = area;
2490 continue;
2491 }
2492
2493 /*
2494 * This area fits, move on to the previous one. If
2495 * the previous one is the terminal one, we're done.
2496 */
2497 area = (area + nr_vms - 1) % nr_vms;
2498 if (area == term_area)
2499 break;
2500 start = offsets[area];
2501 end = start + sizes[area];
2502 pvm_find_next_prev(base + end, &next, &prev);
2503 }
2504found:
2505 /* we've found a fitting base, insert all va's */
2506 for (area = 0; area < nr_vms; area++) {
2507 struct vmap_area *va = vas[area];
2508
2509 va->va_start = base + offsets[area];
2510 va->va_end = va->va_start + sizes[area];
2511 __insert_vmap_area(va);
2512 }
2513
2514 vmap_area_pcpu_hole = base + offsets[last_area];
2515
2516 spin_unlock(&vmap_area_lock);
2517
2518 /* insert all vm's */
2519 for (area = 0; area < nr_vms; area++)
Zhang Yanfei3645cb42013-07-03 15:04:48 -07002520 setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
2521 pcpu_get_vm_areas);
Tejun Heoca23e402009-08-14 15:00:52 +09002522
2523 kfree(vas);
2524 return vms;
2525
2526err_free:
2527 for (area = 0; area < nr_vms; area++) {
Kautuk Consulf1db7af2012-01-12 17:20:08 -08002528 kfree(vas[area]);
2529 kfree(vms[area]);
Tejun Heoca23e402009-08-14 15:00:52 +09002530 }
Kautuk Consulf1db7af2012-01-12 17:20:08 -08002531err_free2:
Tejun Heoca23e402009-08-14 15:00:52 +09002532 kfree(vas);
2533 kfree(vms);
2534 return NULL;
2535}
2536
2537/**
2538 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
2539 * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
2540 * @nr_vms: the number of allocated areas
2541 *
2542 * Free vm_structs and the array allocated by pcpu_get_vm_areas().
2543 */
2544void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
2545{
2546 int i;
2547
2548 for (i = 0; i < nr_vms; i++)
2549 free_vm_area(vms[i]);
2550 kfree(vms);
2551}
Tejun Heo4f8b02b2010-09-03 18:22:47 +02002552#endif /* CONFIG_SMP */
Christoph Lametera10aa572008-04-28 02:12:40 -07002553
2554#ifdef CONFIG_PROC_FS
2555static void *s_start(struct seq_file *m, loff_t *pos)
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002556 __acquires(&vmap_area_lock)
Christoph Lametera10aa572008-04-28 02:12:40 -07002557{
2558 loff_t n = *pos;
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002559 struct vmap_area *va;
Christoph Lametera10aa572008-04-28 02:12:40 -07002560
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002561 spin_lock(&vmap_area_lock);
2562 va = list_entry((&vmap_area_list)->next, typeof(*va), list);
2563 while (n > 0 && &va->list != &vmap_area_list) {
Christoph Lametera10aa572008-04-28 02:12:40 -07002564 n--;
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002565 va = list_entry(va->list.next, typeof(*va), list);
Christoph Lametera10aa572008-04-28 02:12:40 -07002566 }
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002567 if (!n && &va->list != &vmap_area_list)
2568 return va;
Christoph Lametera10aa572008-04-28 02:12:40 -07002569
2570 return NULL;
2571
2572}
2573
2574static void *s_next(struct seq_file *m, void *p, loff_t *pos)
2575{
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002576 struct vmap_area *va = p, *next;
Christoph Lametera10aa572008-04-28 02:12:40 -07002577
2578 ++*pos;
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002579 next = list_entry(va->list.next, typeof(*va), list);
2580 if (&next->list != &vmap_area_list)
2581 return next;
2582
2583 return NULL;
Christoph Lametera10aa572008-04-28 02:12:40 -07002584}
2585
2586static void s_stop(struct seq_file *m, void *p)
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002587 __releases(&vmap_area_lock)
Christoph Lametera10aa572008-04-28 02:12:40 -07002588{
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002589 spin_unlock(&vmap_area_lock);
Christoph Lametera10aa572008-04-28 02:12:40 -07002590}
2591
Eric Dumazeta47a1262008-07-23 21:27:38 -07002592static void show_numa_info(struct seq_file *m, struct vm_struct *v)
2593{
Kirill A. Shutemove5adfff2012-12-11 16:00:29 -08002594 if (IS_ENABLED(CONFIG_NUMA)) {
Eric Dumazeta47a1262008-07-23 21:27:38 -07002595 unsigned int nr, *counters = m->private;
2596
2597 if (!counters)
2598 return;
2599
Wanpeng Liaf123462013-11-12 15:07:32 -08002600 if (v->flags & VM_UNINITIALIZED)
2601 return;
Dmitry Vyukov7e5b5282014-12-12 16:56:30 -08002602 /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
2603 smp_rmb();
Wanpeng Liaf123462013-11-12 15:07:32 -08002604
Eric Dumazeta47a1262008-07-23 21:27:38 -07002605 memset(counters, 0, nr_node_ids * sizeof(unsigned int));
2606
2607 for (nr = 0; nr < v->nr_pages; nr++)
2608 counters[page_to_nid(v->pages[nr])]++;
2609
2610 for_each_node_state(nr, N_HIGH_MEMORY)
2611 if (counters[nr])
2612 seq_printf(m, " N%u=%u", nr, counters[nr]);
2613 }
2614}
2615
Christoph Lametera10aa572008-04-28 02:12:40 -07002616static int s_show(struct seq_file *m, void *p)
2617{
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002618 struct vmap_area *va = p;
2619 struct vm_struct *v;
2620
Wanpeng Lic2ce8c12013-11-12 15:07:31 -08002621 /*
2622 * s_show can encounter race with remove_vm_area, !VM_VM_AREA on
2623 * behalf of vmap area is being tear down or vm_map_ram allocation.
2624 */
2625 if (!(va->flags & VM_VM_AREA))
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002626 return 0;
2627
Joonsoo Kimd4033af2013-04-29 15:07:35 -07002628 v = va->vm;
Christoph Lametera10aa572008-04-28 02:12:40 -07002629
Kees Cook45ec1692012-10-08 16:34:09 -07002630 seq_printf(m, "0x%pK-0x%pK %7ld",
Christoph Lametera10aa572008-04-28 02:12:40 -07002631 v->addr, v->addr + v->size, v->size);
2632
Joe Perches62c70bc2011-01-13 15:45:52 -08002633 if (v->caller)
2634 seq_printf(m, " %pS", v->caller);
Christoph Lameter23016962008-04-28 02:12:42 -07002635
Christoph Lametera10aa572008-04-28 02:12:40 -07002636 if (v->nr_pages)
2637 seq_printf(m, " pages=%d", v->nr_pages);
2638
2639 if (v->phys_addr)
Kenji Kaneshigeffa71f32010-06-18 12:22:40 +09002640 seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
Christoph Lametera10aa572008-04-28 02:12:40 -07002641
2642 if (v->flags & VM_IOREMAP)
Fabian Frederickf4527c92014-06-04 16:08:09 -07002643 seq_puts(m, " ioremap");
Christoph Lametera10aa572008-04-28 02:12:40 -07002644
2645 if (v->flags & VM_ALLOC)
Fabian Frederickf4527c92014-06-04 16:08:09 -07002646 seq_puts(m, " vmalloc");
Christoph Lametera10aa572008-04-28 02:12:40 -07002647
2648 if (v->flags & VM_MAP)
Fabian Frederickf4527c92014-06-04 16:08:09 -07002649 seq_puts(m, " vmap");
Christoph Lametera10aa572008-04-28 02:12:40 -07002650
2651 if (v->flags & VM_USERMAP)
Fabian Frederickf4527c92014-06-04 16:08:09 -07002652 seq_puts(m, " user");
Christoph Lametera10aa572008-04-28 02:12:40 -07002653
2654 if (v->flags & VM_VPAGES)
Fabian Frederickf4527c92014-06-04 16:08:09 -07002655 seq_puts(m, " vpages");
Christoph Lametera10aa572008-04-28 02:12:40 -07002656
Eric Dumazeta47a1262008-07-23 21:27:38 -07002657 show_numa_info(m, v);
Christoph Lametera10aa572008-04-28 02:12:40 -07002658 seq_putc(m, '\n');
2659 return 0;
2660}
2661
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +04002662static const struct seq_operations vmalloc_op = {
Christoph Lametera10aa572008-04-28 02:12:40 -07002663 .start = s_start,
2664 .next = s_next,
2665 .stop = s_stop,
2666 .show = s_show,
2667};
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +04002668
2669static int vmalloc_open(struct inode *inode, struct file *file)
2670{
Rob Jones703394c2014-10-09 15:28:01 -07002671 if (IS_ENABLED(CONFIG_NUMA))
2672 return seq_open_private(file, &vmalloc_op,
2673 nr_node_ids * sizeof(unsigned int));
2674 else
2675 return seq_open(file, &vmalloc_op);
Alexey Dobriyan5f6a6a92008-10-06 03:50:47 +04002676}
2677
2678static const struct file_operations proc_vmalloc_operations = {
2679 .open = vmalloc_open,
2680 .read = seq_read,
2681 .llseek = seq_lseek,
2682 .release = seq_release_private,
2683};
2684
2685static int __init proc_vmalloc_init(void)
2686{
2687 proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
2688 return 0;
2689}
2690module_init(proc_vmalloc_init);
Joonsoo Kimdb3808c2013-04-29 15:07:28 -07002691
Christoph Lametera10aa572008-04-28 02:12:40 -07002692#endif
2693