blob: 79c3cac87afa1d1f96fc9397f50bb370980e0a84 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/nommu.c
3 *
4 * Replacement code for mm functions to support CPU's that don't
5 * have any form of memory management unit (thus no virtual memory).
6 *
7 * See Documentation/nommu-mmap.txt
8 *
David Howells8feae132009-01-08 12:04:47 +00009 * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
11 * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
12 * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com>
Paul Mundt29c185e2010-12-24 12:08:30 +090013 * Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014 */
15
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040016#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/mm.h>
18#include <linux/mman.h>
19#include <linux/swap.h>
20#include <linux/file.h>
21#include <linux/highmem.h>
22#include <linux/pagemap.h>
23#include <linux/slab.h>
24#include <linux/vmalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/blkdev.h>
26#include <linux/backing-dev.h>
27#include <linux/mount.h>
28#include <linux/personality.h>
29#include <linux/security.h>
30#include <linux/syscalls.h>
Al Viro120a7952010-10-30 02:54:44 -040031#include <linux/audit.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
33#include <asm/uaccess.h>
34#include <asm/tlb.h>
35#include <asm/tlbflush.h>
Bernd Schmidteb8cdec2009-09-21 17:03:57 -070036#include <asm/mmu_context.h>
David Howells8feae132009-01-08 12:04:47 +000037#include "internal.h"
38
David Howells8feae132009-01-08 12:04:47 +000039#if 0
40#define kenter(FMT, ...) \
41 printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
42#define kleave(FMT, ...) \
43 printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
44#define kdebug(FMT, ...) \
45 printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__)
46#else
47#define kenter(FMT, ...) \
48 no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
49#define kleave(FMT, ...) \
50 no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
51#define kdebug(FMT, ...) \
52 no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__)
53#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55void *high_memory;
56struct page *mem_map;
57unsigned long max_mapnr;
58unsigned long num_physpages;
Hugh Dickins4266c972009-09-23 17:05:53 +010059unsigned long highest_memmap_pfn;
KOSAKI Motohiro00a62ce2009-04-30 15:08:51 -070060struct percpu_counter vm_committed_as;
Linus Torvalds1da177e2005-04-16 15:20:36 -070061int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
62int sysctl_overcommit_ratio = 50; /* default is 50% */
63int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
David Howellsfc4d5c22009-05-06 16:03:05 -070064int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
Linus Torvalds1da177e2005-04-16 15:20:36 -070065int heap_stack_gap = 0;
66
David Howells33e5d7692009-04-02 16:56:32 -070067atomic_long_t mmap_pages_allocated;
David Howells8feae132009-01-08 12:04:47 +000068
K. Y. Srinivasan997071b2012-11-15 14:34:42 -080069/*
70 * The global memory commitment made in the system can be a metric
71 * that can be used to drive ballooning decisions when Linux is hosted
72 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
73 * balancing memory across competing virtual machines that are hosted.
74 * Several metrics drive this policy engine including the guest reported
75 * memory commitment.
76 */
77unsigned long vm_memory_committed(void)
78{
79 return percpu_counter_read_positive(&vm_committed_as);
80}
81
82EXPORT_SYMBOL_GPL(vm_memory_committed);
83
Linus Torvalds1da177e2005-04-16 15:20:36 -070084EXPORT_SYMBOL(mem_map);
Wu, Bryan6a04de62007-04-11 23:28:47 -070085EXPORT_SYMBOL(num_physpages);
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
David Howells8feae132009-01-08 12:04:47 +000087/* list of mapped, potentially shareable regions */
88static struct kmem_cache *vm_region_jar;
89struct rb_root nommu_region_tree = RB_ROOT;
90DECLARE_RWSEM(nommu_region_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -070091
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +040092const struct vm_operations_struct generic_file_vm_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070093};
94
95/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 * Return the total memory allocated for this pointer, not
97 * just what the caller asked for.
98 *
99 * Doesn't have to be accurate, i.e. may have races.
100 */
101unsigned int kobjsize(const void *objp)
102{
103 struct page *page;
104
Michael Hennerich4016a132008-04-28 02:13:38 -0700105 /*
106 * If the object we have should not have ksize performed on it,
107 * return size of 0
108 */
Paul Mundt5a1603b2008-06-12 16:29:55 +0900109 if (!objp || !virt_addr_valid(objp))
Paul Mundt6cfd53fc2008-06-05 22:46:08 -0700110 return 0;
111
112 page = virt_to_head_page(objp);
Paul Mundt6cfd53fc2008-06-05 22:46:08 -0700113
114 /*
115 * If the allocator sets PageSlab, we know the pointer came from
116 * kmalloc().
117 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 if (PageSlab(page))
119 return ksize(objp);
120
Paul Mundt6cfd53fc2008-06-05 22:46:08 -0700121 /*
Paul Mundtab2e83e2009-01-08 12:04:48 +0000122 * If it's not a compound page, see if we have a matching VMA
123 * region. This test is intentionally done in reverse order,
124 * so if there's no VMA, we still fall through and hand back
125 * PAGE_SIZE for 0-order pages.
126 */
127 if (!PageCompound(page)) {
128 struct vm_area_struct *vma;
129
130 vma = find_vma(current->mm, (unsigned long)objp);
131 if (vma)
132 return vma->vm_end - vma->vm_start;
133 }
134
135 /*
Paul Mundt6cfd53fc2008-06-05 22:46:08 -0700136 * The ksize() function is only guaranteed to work for pointers
Paul Mundt5a1603b2008-06-12 16:29:55 +0900137 * returned by kmalloc(). So handle arbitrary pointers here.
Paul Mundt6cfd53fc2008-06-05 22:46:08 -0700138 */
Paul Mundt5a1603b2008-06-12 16:29:55 +0900139 return PAGE_SIZE << compound_order(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140}
141
Nick Pigginb291f002008-10-18 20:26:44 -0700142int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
Hugh Dickins4266c972009-09-23 17:05:53 +0100143 unsigned long start, int nr_pages, unsigned int foll_flags,
Michel Lespinasse53a77062011-01-13 15:46:14 -0800144 struct page **pages, struct vm_area_struct **vmas,
145 int *retry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146{
Sonic Zhang910e46d2006-09-27 01:50:17 -0700147 struct vm_area_struct *vma;
David Howells7b4d5b82006-09-27 01:50:18 -0700148 unsigned long vm_flags;
149 int i;
150
151 /* calculate required read or write permissions.
Hugh Dickins58fa8792009-09-21 17:03:31 -0700152 * If FOLL_FORCE is set, we only require the "MAY" flags.
David Howells7b4d5b82006-09-27 01:50:18 -0700153 */
Hugh Dickins58fa8792009-09-21 17:03:31 -0700154 vm_flags = (foll_flags & FOLL_WRITE) ?
155 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
156 vm_flags &= (foll_flags & FOLL_FORCE) ?
157 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158
Peter Zijlstra9d737772009-06-25 11:58:55 +0200159 for (i = 0; i < nr_pages; i++) {
David Howells7561e8c2010-03-25 16:48:38 +0000160 vma = find_vma(mm, start);
David Howells7b4d5b82006-09-27 01:50:18 -0700161 if (!vma)
162 goto finish_or_fault;
163
164 /* protect what we can, including chardevs */
Hugh Dickins1c3aff12009-09-21 17:03:24 -0700165 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
166 !(vm_flags & vma->vm_flags))
David Howells7b4d5b82006-09-27 01:50:18 -0700167 goto finish_or_fault;
Sonic Zhang910e46d2006-09-27 01:50:17 -0700168
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 if (pages) {
170 pages[i] = virt_to_page(start);
171 if (pages[i])
172 page_cache_get(pages[i]);
173 }
174 if (vmas)
Sonic Zhang910e46d2006-09-27 01:50:17 -0700175 vmas[i] = vma;
David Howellse1ee65d2010-03-25 16:48:44 +0000176 start = (start + PAGE_SIZE) & PAGE_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 }
David Howells7b4d5b82006-09-27 01:50:18 -0700178
179 return i;
180
181finish_or_fault:
182 return i ? : -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183}
Nick Pigginb291f002008-10-18 20:26:44 -0700184
Nick Pigginb291f002008-10-18 20:26:44 -0700185/*
186 * get a list of pages in an address range belonging to the specified process
187 * and indicate the VMA that covers each page
188 * - this is potentially dodgy as we may end incrementing the page count of a
189 * slab page or a secondary page from a compound page
190 * - don't permit access to VMAs that don't support it, such as I/O mappings
191 */
192int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
Peter Zijlstra9d737772009-06-25 11:58:55 +0200193 unsigned long start, int nr_pages, int write, int force,
Nick Pigginb291f002008-10-18 20:26:44 -0700194 struct page **pages, struct vm_area_struct **vmas)
195{
196 int flags = 0;
197
198 if (write)
Hugh Dickins58fa8792009-09-21 17:03:31 -0700199 flags |= FOLL_WRITE;
Nick Pigginb291f002008-10-18 20:26:44 -0700200 if (force)
Hugh Dickins58fa8792009-09-21 17:03:31 -0700201 flags |= FOLL_FORCE;
Nick Pigginb291f002008-10-18 20:26:44 -0700202
Michel Lespinasse53a77062011-01-13 15:46:14 -0800203 return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas,
204 NULL);
Nick Pigginb291f002008-10-18 20:26:44 -0700205}
Greg Ungerer66aa2b42005-09-12 11:18:10 +1000206EXPORT_SYMBOL(get_user_pages);
207
Paul Mundtdfc2f912009-06-26 04:31:57 +0900208/**
209 * follow_pfn - look up PFN at a user virtual address
210 * @vma: memory mapping
211 * @address: user virtual address
212 * @pfn: location to store found PFN
213 *
214 * Only IO mappings and raw PFN mappings are allowed.
215 *
216 * Returns zero and the pfn at @pfn on success, -ve otherwise.
217 */
218int follow_pfn(struct vm_area_struct *vma, unsigned long address,
219 unsigned long *pfn)
220{
221 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
222 return -EINVAL;
223
224 *pfn = address >> PAGE_SHIFT;
225 return 0;
226}
227EXPORT_SYMBOL(follow_pfn);
228
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229DEFINE_RWLOCK(vmlist_lock);
230struct vm_struct *vmlist;
231
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800232void vfree(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233{
234 kfree(addr);
235}
Paul Mundtb5073172007-07-21 04:37:25 -0700236EXPORT_SYMBOL(vfree);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237
Al Virodd0fc662005-10-07 07:46:04 +0100238void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239{
240 /*
Robert P. J. Day85186092007-10-19 23:11:38 +0200241 * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
242 * returns only a logical address.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 */
Nick Piggin84097512006-03-22 00:08:34 -0800244 return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245}
Paul Mundtb5073172007-07-21 04:37:25 -0700246EXPORT_SYMBOL(__vmalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247
Paul Mundtf905bc42008-02-04 22:29:59 -0800248void *vmalloc_user(unsigned long size)
249{
250 void *ret;
251
252 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
253 PAGE_KERNEL);
254 if (ret) {
255 struct vm_area_struct *vma;
256
257 down_write(&current->mm->mmap_sem);
258 vma = find_vma(current->mm, (unsigned long)ret);
259 if (vma)
260 vma->vm_flags |= VM_USERMAP;
261 up_write(&current->mm->mmap_sem);
262 }
263
264 return ret;
265}
266EXPORT_SYMBOL(vmalloc_user);
267
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800268struct page *vmalloc_to_page(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269{
270 return virt_to_page(addr);
271}
Paul Mundtb5073172007-07-21 04:37:25 -0700272EXPORT_SYMBOL(vmalloc_to_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800274unsigned long vmalloc_to_pfn(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275{
276 return page_to_pfn(virt_to_page(addr));
277}
Paul Mundtb5073172007-07-21 04:37:25 -0700278EXPORT_SYMBOL(vmalloc_to_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
280long vread(char *buf, char *addr, unsigned long count)
281{
282 memcpy(buf, addr, count);
283 return count;
284}
285
286long vwrite(char *buf, char *addr, unsigned long count)
287{
288 /* Don't allow overflow */
289 if ((unsigned long) addr + count < count)
290 count = -(unsigned long) addr;
291
292 memcpy(addr, buf, count);
293 return(count);
294}
295
296/*
297 * vmalloc - allocate virtually continguos memory
298 *
299 * @size: allocation size
300 *
301 * Allocate enough pages to cover @size from the page level
302 * allocator and map them into continguos kernel virtual space.
303 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200304 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 * use __vmalloc() instead.
306 */
307void *vmalloc(unsigned long size)
308{
309 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
310}
Andrew Mortonf6138882006-02-28 16:59:18 -0800311EXPORT_SYMBOL(vmalloc);
312
Dave Younge1ca7782010-10-26 14:22:06 -0700313/*
314 * vzalloc - allocate virtually continguos memory with zero fill
315 *
316 * @size: allocation size
317 *
318 * Allocate enough pages to cover @size from the page level
319 * allocator and map them into continguos kernel virtual space.
320 * The memory allocated is set to zero.
321 *
322 * For tight control over page level allocator and protection flags
323 * use __vmalloc() instead.
324 */
325void *vzalloc(unsigned long size)
326{
327 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
328 PAGE_KERNEL);
329}
330EXPORT_SYMBOL(vzalloc);
331
332/**
333 * vmalloc_node - allocate memory on a specific node
334 * @size: allocation size
335 * @node: numa node
336 *
337 * Allocate enough pages to cover @size from the page level
338 * allocator and map them into contiguous kernel virtual space.
339 *
340 * For tight control over page level allocator and protection flags
341 * use __vmalloc() instead.
342 */
Andrew Mortonf6138882006-02-28 16:59:18 -0800343void *vmalloc_node(unsigned long size, int node)
344{
345 return vmalloc(size);
346}
Paul Mundt9a14f652010-12-24 11:50:34 +0900347EXPORT_SYMBOL(vmalloc_node);
Dave Younge1ca7782010-10-26 14:22:06 -0700348
349/**
350 * vzalloc_node - allocate memory on a specific node with zero fill
351 * @size: allocation size
352 * @node: numa node
353 *
354 * Allocate enough pages to cover @size from the page level
355 * allocator and map them into contiguous kernel virtual space.
356 * The memory allocated is set to zero.
357 *
358 * For tight control over page level allocator and protection flags
359 * use __vmalloc() instead.
360 */
361void *vzalloc_node(unsigned long size, int node)
362{
363 return vzalloc(size);
364}
365EXPORT_SYMBOL(vzalloc_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366
Paul Mundt1af446e2008-08-04 16:01:47 +0900367#ifndef PAGE_KERNEL_EXEC
368# define PAGE_KERNEL_EXEC PAGE_KERNEL
369#endif
370
371/**
372 * vmalloc_exec - allocate virtually contiguous, executable memory
373 * @size: allocation size
374 *
375 * Kernel-internal function to allocate enough pages to cover @size
376 * the page level allocator and map them into contiguous and
377 * executable kernel virtual space.
378 *
379 * For tight control over page level allocator and protection flags
380 * use __vmalloc() instead.
381 */
382
383void *vmalloc_exec(unsigned long size)
384{
385 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
386}
387
Paul Mundtb5073172007-07-21 04:37:25 -0700388/**
389 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 * @size: allocation size
391 *
392 * Allocate enough 32bit PA addressable pages to cover @size from the
393 * page level allocator and map them into continguos kernel virtual space.
394 */
395void *vmalloc_32(unsigned long size)
396{
397 return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
398}
Paul Mundtb5073172007-07-21 04:37:25 -0700399EXPORT_SYMBOL(vmalloc_32);
400
401/**
402 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
403 * @size: allocation size
404 *
405 * The resulting memory area is 32bit addressable and zeroed so it can be
406 * mapped to userspace without leaking data.
Paul Mundtf905bc42008-02-04 22:29:59 -0800407 *
408 * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
409 * remap_vmalloc_range() are permissible.
Paul Mundtb5073172007-07-21 04:37:25 -0700410 */
411void *vmalloc_32_user(unsigned long size)
412{
Paul Mundtf905bc42008-02-04 22:29:59 -0800413 /*
414 * We'll have to sort out the ZONE_DMA bits for 64-bit,
415 * but for now this can simply use vmalloc_user() directly.
416 */
417 return vmalloc_user(size);
Paul Mundtb5073172007-07-21 04:37:25 -0700418}
419EXPORT_SYMBOL(vmalloc_32_user);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420
421void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
422{
423 BUG();
424 return NULL;
425}
Paul Mundtb5073172007-07-21 04:37:25 -0700426EXPORT_SYMBOL(vmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800428void vunmap(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429{
430 BUG();
431}
Paul Mundtb5073172007-07-21 04:37:25 -0700432EXPORT_SYMBOL(vunmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433
Paul Mundteb6434d2009-01-21 17:45:47 +0900434void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
435{
436 BUG();
437 return NULL;
438}
439EXPORT_SYMBOL(vm_map_ram);
440
441void vm_unmap_ram(const void *mem, unsigned int count)
442{
443 BUG();
444}
445EXPORT_SYMBOL(vm_unmap_ram);
446
447void vm_unmap_aliases(void)
448{
449}
450EXPORT_SYMBOL_GPL(vm_unmap_aliases);
451
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452/*
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -0700453 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
454 * have one.
455 */
456void __attribute__((weak)) vmalloc_sync_all(void)
457{
458}
459
Paul Mundt29c185e2010-12-24 12:08:30 +0900460/**
461 * alloc_vm_area - allocate a range of kernel address space
462 * @size: size of the area
463 *
464 * Returns: NULL on failure, vm_struct on success
465 *
466 * This function reserves a range of kernel address space, and
467 * allocates pagetables to map that range. No actual mappings
468 * are created. If the kernel address space is not shared
469 * between processes, it syncs the pagetable across all
470 * processes.
471 */
David Vrabelcd129092011-09-29 16:53:32 +0100472struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
Paul Mundt29c185e2010-12-24 12:08:30 +0900473{
474 BUG();
475 return NULL;
476}
477EXPORT_SYMBOL_GPL(alloc_vm_area);
478
479void free_vm_area(struct vm_struct *area)
480{
481 BUG();
482}
483EXPORT_SYMBOL_GPL(free_vm_area);
484
Paul Mundtb5073172007-07-21 04:37:25 -0700485int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
486 struct page *page)
487{
488 return -EINVAL;
489}
490EXPORT_SYMBOL(vm_insert_page);
491
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -0700492/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 * sys_brk() for the most part doesn't need the global kernel
494 * lock, except when an application is doing something nasty
495 * like trying to un-brk an area that has already been mapped
496 * to a regular file. in this case, the unmapping will need
497 * to invoke file system routines that need the global lock.
498 */
Heiko Carstens6a6160a2009-01-14 14:14:15 +0100499SYSCALL_DEFINE1(brk, unsigned long, brk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500{
501 struct mm_struct *mm = current->mm;
502
503 if (brk < mm->start_brk || brk > mm->context.end_brk)
504 return mm->brk;
505
506 if (mm->brk == brk)
507 return mm->brk;
508
509 /*
510 * Always allow shrinking brk
511 */
512 if (brk <= mm->brk) {
513 mm->brk = brk;
514 return brk;
515 }
516
517 /*
518 * Ok, looks good - let it rip.
519 */
Mike Frysingercfe79c02010-01-06 17:23:23 +0000520 flush_icache_range(mm->brk, brk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 return mm->brk = brk;
522}
523
David Howells8feae132009-01-08 12:04:47 +0000524/*
525 * initialise the VMA and region record slabs
526 */
527void __init mmap_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528{
KOSAKI Motohiro00a62ce2009-04-30 15:08:51 -0700529 int ret;
530
531 ret = percpu_counter_init(&vm_committed_as, 0);
532 VM_BUG_ON(ret);
David Howells33e5d7692009-04-02 16:56:32 -0700533 vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC);
David Howells8feae132009-01-08 12:04:47 +0000534}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535
David Howells8feae132009-01-08 12:04:47 +0000536/*
537 * validate the region tree
538 * - the caller must hold the region lock
539 */
540#ifdef CONFIG_DEBUG_NOMMU_REGIONS
541static noinline void validate_nommu_regions(void)
542{
543 struct vm_region *region, *last;
544 struct rb_node *p, *lastp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545
David Howells8feae132009-01-08 12:04:47 +0000546 lastp = rb_first(&nommu_region_tree);
547 if (!lastp)
548 return;
549
550 last = rb_entry(lastp, struct vm_region, vm_rb);
David Howells33e5d7692009-04-02 16:56:32 -0700551 BUG_ON(unlikely(last->vm_end <= last->vm_start));
552 BUG_ON(unlikely(last->vm_top < last->vm_end));
David Howells8feae132009-01-08 12:04:47 +0000553
554 while ((p = rb_next(lastp))) {
555 region = rb_entry(p, struct vm_region, vm_rb);
556 last = rb_entry(lastp, struct vm_region, vm_rb);
557
David Howells33e5d7692009-04-02 16:56:32 -0700558 BUG_ON(unlikely(region->vm_end <= region->vm_start));
559 BUG_ON(unlikely(region->vm_top < region->vm_end));
560 BUG_ON(unlikely(region->vm_start < last->vm_top));
David Howells8feae132009-01-08 12:04:47 +0000561
562 lastp = p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 }
564}
David Howells8feae132009-01-08 12:04:47 +0000565#else
David Howells33e5d7692009-04-02 16:56:32 -0700566static void validate_nommu_regions(void)
567{
568}
David Howells8feae132009-01-08 12:04:47 +0000569#endif
570
571/*
572 * add a region into the global tree
573 */
574static void add_nommu_region(struct vm_region *region)
575{
576 struct vm_region *pregion;
577 struct rb_node **p, *parent;
578
579 validate_nommu_regions();
580
David Howells8feae132009-01-08 12:04:47 +0000581 parent = NULL;
582 p = &nommu_region_tree.rb_node;
583 while (*p) {
584 parent = *p;
585 pregion = rb_entry(parent, struct vm_region, vm_rb);
586 if (region->vm_start < pregion->vm_start)
587 p = &(*p)->rb_left;
588 else if (region->vm_start > pregion->vm_start)
589 p = &(*p)->rb_right;
590 else if (pregion == region)
591 return;
592 else
593 BUG();
594 }
595
596 rb_link_node(&region->vm_rb, parent, p);
597 rb_insert_color(&region->vm_rb, &nommu_region_tree);
598
599 validate_nommu_regions();
600}
601
602/*
603 * delete a region from the global tree
604 */
605static void delete_nommu_region(struct vm_region *region)
606{
607 BUG_ON(!nommu_region_tree.rb_node);
608
609 validate_nommu_regions();
610 rb_erase(&region->vm_rb, &nommu_region_tree);
611 validate_nommu_regions();
612}
613
614/*
615 * free a contiguous series of pages
616 */
617static void free_page_series(unsigned long from, unsigned long to)
618{
619 for (; from < to; from += PAGE_SIZE) {
620 struct page *page = virt_to_page(from);
621
622 kdebug("- free %lx", from);
David Howells33e5d7692009-04-02 16:56:32 -0700623 atomic_long_dec(&mmap_pages_allocated);
David Howells8feae132009-01-08 12:04:47 +0000624 if (page_count(page) != 1)
David Howells33e5d7692009-04-02 16:56:32 -0700625 kdebug("free page %p: refcount not one: %d",
626 page, page_count(page));
David Howells8feae132009-01-08 12:04:47 +0000627 put_page(page);
628 }
629}
630
631/*
632 * release a reference to a region
David Howells33e5d7692009-04-02 16:56:32 -0700633 * - the caller must hold the region semaphore for writing, which this releases
Paul Mundtdd8632a2009-01-08 12:04:47 +0000634 * - the region may not have been added to the tree yet, in which case vm_top
David Howells8feae132009-01-08 12:04:47 +0000635 * will equal vm_start
636 */
637static void __put_nommu_region(struct vm_region *region)
638 __releases(nommu_region_sem)
639{
David Howells1e2ae592010-01-15 17:01:33 -0800640 kenter("%p{%d}", region, region->vm_usage);
David Howells8feae132009-01-08 12:04:47 +0000641
642 BUG_ON(!nommu_region_tree.rb_node);
643
David Howells1e2ae592010-01-15 17:01:33 -0800644 if (--region->vm_usage == 0) {
Paul Mundtdd8632a2009-01-08 12:04:47 +0000645 if (region->vm_top > region->vm_start)
David Howells8feae132009-01-08 12:04:47 +0000646 delete_nommu_region(region);
647 up_write(&nommu_region_sem);
648
649 if (region->vm_file)
650 fput(region->vm_file);
651
652 /* IO memory and memory shared directly out of the pagecache
653 * from ramfs/tmpfs mustn't be released here */
654 if (region->vm_flags & VM_MAPPED_COPY) {
655 kdebug("free series");
Paul Mundtdd8632a2009-01-08 12:04:47 +0000656 free_page_series(region->vm_start, region->vm_top);
David Howells8feae132009-01-08 12:04:47 +0000657 }
658 kmem_cache_free(vm_region_jar, region);
659 } else {
660 up_write(&nommu_region_sem);
661 }
662}
663
664/*
665 * release a reference to a region
666 */
667static void put_nommu_region(struct vm_region *region)
668{
669 down_write(&nommu_region_sem);
670 __put_nommu_region(region);
671}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672
David Howells30340972006-09-27 01:50:20 -0700673/*
Bernd Schmidteb8cdec2009-09-21 17:03:57 -0700674 * update protection on a vma
675 */
676static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
677{
678#ifdef CONFIG_MPU
679 struct mm_struct *mm = vma->vm_mm;
680 long start = vma->vm_start & PAGE_MASK;
681 while (start < vma->vm_end) {
682 protect_page(mm, start, flags);
683 start += PAGE_SIZE;
684 }
685 update_protections(mm);
686#endif
687}
688
689/*
David Howells30340972006-09-27 01:50:20 -0700690 * add a VMA into a process's mm_struct in the appropriate place in the list
David Howells8feae132009-01-08 12:04:47 +0000691 * and tree and add to the address space's page tree also if not an anonymous
692 * page
David Howells30340972006-09-27 01:50:20 -0700693 * - should be called with mm->mmap_sem held writelocked
694 */
David Howells8feae132009-01-08 12:04:47 +0000695static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
David Howells30340972006-09-27 01:50:20 -0700696{
Namhyung Kim6038def2011-05-24 17:11:22 -0700697 struct vm_area_struct *pvma, *prev;
David Howells8feae132009-01-08 12:04:47 +0000698 struct address_space *mapping;
Namhyung Kim6038def2011-05-24 17:11:22 -0700699 struct rb_node **p, *parent, *rb_prev;
David Howells30340972006-09-27 01:50:20 -0700700
David Howells8feae132009-01-08 12:04:47 +0000701 kenter(",%p", vma);
702
703 BUG_ON(!vma->vm_region);
704
705 mm->map_count++;
706 vma->vm_mm = mm;
707
Bernd Schmidteb8cdec2009-09-21 17:03:57 -0700708 protect_vma(vma, vma->vm_flags);
709
David Howells8feae132009-01-08 12:04:47 +0000710 /* add the VMA to the mapping */
711 if (vma->vm_file) {
712 mapping = vma->vm_file->f_mapping;
713
David Howells918e5562012-02-23 13:50:35 +0000714 mutex_lock(&mapping->i_mmap_mutex);
David Howells8feae132009-01-08 12:04:47 +0000715 flush_dcache_mmap_lock(mapping);
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700716 vma_interval_tree_insert(vma, &mapping->i_mmap);
David Howells8feae132009-01-08 12:04:47 +0000717 flush_dcache_mmap_unlock(mapping);
David Howells918e5562012-02-23 13:50:35 +0000718 mutex_unlock(&mapping->i_mmap_mutex);
David Howells8feae132009-01-08 12:04:47 +0000719 }
720
721 /* add the VMA to the tree */
Namhyung Kim6038def2011-05-24 17:11:22 -0700722 parent = rb_prev = NULL;
David Howells8feae132009-01-08 12:04:47 +0000723 p = &mm->mm_rb.rb_node;
724 while (*p) {
725 parent = *p;
726 pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
727
728 /* sort by: start addr, end addr, VMA struct addr in that order
729 * (the latter is necessary as we may get identical VMAs) */
730 if (vma->vm_start < pvma->vm_start)
731 p = &(*p)->rb_left;
Namhyung Kim6038def2011-05-24 17:11:22 -0700732 else if (vma->vm_start > pvma->vm_start) {
733 rb_prev = parent;
David Howells8feae132009-01-08 12:04:47 +0000734 p = &(*p)->rb_right;
Namhyung Kim6038def2011-05-24 17:11:22 -0700735 } else if (vma->vm_end < pvma->vm_end)
David Howells8feae132009-01-08 12:04:47 +0000736 p = &(*p)->rb_left;
Namhyung Kim6038def2011-05-24 17:11:22 -0700737 else if (vma->vm_end > pvma->vm_end) {
738 rb_prev = parent;
David Howells8feae132009-01-08 12:04:47 +0000739 p = &(*p)->rb_right;
Namhyung Kim6038def2011-05-24 17:11:22 -0700740 } else if (vma < pvma)
David Howells8feae132009-01-08 12:04:47 +0000741 p = &(*p)->rb_left;
Namhyung Kim6038def2011-05-24 17:11:22 -0700742 else if (vma > pvma) {
743 rb_prev = parent;
David Howells8feae132009-01-08 12:04:47 +0000744 p = &(*p)->rb_right;
Namhyung Kim6038def2011-05-24 17:11:22 -0700745 } else
David Howells8feae132009-01-08 12:04:47 +0000746 BUG();
747 }
748
749 rb_link_node(&vma->vm_rb, parent, p);
750 rb_insert_color(&vma->vm_rb, &mm->mm_rb);
751
752 /* add VMA to the VMA list also */
Namhyung Kim6038def2011-05-24 17:11:22 -0700753 prev = NULL;
754 if (rb_prev)
755 prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
David Howells30340972006-09-27 01:50:20 -0700756
Namhyung Kim6038def2011-05-24 17:11:22 -0700757 __vma_link_list(mm, vma, prev, parent);
David Howells8feae132009-01-08 12:04:47 +0000758}
759
760/*
761 * delete a VMA from its owning mm_struct and address space
762 */
763static void delete_vma_from_mm(struct vm_area_struct *vma)
764{
David Howells8feae132009-01-08 12:04:47 +0000765 struct address_space *mapping;
766 struct mm_struct *mm = vma->vm_mm;
767
768 kenter("%p", vma);
769
Bernd Schmidteb8cdec2009-09-21 17:03:57 -0700770 protect_vma(vma, 0);
771
David Howells8feae132009-01-08 12:04:47 +0000772 mm->map_count--;
773 if (mm->mmap_cache == vma)
774 mm->mmap_cache = NULL;
775
776 /* remove the VMA from the mapping */
777 if (vma->vm_file) {
778 mapping = vma->vm_file->f_mapping;
779
David Howells918e5562012-02-23 13:50:35 +0000780 mutex_lock(&mapping->i_mmap_mutex);
David Howells8feae132009-01-08 12:04:47 +0000781 flush_dcache_mmap_lock(mapping);
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700782 vma_interval_tree_remove(vma, &mapping->i_mmap);
David Howells8feae132009-01-08 12:04:47 +0000783 flush_dcache_mmap_unlock(mapping);
David Howells918e5562012-02-23 13:50:35 +0000784 mutex_unlock(&mapping->i_mmap_mutex);
David Howells8feae132009-01-08 12:04:47 +0000785 }
786
787 /* remove from the MM's tree and list */
788 rb_erase(&vma->vm_rb, &mm->mm_rb);
Namhyung Kimb951bf22011-05-24 17:11:23 -0700789
790 if (vma->vm_prev)
791 vma->vm_prev->vm_next = vma->vm_next;
792 else
793 mm->mmap = vma->vm_next;
794
795 if (vma->vm_next)
796 vma->vm_next->vm_prev = vma->vm_prev;
David Howells8feae132009-01-08 12:04:47 +0000797}
798
799/*
800 * destroy a VMA record
801 */
802static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
803{
804 kenter("%p", vma);
805 if (vma->vm_ops && vma->vm_ops->close)
806 vma->vm_ops->close(vma);
Konstantin Khlebnikove9714ac2012-10-08 16:28:54 -0700807 if (vma->vm_file)
David Howells8feae132009-01-08 12:04:47 +0000808 fput(vma->vm_file);
David Howells8feae132009-01-08 12:04:47 +0000809 put_nommu_region(vma->vm_region);
810 kmem_cache_free(vm_area_cachep, vma);
David Howells30340972006-09-27 01:50:20 -0700811}
812
813/*
814 * look up the first VMA in which addr resides, NULL if none
815 * - should be called with mm->mmap_sem at least held readlocked
816 */
817struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
818{
David Howells8feae132009-01-08 12:04:47 +0000819 struct vm_area_struct *vma;
David Howells30340972006-09-27 01:50:20 -0700820
David Howells8feae132009-01-08 12:04:47 +0000821 /* check the cache first */
822 vma = mm->mmap_cache;
823 if (vma && vma->vm_start <= addr && vma->vm_end > addr)
824 return vma;
825
Namhyung Kime922c4c2011-05-24 17:11:24 -0700826 /* trawl the list (there may be multiple mappings in which addr
David Howells8feae132009-01-08 12:04:47 +0000827 * resides) */
Namhyung Kime922c4c2011-05-24 17:11:24 -0700828 for (vma = mm->mmap; vma; vma = vma->vm_next) {
David Howells8feae132009-01-08 12:04:47 +0000829 if (vma->vm_start > addr)
830 return NULL;
831 if (vma->vm_end > addr) {
832 mm->mmap_cache = vma;
833 return vma;
834 }
David Howells30340972006-09-27 01:50:20 -0700835 }
836
David Howells30340972006-09-27 01:50:20 -0700837 return NULL;
838}
839EXPORT_SYMBOL(find_vma);
840
841/*
David Howells930e6522006-09-27 01:50:22 -0700842 * find a VMA
843 * - we don't extend stack VMAs under NOMMU conditions
844 */
845struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
846{
David Howells7561e8c2010-03-25 16:48:38 +0000847 return find_vma(mm, addr);
David Howells930e6522006-09-27 01:50:22 -0700848}
849
David Howells8feae132009-01-08 12:04:47 +0000850/*
851 * expand a stack to a given address
852 * - not supported under NOMMU conditions
853 */
Greg Ungerer57c8f632007-07-15 23:38:28 -0700854int expand_stack(struct vm_area_struct *vma, unsigned long address)
855{
856 return -ENOMEM;
857}
858
David Howells930e6522006-09-27 01:50:22 -0700859/*
David Howells6fa5f802006-09-27 01:50:21 -0700860 * look up the first VMA exactly that exactly matches addr
861 * - should be called with mm->mmap_sem at least held readlocked
862 */
David Howells8feae132009-01-08 12:04:47 +0000863static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
864 unsigned long addr,
865 unsigned long len)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866{
867 struct vm_area_struct *vma;
David Howells8feae132009-01-08 12:04:47 +0000868 unsigned long end = addr + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869
David Howells8feae132009-01-08 12:04:47 +0000870 /* check the cache first */
871 vma = mm->mmap_cache;
872 if (vma && vma->vm_start == addr && vma->vm_end == end)
873 return vma;
874
Namhyung Kime922c4c2011-05-24 17:11:24 -0700875 /* trawl the list (there may be multiple mappings in which addr
David Howells8feae132009-01-08 12:04:47 +0000876 * resides) */
Namhyung Kime922c4c2011-05-24 17:11:24 -0700877 for (vma = mm->mmap; vma; vma = vma->vm_next) {
David Howells8feae132009-01-08 12:04:47 +0000878 if (vma->vm_start < addr)
879 continue;
880 if (vma->vm_start > addr)
881 return NULL;
882 if (vma->vm_end == end) {
883 mm->mmap_cache = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 return vma;
David Howells8feae132009-01-08 12:04:47 +0000885 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 }
887
888 return NULL;
889}
890
David Howells30340972006-09-27 01:50:20 -0700891/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 * determine whether a mapping should be permitted and, if so, what sort of
893 * mapping we're capable of supporting
894 */
895static int validate_mmap_request(struct file *file,
896 unsigned long addr,
897 unsigned long len,
898 unsigned long prot,
899 unsigned long flags,
900 unsigned long pgoff,
901 unsigned long *_capabilities)
902{
David Howells8feae132009-01-08 12:04:47 +0000903 unsigned long capabilities, rlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 int ret;
905
906 /* do the simple checks first */
David Howells06aab5a2009-09-24 12:33:48 +0100907 if (flags & MAP_FIXED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 printk(KERN_DEBUG
909 "%d: Can't do fixed-address/overlay mmap of RAM\n",
910 current->pid);
911 return -EINVAL;
912 }
913
914 if ((flags & MAP_TYPE) != MAP_PRIVATE &&
915 (flags & MAP_TYPE) != MAP_SHARED)
916 return -EINVAL;
917
Mike Frysingerf81cff02006-12-06 12:02:59 +1000918 if (!len)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919 return -EINVAL;
920
Mike Frysingerf81cff02006-12-06 12:02:59 +1000921 /* Careful about overflows.. */
David Howells8feae132009-01-08 12:04:47 +0000922 rlen = PAGE_ALIGN(len);
923 if (!rlen || rlen > TASK_SIZE)
Mike Frysingerf81cff02006-12-06 12:02:59 +1000924 return -ENOMEM;
925
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 /* offset overflow? */
David Howells8feae132009-01-08 12:04:47 +0000927 if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
Mike Frysingerf81cff02006-12-06 12:02:59 +1000928 return -EOVERFLOW;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
930 if (file) {
931 /* validate file mapping requests */
932 struct address_space *mapping;
933
934 /* files must support mmap */
935 if (!file->f_op || !file->f_op->mmap)
936 return -ENODEV;
937
938 /* work out if what we've got could possibly be shared
939 * - we support chardevs that provide their own "memory"
940 * - we support files/blockdevs that are memory backed
941 */
942 mapping = file->f_mapping;
943 if (!mapping)
Josef Sipeke9536ae2006-12-08 02:37:21 -0800944 mapping = file->f_path.dentry->d_inode->i_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945
946 capabilities = 0;
947 if (mapping && mapping->backing_dev_info)
948 capabilities = mapping->backing_dev_info->capabilities;
949
950 if (!capabilities) {
951 /* no explicit capabilities set, so assume some
952 * defaults */
Josef Sipeke9536ae2006-12-08 02:37:21 -0800953 switch (file->f_path.dentry->d_inode->i_mode & S_IFMT) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 case S_IFREG:
955 case S_IFBLK:
956 capabilities = BDI_CAP_MAP_COPY;
957 break;
958
959 case S_IFCHR:
960 capabilities =
961 BDI_CAP_MAP_DIRECT |
962 BDI_CAP_READ_MAP |
963 BDI_CAP_WRITE_MAP;
964 break;
965
966 default:
967 return -EINVAL;
968 }
969 }
970
971 /* eliminate any capabilities that we can't support on this
972 * device */
973 if (!file->f_op->get_unmapped_area)
974 capabilities &= ~BDI_CAP_MAP_DIRECT;
975 if (!file->f_op->read)
976 capabilities &= ~BDI_CAP_MAP_COPY;
977
Graff Yang28d7a6a2009-08-18 14:11:17 -0700978 /* The file shall have been opened with read permission. */
979 if (!(file->f_mode & FMODE_READ))
980 return -EACCES;
981
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 if (flags & MAP_SHARED) {
983 /* do checks for writing, appending and locking */
984 if ((prot & PROT_WRITE) &&
985 !(file->f_mode & FMODE_WRITE))
986 return -EACCES;
987
Josef Sipeke9536ae2006-12-08 02:37:21 -0800988 if (IS_APPEND(file->f_path.dentry->d_inode) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 (file->f_mode & FMODE_WRITE))
990 return -EACCES;
991
Josef Sipeke9536ae2006-12-08 02:37:21 -0800992 if (locks_verify_locked(file->f_path.dentry->d_inode))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 return -EAGAIN;
994
995 if (!(capabilities & BDI_CAP_MAP_DIRECT))
996 return -ENODEV;
997
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 /* we mustn't privatise shared mappings */
999 capabilities &= ~BDI_CAP_MAP_COPY;
1000 }
1001 else {
1002 /* we're going to read the file into private memory we
1003 * allocate */
1004 if (!(capabilities & BDI_CAP_MAP_COPY))
1005 return -ENODEV;
1006
1007 /* we don't permit a private writable mapping to be
1008 * shared with the backing device */
1009 if (prot & PROT_WRITE)
1010 capabilities &= ~BDI_CAP_MAP_DIRECT;
1011 }
1012
Bernd Schmidt3c7b2042010-05-25 23:43:00 -07001013 if (capabilities & BDI_CAP_MAP_DIRECT) {
1014 if (((prot & PROT_READ) && !(capabilities & BDI_CAP_READ_MAP)) ||
1015 ((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) ||
1016 ((prot & PROT_EXEC) && !(capabilities & BDI_CAP_EXEC_MAP))
1017 ) {
1018 capabilities &= ~BDI_CAP_MAP_DIRECT;
1019 if (flags & MAP_SHARED) {
1020 printk(KERN_WARNING
1021 "MAP_SHARED not completely supported on !MMU\n");
1022 return -EINVAL;
1023 }
1024 }
1025 }
1026
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 /* handle executable mappings and implied executable
1028 * mappings */
Josef Sipeke9536ae2006-12-08 02:37:21 -08001029 if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030 if (prot & PROT_EXEC)
1031 return -EPERM;
1032 }
1033 else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
1034 /* handle implication of PROT_EXEC by PROT_READ */
1035 if (current->personality & READ_IMPLIES_EXEC) {
1036 if (capabilities & BDI_CAP_EXEC_MAP)
1037 prot |= PROT_EXEC;
1038 }
1039 }
1040 else if ((prot & PROT_READ) &&
1041 (prot & PROT_EXEC) &&
1042 !(capabilities & BDI_CAP_EXEC_MAP)
1043 ) {
1044 /* backing file is not executable, try to copy */
1045 capabilities &= ~BDI_CAP_MAP_DIRECT;
1046 }
1047 }
1048 else {
1049 /* anonymous mappings are always memory backed and can be
1050 * privately mapped
1051 */
1052 capabilities = BDI_CAP_MAP_COPY;
1053
1054 /* handle PROT_EXEC implication by PROT_READ */
1055 if ((prot & PROT_READ) &&
1056 (current->personality & READ_IMPLIES_EXEC))
1057 prot |= PROT_EXEC;
1058 }
1059
1060 /* allow the security API to have its say */
Al Viroe5467852012-05-30 13:30:51 -04001061 ret = security_mmap_addr(addr);
1062 if (ret < 0)
1063 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064
1065 /* looks okay */
1066 *_capabilities = capabilities;
1067 return 0;
1068}
1069
1070/*
1071 * we've determined that we can make the mapping, now translate what we
1072 * now know into VMA flags
1073 */
1074static unsigned long determine_vm_flags(struct file *file,
1075 unsigned long prot,
1076 unsigned long flags,
1077 unsigned long capabilities)
1078{
1079 unsigned long vm_flags;
1080
1081 vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082 /* vm_flags |= mm->def_flags; */
1083
1084 if (!(capabilities & BDI_CAP_MAP_DIRECT)) {
1085 /* attempt to share read-only copies of mapped file chunks */
Bernd Schmidt3c7b2042010-05-25 23:43:00 -07001086 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087 if (file && !(prot & PROT_WRITE))
1088 vm_flags |= VM_MAYSHARE;
Bernd Schmidt3c7b2042010-05-25 23:43:00 -07001089 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 /* overlay a shareable mapping on the backing device or inode
1091 * if possible - used for chardevs, ramfs/tmpfs/shmfs and
1092 * romfs/cramfs */
Bernd Schmidt3c7b2042010-05-25 23:43:00 -07001093 vm_flags |= VM_MAYSHARE | (capabilities & BDI_CAP_VMFLAGS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 if (flags & MAP_SHARED)
Bernd Schmidt3c7b2042010-05-25 23:43:00 -07001095 vm_flags |= VM_SHARED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096 }
1097
1098 /* refuse to let anyone share private mappings with this process if
1099 * it's being traced - otherwise breakpoints set in it may interfere
1100 * with another untraced process
1101 */
Tejun Heoa288eec2011-06-17 16:50:37 +02001102 if ((flags & MAP_PRIVATE) && current->ptrace)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103 vm_flags &= ~VM_MAYSHARE;
1104
1105 return vm_flags;
1106}
1107
1108/*
David Howells8feae132009-01-08 12:04:47 +00001109 * set up a shared mapping on a file (the driver or filesystem provides and
1110 * pins the storage)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111 */
David Howells8feae132009-01-08 12:04:47 +00001112static int do_mmap_shared_file(struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113{
1114 int ret;
1115
1116 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
Paul Mundtdd8632a2009-01-08 12:04:47 +00001117 if (ret == 0) {
1118 vma->vm_region->vm_top = vma->vm_region->vm_end;
David Howells645d83c2009-09-24 15:13:10 +01001119 return 0;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001120 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 if (ret != -ENOSYS)
1122 return ret;
1123
David Howells3fa30462010-03-23 13:35:21 -07001124 /* getting -ENOSYS indicates that direct mmap isn't possible (as
1125 * opposed to tried but failed) so we can only give a suitable error as
1126 * it's not possible to make a private copy if MAP_SHARED was given */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 return -ENODEV;
1128}
1129
1130/*
1131 * set up a private mapping or an anonymous shared mapping
1132 */
David Howells8feae132009-01-08 12:04:47 +00001133static int do_mmap_private(struct vm_area_struct *vma,
1134 struct vm_region *region,
David Howells645d83c2009-09-24 15:13:10 +01001135 unsigned long len,
1136 unsigned long capabilities)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137{
David Howells8feae132009-01-08 12:04:47 +00001138 struct page *pages;
Bob Liuf67d9b12011-05-24 17:12:56 -07001139 unsigned long total, point, n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140 void *base;
David Howells8feae132009-01-08 12:04:47 +00001141 int ret, order;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142
1143 /* invoke the file's mapping function so that it can keep track of
1144 * shared mappings on devices or memory
1145 * - VM_MAYSHARE will be set if it may attempt to share
1146 */
David Howells645d83c2009-09-24 15:13:10 +01001147 if (capabilities & BDI_CAP_MAP_DIRECT) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
Paul Mundtdd8632a2009-01-08 12:04:47 +00001149 if (ret == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150 /* shouldn't return success if we're not sharing */
Paul Mundtdd8632a2009-01-08 12:04:47 +00001151 BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
1152 vma->vm_region->vm_top = vma->vm_region->vm_end;
David Howells645d83c2009-09-24 15:13:10 +01001153 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154 }
Paul Mundtdd8632a2009-01-08 12:04:47 +00001155 if (ret != -ENOSYS)
1156 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157
1158 /* getting an ENOSYS error indicates that direct mmap isn't
1159 * possible (as opposed to tried but failed) so we'll try to
1160 * make a private copy of the data and map that instead */
1161 }
1162
David Howells8feae132009-01-08 12:04:47 +00001163
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 /* allocate some memory to hold the mapping
1165 * - note that this may not return a page-aligned address if the object
1166 * we're allocating is smaller than a page
1167 */
Bob Liuf67d9b12011-05-24 17:12:56 -07001168 order = get_order(len);
David Howells8feae132009-01-08 12:04:47 +00001169 kdebug("alloc order %d for %lx", order, len);
1170
1171 pages = alloc_pages(GFP_KERNEL, order);
1172 if (!pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 goto enomem;
1174
David Howells8feae132009-01-08 12:04:47 +00001175 total = 1 << order;
David Howells33e5d7692009-04-02 16:56:32 -07001176 atomic_long_add(total, &mmap_pages_allocated);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177
Bob Liuf67d9b12011-05-24 17:12:56 -07001178 point = len >> PAGE_SHIFT;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001179
1180 /* we allocated a power-of-2 sized page set, so we may want to trim off
1181 * the excess */
1182 if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages) {
1183 while (total > point) {
1184 order = ilog2(total - point);
1185 n = 1 << order;
1186 kdebug("shave %lu/%lu @%lu", n, total - point, total);
David Howells33e5d7692009-04-02 16:56:32 -07001187 atomic_long_sub(n, &mmap_pages_allocated);
Paul Mundtdd8632a2009-01-08 12:04:47 +00001188 total -= n;
1189 set_page_refcounted(pages + total);
1190 __free_pages(pages + total, order);
1191 }
David Howells8feae132009-01-08 12:04:47 +00001192 }
1193
David Howells8feae132009-01-08 12:04:47 +00001194 for (point = 1; point < total; point++)
1195 set_page_refcounted(&pages[point]);
1196
1197 base = page_address(pages);
1198 region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
1199 region->vm_start = (unsigned long) base;
Bob Liuf67d9b12011-05-24 17:12:56 -07001200 region->vm_end = region->vm_start + len;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001201 region->vm_top = region->vm_start + (total << PAGE_SHIFT);
David Howells8feae132009-01-08 12:04:47 +00001202
1203 vma->vm_start = region->vm_start;
1204 vma->vm_end = region->vm_start + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205
1206 if (vma->vm_file) {
1207 /* read the contents of a file into the copy */
1208 mm_segment_t old_fs;
1209 loff_t fpos;
1210
1211 fpos = vma->vm_pgoff;
1212 fpos <<= PAGE_SHIFT;
1213
1214 old_fs = get_fs();
1215 set_fs(KERNEL_DS);
Bob Liuf67d9b12011-05-24 17:12:56 -07001216 ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217 set_fs(old_fs);
1218
1219 if (ret < 0)
1220 goto error_free;
1221
1222 /* clear the last little bit */
Bob Liuf67d9b12011-05-24 17:12:56 -07001223 if (ret < len)
1224 memset(base + ret, 0, len - ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 }
1227
1228 return 0;
1229
1230error_free:
Namhyung Kim7223bb42011-05-24 17:11:26 -07001231 free_page_series(region->vm_start, region->vm_top);
David Howells8feae132009-01-08 12:04:47 +00001232 region->vm_start = vma->vm_start = 0;
1233 region->vm_end = vma->vm_end = 0;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001234 region->vm_top = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235 return ret;
1236
1237enomem:
Greg Ungerer05ae6fa2009-01-13 17:30:22 +10001238 printk("Allocation of length %lu from process %d (%s) failed\n",
1239 len, current->pid, current->comm);
David Rientjes7bf02ea2011-05-24 17:11:16 -07001240 show_free_areas(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 return -ENOMEM;
1242}
1243
1244/*
1245 * handle mapping creation for uClinux
1246 */
Al Viroe3fc6292012-05-30 20:08:42 -04001247unsigned long do_mmap_pgoff(struct file *file,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248 unsigned long addr,
1249 unsigned long len,
1250 unsigned long prot,
1251 unsigned long flags,
1252 unsigned long pgoff)
1253{
David Howells8feae132009-01-08 12:04:47 +00001254 struct vm_area_struct *vma;
1255 struct vm_region *region;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 struct rb_node *rb;
David Howells8feae132009-01-08 12:04:47 +00001257 unsigned long capabilities, vm_flags, result;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258 int ret;
1259
David Howells8feae132009-01-08 12:04:47 +00001260 kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff);
1261
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262 /* decide whether we should attempt the mapping, and if so what sort of
1263 * mapping */
1264 ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
1265 &capabilities);
David Howells8feae132009-01-08 12:04:47 +00001266 if (ret < 0) {
1267 kleave(" = %d [val]", ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 return ret;
David Howells8feae132009-01-08 12:04:47 +00001269 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270
David Howells06aab5a2009-09-24 12:33:48 +01001271 /* we ignore the address hint */
1272 addr = 0;
Bob Liuf67d9b12011-05-24 17:12:56 -07001273 len = PAGE_ALIGN(len);
David Howells06aab5a2009-09-24 12:33:48 +01001274
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275 /* we've determined that we can make the mapping, now translate what we
1276 * now know into VMA flags */
1277 vm_flags = determine_vm_flags(file, prot, flags, capabilities);
1278
David Howells8feae132009-01-08 12:04:47 +00001279 /* we're going to need to record the mapping */
1280 region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
1281 if (!region)
1282 goto error_getting_region;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283
David Howells8feae132009-01-08 12:04:47 +00001284 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1285 if (!vma)
1286 goto error_getting_vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287
David Howells1e2ae592010-01-15 17:01:33 -08001288 region->vm_usage = 1;
David Howells8feae132009-01-08 12:04:47 +00001289 region->vm_flags = vm_flags;
1290 region->vm_pgoff = pgoff;
1291
Rik van Riel5beb4932010-03-05 13:42:07 -08001292 INIT_LIST_HEAD(&vma->anon_vma_chain);
David Howells8feae132009-01-08 12:04:47 +00001293 vma->vm_flags = vm_flags;
1294 vma->vm_pgoff = pgoff;
1295
1296 if (file) {
Al Virocb0942b2012-08-27 14:48:26 -04001297 region->vm_file = get_file(file);
1298 vma->vm_file = get_file(file);
David Howells8feae132009-01-08 12:04:47 +00001299 }
1300
1301 down_write(&nommu_region_sem);
1302
1303 /* if we want to share, we need to check for regions created by other
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304 * mmap() calls that overlap with our proposed mapping
David Howells8feae132009-01-08 12:04:47 +00001305 * - we can only share with a superset match on most regular files
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306 * - shared mappings on character devices and memory backed files are
1307 * permitted to overlap inexactly as far as we are concerned for in
1308 * these cases, sharing is handled in the driver or filesystem rather
1309 * than here
1310 */
1311 if (vm_flags & VM_MAYSHARE) {
David Howells8feae132009-01-08 12:04:47 +00001312 struct vm_region *pregion;
1313 unsigned long pglen, rpglen, pgend, rpgend, start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314
David Howells8feae132009-01-08 12:04:47 +00001315 pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1316 pgend = pgoff + pglen;
David Howells165b2392007-03-22 00:11:24 -08001317
David Howells8feae132009-01-08 12:04:47 +00001318 for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
1319 pregion = rb_entry(rb, struct vm_region, vm_rb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320
David Howells8feae132009-01-08 12:04:47 +00001321 if (!(pregion->vm_flags & VM_MAYSHARE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 continue;
1323
1324 /* search for overlapping mappings on the same file */
David Howells8feae132009-01-08 12:04:47 +00001325 if (pregion->vm_file->f_path.dentry->d_inode !=
1326 file->f_path.dentry->d_inode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327 continue;
1328
David Howells8feae132009-01-08 12:04:47 +00001329 if (pregion->vm_pgoff >= pgend)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330 continue;
1331
David Howells8feae132009-01-08 12:04:47 +00001332 rpglen = pregion->vm_end - pregion->vm_start;
1333 rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1334 rpgend = pregion->vm_pgoff + rpglen;
1335 if (pgoff >= rpgend)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 continue;
1337
David Howells8feae132009-01-08 12:04:47 +00001338 /* handle inexactly overlapping matches between
1339 * mappings */
1340 if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
1341 !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1342 /* new mapping is not a subset of the region */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 if (!(capabilities & BDI_CAP_MAP_DIRECT))
1344 goto sharing_violation;
1345 continue;
1346 }
1347
David Howells8feae132009-01-08 12:04:47 +00001348 /* we've found a region we can share */
David Howells1e2ae592010-01-15 17:01:33 -08001349 pregion->vm_usage++;
David Howells8feae132009-01-08 12:04:47 +00001350 vma->vm_region = pregion;
1351 start = pregion->vm_start;
1352 start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
1353 vma->vm_start = start;
1354 vma->vm_end = start + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355
David Howells8feae132009-01-08 12:04:47 +00001356 if (pregion->vm_flags & VM_MAPPED_COPY) {
1357 kdebug("share copy");
1358 vma->vm_flags |= VM_MAPPED_COPY;
1359 } else {
1360 kdebug("share mmap");
1361 ret = do_mmap_shared_file(vma);
1362 if (ret < 0) {
1363 vma->vm_region = NULL;
1364 vma->vm_start = 0;
1365 vma->vm_end = 0;
David Howells1e2ae592010-01-15 17:01:33 -08001366 pregion->vm_usage--;
David Howells8feae132009-01-08 12:04:47 +00001367 pregion = NULL;
1368 goto error_just_free;
1369 }
1370 }
1371 fput(region->vm_file);
1372 kmem_cache_free(vm_region_jar, region);
1373 region = pregion;
1374 result = start;
1375 goto share;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 }
1377
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 /* obtain the address at which to make a shared mapping
1379 * - this is the hook for quasi-memory character devices to
1380 * tell us the location of a shared mapping
1381 */
David Howells645d83c2009-09-24 15:13:10 +01001382 if (capabilities & BDI_CAP_MAP_DIRECT) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 addr = file->f_op->get_unmapped_area(file, addr, len,
1384 pgoff, flags);
Namhyung Kimbb005a52011-05-24 17:11:27 -07001385 if (IS_ERR_VALUE(addr)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386 ret = addr;
Namhyung Kimbb005a52011-05-24 17:11:27 -07001387 if (ret != -ENOSYS)
David Howells8feae132009-01-08 12:04:47 +00001388 goto error_just_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389
1390 /* the driver refused to tell us where to site
1391 * the mapping so we'll have to attempt to copy
1392 * it */
Namhyung Kimbb005a52011-05-24 17:11:27 -07001393 ret = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394 if (!(capabilities & BDI_CAP_MAP_COPY))
David Howells8feae132009-01-08 12:04:47 +00001395 goto error_just_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396
1397 capabilities &= ~BDI_CAP_MAP_DIRECT;
David Howells8feae132009-01-08 12:04:47 +00001398 } else {
1399 vma->vm_start = region->vm_start = addr;
1400 vma->vm_end = region->vm_end = addr + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401 }
1402 }
1403 }
1404
David Howells8feae132009-01-08 12:04:47 +00001405 vma->vm_region = region;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406
David Howells645d83c2009-09-24 15:13:10 +01001407 /* set up the mapping
1408 * - the region is filled in if BDI_CAP_MAP_DIRECT is still set
1409 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 if (file && vma->vm_flags & VM_SHARED)
David Howells8feae132009-01-08 12:04:47 +00001411 ret = do_mmap_shared_file(vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001412 else
David Howells645d83c2009-09-24 15:13:10 +01001413 ret = do_mmap_private(vma, region, len, capabilities);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414 if (ret < 0)
David Howells645d83c2009-09-24 15:13:10 +01001415 goto error_just_free;
1416 add_nommu_region(region);
David Howells8feae132009-01-08 12:04:47 +00001417
Jie Zhangea637632009-12-14 18:00:02 -08001418 /* clear anonymous mappings that don't ask for uninitialized data */
1419 if (!vma->vm_file && !(flags & MAP_UNINITIALIZED))
1420 memset((void *)region->vm_start, 0,
1421 region->vm_end - region->vm_start);
1422
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423 /* okay... we have a mapping; now we have to register it */
David Howells8feae132009-01-08 12:04:47 +00001424 result = vma->vm_start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426 current->mm->total_vm += len >> PAGE_SHIFT;
1427
David Howells8feae132009-01-08 12:04:47 +00001428share:
1429 add_vma_to_mm(current->mm, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430
Mike Frysingercfe79c02010-01-06 17:23:23 +00001431 /* we flush the region from the icache only when the first executable
1432 * mapping of it is made */
1433 if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
1434 flush_icache_range(region->vm_start, region->vm_end);
1435 region->vm_icache_flushed = true;
1436 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001437
Mike Frysingercfe79c02010-01-06 17:23:23 +00001438 up_write(&nommu_region_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439
David Howells8feae132009-01-08 12:04:47 +00001440 kleave(" = %lx", result);
1441 return result;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442
David Howells8feae132009-01-08 12:04:47 +00001443error_just_free:
1444 up_write(&nommu_region_sem);
1445error:
David Howells89a86402009-10-30 13:13:26 +00001446 if (region->vm_file)
1447 fput(region->vm_file);
David Howells8feae132009-01-08 12:04:47 +00001448 kmem_cache_free(vm_region_jar, region);
David Howells89a86402009-10-30 13:13:26 +00001449 if (vma->vm_file)
1450 fput(vma->vm_file);
David Howells8feae132009-01-08 12:04:47 +00001451 kmem_cache_free(vm_area_cachep, vma);
1452 kleave(" = %d", ret);
1453 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454
David Howells8feae132009-01-08 12:04:47 +00001455sharing_violation:
1456 up_write(&nommu_region_sem);
1457 printk(KERN_WARNING "Attempt to share mismatched mappings\n");
1458 ret = -EINVAL;
1459 goto error;
1460
1461error_getting_vma:
1462 kmem_cache_free(vm_region_jar, region);
1463 printk(KERN_WARNING "Allocation of vma for %lu byte allocation"
1464 " from process %d failed\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465 len, current->pid);
David Rientjes7bf02ea2011-05-24 17:11:16 -07001466 show_free_areas(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467 return -ENOMEM;
1468
David Howells8feae132009-01-08 12:04:47 +00001469error_getting_region:
1470 printk(KERN_WARNING "Allocation of vm region for %lu byte allocation"
1471 " from process %d failed\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472 len, current->pid);
David Rientjes7bf02ea2011-05-24 17:11:16 -07001473 show_free_areas(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474 return -ENOMEM;
1475}
Linus Torvalds6be5ceb2012-04-20 17:13:58 -07001476
Hugh Dickins66f0dc42009-12-30 20:17:34 +00001477SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1478 unsigned long, prot, unsigned long, flags,
1479 unsigned long, fd, unsigned long, pgoff)
1480{
1481 struct file *file = NULL;
1482 unsigned long retval = -EBADF;
1483
Al Viro120a7952010-10-30 02:54:44 -04001484 audit_mmap_fd(fd, flags);
Hugh Dickins66f0dc42009-12-30 20:17:34 +00001485 if (!(flags & MAP_ANONYMOUS)) {
1486 file = fget(fd);
1487 if (!file)
1488 goto out;
1489 }
1490
1491 flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1492
Greg Ungererad1ed292012-06-04 14:29:59 +10001493 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
Hugh Dickins66f0dc42009-12-30 20:17:34 +00001494
1495 if (file)
1496 fput(file);
1497out:
1498 return retval;
1499}
1500
Christoph Hellwiga4679372010-03-10 15:21:15 -08001501#ifdef __ARCH_WANT_SYS_OLD_MMAP
1502struct mmap_arg_struct {
1503 unsigned long addr;
1504 unsigned long len;
1505 unsigned long prot;
1506 unsigned long flags;
1507 unsigned long fd;
1508 unsigned long offset;
1509};
1510
1511SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1512{
1513 struct mmap_arg_struct a;
1514
1515 if (copy_from_user(&a, arg, sizeof(a)))
1516 return -EFAULT;
1517 if (a.offset & ~PAGE_MASK)
1518 return -EINVAL;
1519
1520 return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1521 a.offset >> PAGE_SHIFT);
1522}
1523#endif /* __ARCH_WANT_SYS_OLD_MMAP */
1524
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525/*
David Howells8feae132009-01-08 12:04:47 +00001526 * split a vma into two pieces at address 'addr', a new vma is allocated either
1527 * for the first part or the tail.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528 */
David Howells8feae132009-01-08 12:04:47 +00001529int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
1530 unsigned long addr, int new_below)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531{
David Howells8feae132009-01-08 12:04:47 +00001532 struct vm_area_struct *new;
1533 struct vm_region *region;
1534 unsigned long npages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535
David Howells8feae132009-01-08 12:04:47 +00001536 kenter("");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537
David Howells779c1022010-01-15 17:01:34 -08001538 /* we're only permitted to split anonymous regions (these should have
1539 * only a single usage on the region) */
1540 if (vma->vm_file)
David Howells8feae132009-01-08 12:04:47 +00001541 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542
David Howells8feae132009-01-08 12:04:47 +00001543 if (mm->map_count >= sysctl_max_map_count)
1544 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545
David Howells8feae132009-01-08 12:04:47 +00001546 region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1547 if (!region)
1548 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
David Howells8feae132009-01-08 12:04:47 +00001550 new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
1551 if (!new) {
1552 kmem_cache_free(vm_region_jar, region);
1553 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554 }
David Howells8feae132009-01-08 12:04:47 +00001555
1556 /* most fields are the same, copy all, and then fixup */
1557 *new = *vma;
1558 *region = *vma->vm_region;
1559 new->vm_region = region;
1560
1561 npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1562
1563 if (new_below) {
Paul Mundtdd8632a2009-01-08 12:04:47 +00001564 region->vm_top = region->vm_end = new->vm_end = addr;
David Howells8feae132009-01-08 12:04:47 +00001565 } else {
1566 region->vm_start = new->vm_start = addr;
1567 region->vm_pgoff = new->vm_pgoff += npages;
1568 }
1569
1570 if (new->vm_ops && new->vm_ops->open)
1571 new->vm_ops->open(new);
1572
1573 delete_vma_from_mm(vma);
1574 down_write(&nommu_region_sem);
1575 delete_nommu_region(vma->vm_region);
1576 if (new_below) {
1577 vma->vm_region->vm_start = vma->vm_start = addr;
1578 vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1579 } else {
1580 vma->vm_region->vm_end = vma->vm_end = addr;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001581 vma->vm_region->vm_top = addr;
David Howells8feae132009-01-08 12:04:47 +00001582 }
1583 add_nommu_region(vma->vm_region);
1584 add_nommu_region(new->vm_region);
1585 up_write(&nommu_region_sem);
1586 add_vma_to_mm(mm, vma);
1587 add_vma_to_mm(mm, new);
1588 return 0;
1589}
1590
1591/*
1592 * shrink a VMA by removing the specified chunk from either the beginning or
1593 * the end
1594 */
1595static int shrink_vma(struct mm_struct *mm,
1596 struct vm_area_struct *vma,
1597 unsigned long from, unsigned long to)
1598{
1599 struct vm_region *region;
1600
1601 kenter("");
1602
1603 /* adjust the VMA's pointers, which may reposition it in the MM's tree
1604 * and list */
1605 delete_vma_from_mm(vma);
1606 if (from > vma->vm_start)
1607 vma->vm_end = from;
1608 else
1609 vma->vm_start = to;
1610 add_vma_to_mm(mm, vma);
1611
1612 /* cut the backing region down to size */
1613 region = vma->vm_region;
David Howells1e2ae592010-01-15 17:01:33 -08001614 BUG_ON(region->vm_usage != 1);
David Howells8feae132009-01-08 12:04:47 +00001615
1616 down_write(&nommu_region_sem);
1617 delete_nommu_region(region);
Paul Mundtdd8632a2009-01-08 12:04:47 +00001618 if (from > region->vm_start) {
1619 to = region->vm_top;
1620 region->vm_top = region->vm_end = from;
1621 } else {
David Howells8feae132009-01-08 12:04:47 +00001622 region->vm_start = to;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001623 }
David Howells8feae132009-01-08 12:04:47 +00001624 add_nommu_region(region);
1625 up_write(&nommu_region_sem);
1626
1627 free_page_series(from, to);
1628 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629}
1630
David Howells30340972006-09-27 01:50:20 -07001631/*
1632 * release a mapping
David Howells8feae132009-01-08 12:04:47 +00001633 * - under NOMMU conditions the chunk to be unmapped must be backed by a single
1634 * VMA, though it need not cover the whole VMA
David Howells30340972006-09-27 01:50:20 -07001635 */
David Howells8feae132009-01-08 12:04:47 +00001636int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637{
David Howells8feae132009-01-08 12:04:47 +00001638 struct vm_area_struct *vma;
Bob Liuf67d9b12011-05-24 17:12:56 -07001639 unsigned long end;
David Howells8feae132009-01-08 12:04:47 +00001640 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641
David Howells8feae132009-01-08 12:04:47 +00001642 kenter(",%lx,%zx", start, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643
Bob Liuf67d9b12011-05-24 17:12:56 -07001644 len = PAGE_ALIGN(len);
David Howells8feae132009-01-08 12:04:47 +00001645 if (len == 0)
1646 return -EINVAL;
1647
Bob Liuf67d9b12011-05-24 17:12:56 -07001648 end = start + len;
1649
David Howells8feae132009-01-08 12:04:47 +00001650 /* find the first potentially overlapping VMA */
1651 vma = find_vma(mm, start);
1652 if (!vma) {
David Howells33e5d7692009-04-02 16:56:32 -07001653 static int limit = 0;
1654 if (limit < 5) {
1655 printk(KERN_WARNING
1656 "munmap of memory not mmapped by process %d"
1657 " (%s): 0x%lx-0x%lx\n",
1658 current->pid, current->comm,
1659 start, start + len - 1);
1660 limit++;
1661 }
David Howells8feae132009-01-08 12:04:47 +00001662 return -EINVAL;
David Howells30340972006-09-27 01:50:20 -07001663 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664
David Howells8feae132009-01-08 12:04:47 +00001665 /* we're allowed to split an anonymous VMA but not a file-backed one */
1666 if (vma->vm_file) {
1667 do {
1668 if (start > vma->vm_start) {
1669 kleave(" = -EINVAL [miss]");
1670 return -EINVAL;
1671 }
1672 if (end == vma->vm_end)
1673 goto erase_whole_vma;
Namhyung Kimd75a3102011-05-24 17:11:25 -07001674 vma = vma->vm_next;
1675 } while (vma);
David Howells8feae132009-01-08 12:04:47 +00001676 kleave(" = -EINVAL [split file]");
1677 return -EINVAL;
1678 } else {
1679 /* the chunk must be a subset of the VMA found */
1680 if (start == vma->vm_start && end == vma->vm_end)
1681 goto erase_whole_vma;
1682 if (start < vma->vm_start || end > vma->vm_end) {
1683 kleave(" = -EINVAL [superset]");
1684 return -EINVAL;
1685 }
1686 if (start & ~PAGE_MASK) {
1687 kleave(" = -EINVAL [unaligned start]");
1688 return -EINVAL;
1689 }
1690 if (end != vma->vm_end && end & ~PAGE_MASK) {
1691 kleave(" = -EINVAL [unaligned split]");
1692 return -EINVAL;
1693 }
1694 if (start != vma->vm_start && end != vma->vm_end) {
1695 ret = split_vma(mm, vma, start, 1);
1696 if (ret < 0) {
1697 kleave(" = %d [split]", ret);
1698 return ret;
1699 }
1700 }
1701 return shrink_vma(mm, vma, start, end);
1702 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703
David Howells8feae132009-01-08 12:04:47 +00001704erase_whole_vma:
1705 delete_vma_from_mm(vma);
1706 delete_vma(mm, vma);
1707 kleave(" = 0");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001708 return 0;
1709}
Paul Mundtb5073172007-07-21 04:37:25 -07001710EXPORT_SYMBOL(do_munmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711
Al Virobfce2812012-04-20 21:57:04 -04001712int vm_munmap(unsigned long addr, size_t len)
David Howells30340972006-09-27 01:50:20 -07001713{
Al Virobfce2812012-04-20 21:57:04 -04001714 struct mm_struct *mm = current->mm;
David Howells30340972006-09-27 01:50:20 -07001715 int ret;
David Howells30340972006-09-27 01:50:20 -07001716
1717 down_write(&mm->mmap_sem);
1718 ret = do_munmap(mm, addr, len);
1719 up_write(&mm->mmap_sem);
1720 return ret;
1721}
Linus Torvaldsa46ef992012-04-20 16:20:01 -07001722EXPORT_SYMBOL(vm_munmap);
1723
1724SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1725{
Al Virobfce2812012-04-20 21:57:04 -04001726 return vm_munmap(addr, len);
Linus Torvaldsa46ef992012-04-20 16:20:01 -07001727}
David Howells30340972006-09-27 01:50:20 -07001728
1729/*
David Howells8feae132009-01-08 12:04:47 +00001730 * release all the mappings made in a process's VM space
David Howells30340972006-09-27 01:50:20 -07001731 */
David Howells8feae132009-01-08 12:04:47 +00001732void exit_mmap(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001733{
David Howells8feae132009-01-08 12:04:47 +00001734 struct vm_area_struct *vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735
David Howells8feae132009-01-08 12:04:47 +00001736 if (!mm)
1737 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738
David Howells8feae132009-01-08 12:04:47 +00001739 kenter("");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740
David Howells8feae132009-01-08 12:04:47 +00001741 mm->total_vm = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742
David Howells8feae132009-01-08 12:04:47 +00001743 while ((vma = mm->mmap)) {
1744 mm->mmap = vma->vm_next;
1745 delete_vma_from_mm(vma);
1746 delete_vma(mm, vma);
Steven J. Magnani04c34962010-11-24 12:56:54 -08001747 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748 }
David Howells8feae132009-01-08 12:04:47 +00001749
1750 kleave("");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751}
1752
Linus Torvaldse4eb1ff2012-04-20 15:35:40 -07001753unsigned long vm_brk(unsigned long addr, unsigned long len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754{
1755 return -ENOMEM;
1756}
1757
1758/*
David Howells6fa5f802006-09-27 01:50:21 -07001759 * expand (or shrink) an existing mapping, potentially moving it at the same
1760 * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761 *
David Howells6fa5f802006-09-27 01:50:21 -07001762 * under NOMMU conditions, we only permit changing a mapping's size, and only
David Howells8feae132009-01-08 12:04:47 +00001763 * as long as it stays within the region allocated by do_mmap_private() and the
1764 * block is not shareable
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765 *
David Howells6fa5f802006-09-27 01:50:21 -07001766 * MREMAP_FIXED is not supported under NOMMU conditions
Linus Torvalds1da177e2005-04-16 15:20:36 -07001767 */
1768unsigned long do_mremap(unsigned long addr,
1769 unsigned long old_len, unsigned long new_len,
1770 unsigned long flags, unsigned long new_addr)
1771{
David Howells6fa5f802006-09-27 01:50:21 -07001772 struct vm_area_struct *vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773
1774 /* insanity checks first */
Bob Liuf67d9b12011-05-24 17:12:56 -07001775 old_len = PAGE_ALIGN(old_len);
1776 new_len = PAGE_ALIGN(new_len);
David Howells8feae132009-01-08 12:04:47 +00001777 if (old_len == 0 || new_len == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778 return (unsigned long) -EINVAL;
1779
David Howells8feae132009-01-08 12:04:47 +00001780 if (addr & ~PAGE_MASK)
1781 return -EINVAL;
1782
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783 if (flags & MREMAP_FIXED && new_addr != addr)
1784 return (unsigned long) -EINVAL;
1785
David Howells8feae132009-01-08 12:04:47 +00001786 vma = find_vma_exact(current->mm, addr, old_len);
David Howells6fa5f802006-09-27 01:50:21 -07001787 if (!vma)
1788 return (unsigned long) -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001789
David Howells6fa5f802006-09-27 01:50:21 -07001790 if (vma->vm_end != vma->vm_start + old_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 return (unsigned long) -EFAULT;
1792
David Howells6fa5f802006-09-27 01:50:21 -07001793 if (vma->vm_flags & VM_MAYSHARE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001794 return (unsigned long) -EPERM;
1795
David Howells8feae132009-01-08 12:04:47 +00001796 if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001797 return (unsigned long) -ENOMEM;
1798
1799 /* all checks complete - do it */
David Howells6fa5f802006-09-27 01:50:21 -07001800 vma->vm_end = vma->vm_start + new_len;
David Howells6fa5f802006-09-27 01:50:21 -07001801 return vma->vm_start;
1802}
Paul Mundtb5073172007-07-21 04:37:25 -07001803EXPORT_SYMBOL(do_mremap);
David Howells6fa5f802006-09-27 01:50:21 -07001804
Heiko Carstens6a6160a2009-01-14 14:14:15 +01001805SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
1806 unsigned long, new_len, unsigned long, flags,
1807 unsigned long, new_addr)
David Howells6fa5f802006-09-27 01:50:21 -07001808{
1809 unsigned long ret;
1810
1811 down_write(&current->mm->mmap_sem);
1812 ret = do_mremap(addr, old_len, new_len, flags, new_addr);
1813 up_write(&current->mm->mmap_sem);
1814 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815}
1816
Linus Torvalds6aab3412005-11-28 14:34:23 -08001817struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
Hugh Dickinsdeceb6c2005-10-29 18:16:33 -07001818 unsigned int foll_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819{
1820 return NULL;
1821}
1822
Bob Liu8f3b1322011-07-08 15:39:46 -07001823int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1824 unsigned long pfn, unsigned long size, pgprot_t prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825{
Bob Liu8f3b1322011-07-08 15:39:46 -07001826 if (addr != (pfn << PAGE_SHIFT))
1827 return -EINVAL;
1828
Konstantin Khlebnikov314e51b2012-10-08 16:29:02 -07001829 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
Greg Ungerer66aa2b42005-09-12 11:18:10 +10001830 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831}
Luke Yang22c4af42006-07-14 00:24:09 -07001832EXPORT_SYMBOL(remap_pfn_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833
Paul Mundtf905bc42008-02-04 22:29:59 -08001834int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
1835 unsigned long pgoff)
1836{
1837 unsigned int size = vma->vm_end - vma->vm_start;
1838
1839 if (!(vma->vm_flags & VM_USERMAP))
1840 return -EINVAL;
1841
1842 vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
1843 vma->vm_end = vma->vm_start + size;
1844
1845 return 0;
1846}
1847EXPORT_SYMBOL(remap_vmalloc_range);
1848
Linus Torvalds1da177e2005-04-16 15:20:36 -07001849unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
1850 unsigned long len, unsigned long pgoff, unsigned long flags)
1851{
1852 return -ENOMEM;
1853}
1854
Wolfgang Wander1363c3c2005-06-21 17:14:49 -07001855void arch_unmap_area(struct mm_struct *mm, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856{
1857}
1858
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859void unmap_mapping_range(struct address_space *mapping,
1860 loff_t const holebegin, loff_t const holelen,
1861 int even_cows)
1862{
1863}
Luke Yang22c4af42006-07-14 00:24:09 -07001864EXPORT_SYMBOL(unmap_mapping_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865
1866/*
1867 * Check that a process has enough memory to allocate a new virtual
1868 * mapping. 0 means there is enough memory for the allocation to
1869 * succeed and -ENOMEM implies there is not.
1870 *
1871 * We currently support three overcommit policies, which are set via the
1872 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
1873 *
1874 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
1875 * Additional code 2002 Jul 20 by Robert Love.
1876 *
1877 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
1878 *
1879 * Note this is a helper function intended to be used by LSMs which
1880 * wish to use this logic.
1881 */
Alan Cox34b4e4a2007-08-22 14:01:28 -07001882int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001883{
1884 unsigned long free, allowed;
1885
1886 vm_acct_memory(pages);
1887
1888 /*
1889 * Sometimes we want to use more memory than we have
1890 */
1891 if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
1892 return 0;
1893
1894 if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
Dmitry Finkc15bef32011-07-25 17:12:19 -07001895 free = global_page_state(NR_FREE_PAGES);
1896 free += global_page_state(NR_FILE_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897
Dmitry Finkc15bef32011-07-25 17:12:19 -07001898 /*
1899 * shmem pages shouldn't be counted as free in this
1900 * case, they can't be purged, only swapped out, and
1901 * that won't affect the overall amount of available
1902 * memory in the system.
1903 */
1904 free -= global_page_state(NR_SHMEM);
1905
Linus Torvalds1da177e2005-04-16 15:20:36 -07001906 free += nr_swap_pages;
1907
1908 /*
1909 * Any slabs which are created with the
1910 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
1911 * which are reclaimable, under pressure. The dentry
1912 * cache and most inode caches should fall into this
1913 */
Christoph Lameter972d1a72006-09-25 23:31:51 -07001914 free += global_page_state(NR_SLAB_RECLAIMABLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915
1916 /*
Dmitry Finkc15bef32011-07-25 17:12:19 -07001917 * Leave reserved pages. The pages are not for anonymous pages.
1918 */
1919 if (free <= totalreserve_pages)
1920 goto error;
1921 else
1922 free -= totalreserve_pages;
1923
1924 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001925 * Leave the last 3% for root
1926 */
1927 if (!cap_sys_admin)
1928 free -= free / 32;
1929
1930 if (free > pages)
1931 return 0;
1932
Hideo AOKId5ddc792006-04-10 22:53:01 -07001933 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934 }
1935
1936 allowed = totalram_pages * sysctl_overcommit_ratio / 100;
1937 /*
1938 * Leave the last 3% for root
1939 */
1940 if (!cap_sys_admin)
1941 allowed -= allowed / 32;
1942 allowed += total_swap_pages;
1943
1944 /* Don't let a single process grow too big:
1945 leave 3% of the size of this process for other processes */
Alan Cox731572d2008-10-29 14:01:20 -07001946 if (mm)
1947 allowed -= mm->total_vm / 32;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948
KOSAKI Motohiro00a62ce2009-04-30 15:08:51 -07001949 if (percpu_counter_read_positive(&vm_committed_as) < allowed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950 return 0;
KOSAKI Motohiro00a62ce2009-04-30 15:08:51 -07001951
Hideo AOKId5ddc792006-04-10 22:53:01 -07001952error:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953 vm_unacct_memory(pages);
1954
1955 return -ENOMEM;
1956}
1957
Stephen Wilsoncae5d392011-03-13 15:49:17 -04001958int in_gate_area_no_mm(unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001959{
1960 return 0;
1961}
David Howellsb0e15192006-01-06 00:11:42 -08001962
Nick Piggind0217ac2007-07-19 01:47:03 -07001963int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
David Howellsb0e15192006-01-06 00:11:42 -08001964{
1965 BUG();
Nick Piggind0217ac2007-07-19 01:47:03 -07001966 return 0;
David Howellsb0e15192006-01-06 00:11:42 -08001967}
Paul Mundtb5073172007-07-21 04:37:25 -07001968EXPORT_SYMBOL(filemap_fault);
David Howells0ec76a12006-09-27 01:50:15 -07001969
Konstantin Khlebnikov0b173bc2012-10-08 16:28:46 -07001970int generic_file_remap_pages(struct vm_area_struct *vma, unsigned long addr,
1971 unsigned long size, pgoff_t pgoff)
1972{
1973 BUG();
1974 return 0;
1975}
1976EXPORT_SYMBOL(generic_file_remap_pages);
1977
Mike Frysingerf55f1992011-03-29 14:05:12 +01001978static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
1979 unsigned long addr, void *buf, int len, int write)
David Howells0ec76a12006-09-27 01:50:15 -07001980{
David Howells0ec76a12006-09-27 01:50:15 -07001981 struct vm_area_struct *vma;
David Howells0ec76a12006-09-27 01:50:15 -07001982
1983 down_read(&mm->mmap_sem);
1984
1985 /* the access must start within one of the target process's mappings */
David Howells0159b142006-09-27 01:50:16 -07001986 vma = find_vma(mm, addr);
1987 if (vma) {
David Howells0ec76a12006-09-27 01:50:15 -07001988 /* don't overrun this mapping */
1989 if (addr + len >= vma->vm_end)
1990 len = vma->vm_end - addr;
1991
1992 /* only read or write mappings where it is permitted */
David Howellsd00c7b992006-09-27 01:50:19 -07001993 if (write && vma->vm_flags & VM_MAYWRITE)
Jie Zhang79597222010-01-06 17:23:28 +00001994 copy_to_user_page(vma, NULL, addr,
1995 (void *) addr, buf, len);
David Howellsd00c7b992006-09-27 01:50:19 -07001996 else if (!write && vma->vm_flags & VM_MAYREAD)
Jie Zhang79597222010-01-06 17:23:28 +00001997 copy_from_user_page(vma, NULL, addr,
1998 buf, (void *) addr, len);
David Howells0ec76a12006-09-27 01:50:15 -07001999 else
2000 len = 0;
2001 } else {
2002 len = 0;
2003 }
2004
2005 up_read(&mm->mmap_sem);
Mike Frysingerf55f1992011-03-29 14:05:12 +01002006
2007 return len;
2008}
2009
2010/**
2011 * @access_remote_vm - access another process' address space
2012 * @mm: the mm_struct of the target address space
2013 * @addr: start address to access
2014 * @buf: source or destination buffer
2015 * @len: number of bytes to transfer
2016 * @write: whether the access is a write
2017 *
2018 * The caller must hold a reference on @mm.
2019 */
2020int access_remote_vm(struct mm_struct *mm, unsigned long addr,
2021 void *buf, int len, int write)
2022{
2023 return __access_remote_vm(NULL, mm, addr, buf, len, write);
2024}
2025
2026/*
2027 * Access another process' address space.
2028 * - source/target buffer must be kernel space
2029 */
2030int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
2031{
2032 struct mm_struct *mm;
2033
2034 if (addr + len < addr)
2035 return 0;
2036
2037 mm = get_task_mm(tsk);
2038 if (!mm)
2039 return 0;
2040
2041 len = __access_remote_vm(tsk, mm, addr, buf, len, write);
2042
David Howells0ec76a12006-09-27 01:50:15 -07002043 mmput(mm);
2044 return len;
2045}
David Howells7e660872010-01-15 17:01:39 -08002046
2047/**
2048 * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
2049 * @inode: The inode to check
2050 * @size: The current filesize of the inode
2051 * @newsize: The proposed filesize of the inode
2052 *
2053 * Check the shared mappings on an inode on behalf of a shrinking truncate to
2054 * make sure that that any outstanding VMAs aren't broken and then shrink the
2055 * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
2056 * automatically grant mappings that are too large.
2057 */
2058int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
2059 size_t newsize)
2060{
2061 struct vm_area_struct *vma;
David Howells7e660872010-01-15 17:01:39 -08002062 struct vm_region *region;
2063 pgoff_t low, high;
2064 size_t r_size, r_top;
2065
2066 low = newsize >> PAGE_SHIFT;
2067 high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2068
2069 down_write(&nommu_region_sem);
David Howells918e5562012-02-23 13:50:35 +00002070 mutex_lock(&inode->i_mapping->i_mmap_mutex);
David Howells7e660872010-01-15 17:01:39 -08002071
2072 /* search for VMAs that fall within the dead zone */
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07002073 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
David Howells7e660872010-01-15 17:01:39 -08002074 /* found one - only interested if it's shared out of the page
2075 * cache */
2076 if (vma->vm_flags & VM_SHARED) {
David Howells918e5562012-02-23 13:50:35 +00002077 mutex_unlock(&inode->i_mapping->i_mmap_mutex);
David Howells7e660872010-01-15 17:01:39 -08002078 up_write(&nommu_region_sem);
2079 return -ETXTBSY; /* not quite true, but near enough */
2080 }
2081 }
2082
2083 /* reduce any regions that overlap the dead zone - if in existence,
2084 * these will be pointed to by VMAs that don't overlap the dead zone
2085 *
2086 * we don't check for any regions that start beyond the EOF as there
2087 * shouldn't be any
2088 */
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07002089 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap,
2090 0, ULONG_MAX) {
David Howells7e660872010-01-15 17:01:39 -08002091 if (!(vma->vm_flags & VM_SHARED))
2092 continue;
2093
2094 region = vma->vm_region;
2095 r_size = region->vm_top - region->vm_start;
2096 r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
2097
2098 if (r_top > newsize) {
2099 region->vm_top -= r_top - newsize;
2100 if (region->vm_end > region->vm_top)
2101 region->vm_end = region->vm_top;
2102 }
2103 }
2104
David Howells918e5562012-02-23 13:50:35 +00002105 mutex_unlock(&inode->i_mapping->i_mmap_mutex);
David Howells7e660872010-01-15 17:01:39 -08002106 up_write(&nommu_region_sem);
2107 return 0;
2108}