blob: 3035a56e7616d61a63d1f7fea916091794ca4a28 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/mlock.c
3 *
4 * (C) Copyright 1995 Linus Torvalds
5 * (C) Copyright 2002 Christoph Hellwig
6 */
7
Randy.Dunlapc59ede72006-01-11 12:17:46 -08008#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/mman.h>
10#include <linux/mm.h>
Nick Pigginb291f002008-10-18 20:26:44 -070011#include <linux/swap.h>
12#include <linux/swapops.h>
13#include <linux/pagemap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/mempolicy.h>
15#include <linux/syscalls.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040016#include <linux/sched.h>
17#include <linux/module.h>
Nick Pigginb291f002008-10-18 20:26:44 -070018#include <linux/rmap.h>
19#include <linux/mmzone.h>
20#include <linux/hugetlb.h>
21
22#include "internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040024int can_do_mlock(void)
25{
26 if (capable(CAP_IPC_LOCK))
27 return 1;
28 if (current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur != 0)
29 return 1;
30 return 0;
31}
32EXPORT_SYMBOL(can_do_mlock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
Nick Pigginb291f002008-10-18 20:26:44 -070034#ifdef CONFIG_UNEVICTABLE_LRU
35/*
36 * Mlocked pages are marked with PageMlocked() flag for efficient testing
37 * in vmscan and, possibly, the fault path; and to support semi-accurate
38 * statistics.
39 *
40 * An mlocked page [PageMlocked(page)] is unevictable. As such, it will
41 * be placed on the LRU "unevictable" list, rather than the [in]active lists.
42 * The unevictable list is an LRU sibling list to the [in]active lists.
43 * PageUnevictable is set to indicate the unevictable state.
44 *
45 * When lazy mlocking via vmscan, it is important to ensure that the
46 * vma's VM_LOCKED status is not concurrently being modified, otherwise we
47 * may have mlocked a page that is being munlocked. So lazy mlock must take
48 * the mmap_sem for read, and verify that the vma really is locked
49 * (see mm/rmap.c).
50 */
51
52/*
53 * LRU accounting for clear_page_mlock()
54 */
55void __clear_page_mlock(struct page *page)
56{
57 VM_BUG_ON(!PageLocked(page));
58
59 if (!page->mapping) { /* truncated ? */
60 return;
61 }
62
Nick Piggin5344b7e2008-10-18 20:26:51 -070063 dec_zone_page_state(page, NR_MLOCK);
64 count_vm_event(UNEVICTABLE_PGCLEARED);
Nick Pigginb291f002008-10-18 20:26:44 -070065 if (!isolate_lru_page(page)) {
66 putback_lru_page(page);
67 } else {
68 /*
KOSAKI Motohiro8891d6d2008-11-12 13:26:53 -080069 * We lost the race. the page already moved to evictable list.
Nick Pigginb291f002008-10-18 20:26:44 -070070 */
KOSAKI Motohiro8891d6d2008-11-12 13:26:53 -080071 if (PageUnevictable(page))
Nick Piggin5344b7e2008-10-18 20:26:51 -070072 count_vm_event(UNEVICTABLE_PGSTRANDED);
Nick Pigginb291f002008-10-18 20:26:44 -070073 }
74}
75
76/*
77 * Mark page as mlocked if not already.
78 * If page on LRU, isolate and putback to move to unevictable list.
79 */
80void mlock_vma_page(struct page *page)
81{
82 BUG_ON(!PageLocked(page));
83
Nick Piggin5344b7e2008-10-18 20:26:51 -070084 if (!TestSetPageMlocked(page)) {
85 inc_zone_page_state(page, NR_MLOCK);
86 count_vm_event(UNEVICTABLE_PGMLOCKED);
87 if (!isolate_lru_page(page))
88 putback_lru_page(page);
89 }
Nick Pigginb291f002008-10-18 20:26:44 -070090}
91
92/*
93 * called from munlock()/munmap() path with page supposedly on the LRU.
94 *
95 * Note: unlike mlock_vma_page(), we can't just clear the PageMlocked
96 * [in try_to_munlock()] and then attempt to isolate the page. We must
97 * isolate the page to keep others from messing with its unevictable
98 * and mlocked state while trying to munlock. However, we pre-clear the
99 * mlocked state anyway as we might lose the isolation race and we might
100 * not get another chance to clear PageMlocked. If we successfully
101 * isolate the page and try_to_munlock() detects other VM_LOCKED vmas
102 * mapping the page, it will restore the PageMlocked state, unless the page
103 * is mapped in a non-linear vma. So, we go ahead and SetPageMlocked(),
104 * perhaps redundantly.
105 * If we lose the isolation race, and the page is mapped by other VM_LOCKED
106 * vmas, we'll detect this in vmscan--via try_to_munlock() or try_to_unmap()
107 * either of which will restore the PageMlocked state by calling
108 * mlock_vma_page() above, if it can grab the vma's mmap sem.
109 */
110static void munlock_vma_page(struct page *page)
111{
112 BUG_ON(!PageLocked(page));
113
Nick Piggin5344b7e2008-10-18 20:26:51 -0700114 if (TestClearPageMlocked(page)) {
115 dec_zone_page_state(page, NR_MLOCK);
116 if (!isolate_lru_page(page)) {
117 int ret = try_to_munlock(page);
118 /*
119 * did try_to_unlock() succeed or punt?
120 */
121 if (ret == SWAP_SUCCESS || ret == SWAP_AGAIN)
122 count_vm_event(UNEVICTABLE_PGMUNLOCKED);
123
124 putback_lru_page(page);
125 } else {
126 /*
127 * We lost the race. let try_to_unmap() deal
128 * with it. At least we get the page state and
129 * mlock stats right. However, page is still on
130 * the noreclaim list. We'll fix that up when
131 * the page is eventually freed or we scan the
132 * noreclaim list.
133 */
134 if (PageUnevictable(page))
135 count_vm_event(UNEVICTABLE_PGSTRANDED);
136 else
137 count_vm_event(UNEVICTABLE_PGMUNLOCKED);
138 }
Nick Pigginb291f002008-10-18 20:26:44 -0700139 }
140}
141
Rik van Rielba470de2008-10-18 20:26:50 -0700142/**
143 * __mlock_vma_pages_range() - mlock/munlock a range of pages in the vma.
144 * @vma: target vma
145 * @start: start address
146 * @end: end address
147 * @mlock: 0 indicate munlock, otherwise mlock.
Nick Pigginb291f002008-10-18 20:26:44 -0700148 *
Rik van Rielba470de2008-10-18 20:26:50 -0700149 * If @mlock == 0, unlock an mlocked range;
150 * else mlock the range of pages. This takes care of making the pages present ,
151 * too.
Nick Pigginb291f002008-10-18 20:26:44 -0700152 *
Rik van Rielba470de2008-10-18 20:26:50 -0700153 * return 0 on success, negative error code on error.
154 *
155 * vma->vm_mm->mmap_sem must be held for at least read.
Nick Pigginb291f002008-10-18 20:26:44 -0700156 */
Rik van Rielba470de2008-10-18 20:26:50 -0700157static long __mlock_vma_pages_range(struct vm_area_struct *vma,
158 unsigned long start, unsigned long end,
159 int mlock)
Nick Pigginb291f002008-10-18 20:26:44 -0700160{
161 struct mm_struct *mm = vma->vm_mm;
162 unsigned long addr = start;
163 struct page *pages[16]; /* 16 gives a reasonable batch */
Nick Pigginb291f002008-10-18 20:26:44 -0700164 int nr_pages = (end - start) / PAGE_SIZE;
Helge Deller72eb8c62008-11-17 00:30:57 +0100165 int ret = 0;
Rik van Rielba470de2008-10-18 20:26:50 -0700166 int gup_flags = 0;
Nick Pigginb291f002008-10-18 20:26:44 -0700167
Rik van Rielba470de2008-10-18 20:26:50 -0700168 VM_BUG_ON(start & ~PAGE_MASK);
169 VM_BUG_ON(end & ~PAGE_MASK);
170 VM_BUG_ON(start < vma->vm_start);
171 VM_BUG_ON(end > vma->vm_end);
172 VM_BUG_ON((!rwsem_is_locked(&mm->mmap_sem)) &&
173 (atomic_read(&mm->mm_users) != 0));
174
175 /*
176 * mlock: don't page populate if page has PROT_NONE permission.
177 * munlock: the pages always do munlock althrough
178 * its has PROT_NONE permission.
179 */
180 if (!mlock)
181 gup_flags |= GUP_FLAGS_IGNORE_VMA_PERMISSIONS;
182
183 if (vma->vm_flags & VM_WRITE)
184 gup_flags |= GUP_FLAGS_WRITE;
Nick Pigginb291f002008-10-18 20:26:44 -0700185
Nick Pigginb291f002008-10-18 20:26:44 -0700186 while (nr_pages > 0) {
187 int i;
188
189 cond_resched();
190
191 /*
192 * get_user_pages makes pages present if we are
193 * setting mlock. and this extra reference count will
194 * disable migration of this page. However, page may
195 * still be truncated out from under us.
196 */
Rik van Rielba470de2008-10-18 20:26:50 -0700197 ret = __get_user_pages(current, mm, addr,
Nick Pigginb291f002008-10-18 20:26:44 -0700198 min_t(int, nr_pages, ARRAY_SIZE(pages)),
Rik van Rielba470de2008-10-18 20:26:50 -0700199 gup_flags, pages, NULL);
Nick Pigginb291f002008-10-18 20:26:44 -0700200 /*
201 * This can happen for, e.g., VM_NONLINEAR regions before
202 * a page has been allocated and mapped at a given offset,
203 * or for addresses that map beyond end of a file.
204 * We'll mlock the the pages if/when they get faulted in.
205 */
206 if (ret < 0)
207 break;
208 if (ret == 0) {
209 /*
210 * We know the vma is there, so the only time
211 * we cannot get a single page should be an
212 * error (ret < 0) case.
213 */
214 WARN_ON(1);
215 break;
216 }
217
218 lru_add_drain(); /* push cached pages to LRU */
219
220 for (i = 0; i < ret; i++) {
221 struct page *page = pages[i];
222
223 lock_page(page);
224 /*
225 * Because we lock page here and migration is blocked
226 * by the elevated reference, we need only check for
227 * page truncation (file-cache only).
228 */
Rik van Rielba470de2008-10-18 20:26:50 -0700229 if (page->mapping) {
230 if (mlock)
231 mlock_vma_page(page);
232 else
233 munlock_vma_page(page);
234 }
Nick Pigginb291f002008-10-18 20:26:44 -0700235 unlock_page(page);
236 put_page(page); /* ref from get_user_pages() */
237
238 /*
239 * here we assume that get_user_pages() has given us
240 * a list of virtually contiguous pages.
241 */
242 addr += PAGE_SIZE; /* for next get_user_pages() */
243 nr_pages--;
244 }
Lee Schermerhorn9978ad52008-10-18 20:26:56 -0700245 ret = 0;
Nick Pigginb291f002008-10-18 20:26:44 -0700246 }
247
Lee Schermerhorn9978ad52008-10-18 20:26:56 -0700248 return ret; /* count entire vma as locked_vm */
249}
250
251/*
252 * convert get_user_pages() return value to posix mlock() error
253 */
254static int __mlock_posix_error_return(long retval)
255{
256 if (retval == -EFAULT)
257 retval = -ENOMEM;
258 else if (retval == -ENOMEM)
259 retval = -EAGAIN;
260 return retval;
Nick Pigginb291f002008-10-18 20:26:44 -0700261}
262
Nick Pigginb291f002008-10-18 20:26:44 -0700263#else /* CONFIG_UNEVICTABLE_LRU */
264
265/*
266 * Just make pages present if VM_LOCKED. No-op if unlocking.
267 */
Rik van Rielba470de2008-10-18 20:26:50 -0700268static long __mlock_vma_pages_range(struct vm_area_struct *vma,
269 unsigned long start, unsigned long end,
270 int mlock)
Nick Pigginb291f002008-10-18 20:26:44 -0700271{
Rik van Rielba470de2008-10-18 20:26:50 -0700272 if (mlock && (vma->vm_flags & VM_LOCKED))
Lee Schermerhorn9978ad52008-10-18 20:26:56 -0700273 return make_pages_present(start, end);
Nick Pigginb291f002008-10-18 20:26:44 -0700274 return 0;
275}
Lee Schermerhorn9978ad52008-10-18 20:26:56 -0700276
277static inline int __mlock_posix_error_return(long retval)
278{
279 return 0;
280}
281
Nick Pigginb291f002008-10-18 20:26:44 -0700282#endif /* CONFIG_UNEVICTABLE_LRU */
283
Rik van Rielba470de2008-10-18 20:26:50 -0700284/**
285 * mlock_vma_pages_range() - mlock pages in specified vma range.
286 * @vma - the vma containing the specfied address range
287 * @start - starting address in @vma to mlock
288 * @end - end address [+1] in @vma to mlock
289 *
290 * For mmap()/mremap()/expansion of mlocked vma.
291 *
292 * return 0 on success for "normal" vmas.
293 *
294 * return number of pages [> 0] to be removed from locked_vm on success
295 * of "special" vmas.
296 *
297 * return negative error if vma spanning @start-@range disappears while
298 * mmap semaphore is dropped. Unlikely?
Nick Pigginb291f002008-10-18 20:26:44 -0700299 */
Rik van Rielba470de2008-10-18 20:26:50 -0700300long mlock_vma_pages_range(struct vm_area_struct *vma,
Nick Pigginb291f002008-10-18 20:26:44 -0700301 unsigned long start, unsigned long end)
302{
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700303 struct mm_struct *mm = vma->vm_mm;
Nick Pigginb291f002008-10-18 20:26:44 -0700304 int nr_pages = (end - start) / PAGE_SIZE;
305 BUG_ON(!(vma->vm_flags & VM_LOCKED));
306
307 /*
308 * filter unlockable vmas
309 */
310 if (vma->vm_flags & (VM_IO | VM_PFNMAP))
311 goto no_mlock;
312
313 if (!((vma->vm_flags & (VM_DONTEXPAND | VM_RESERVED)) ||
314 is_vm_hugetlb_page(vma) ||
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700315 vma == get_gate_vma(current))) {
Rik van Rielba470de2008-10-18 20:26:50 -0700316 long error;
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700317 downgrade_write(&mm->mmap_sem);
Rik van Rielba470de2008-10-18 20:26:50 -0700318
319 error = __mlock_vma_pages_range(vma, start, end, 1);
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700320
321 up_read(&mm->mmap_sem);
322 /* vma can change or disappear */
323 down_write(&mm->mmap_sem);
324 vma = find_vma(mm, start);
325 /* non-NULL vma must contain @start, but need to check @end */
326 if (!vma || end > vma->vm_end)
Rik van Rielba470de2008-10-18 20:26:50 -0700327 return -ENOMEM;
328
329 return 0; /* hide other errors from mmap(), et al */
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700330 }
Nick Pigginb291f002008-10-18 20:26:44 -0700331
332 /*
333 * User mapped kernel pages or huge pages:
334 * make these pages present to populate the ptes, but
335 * fall thru' to reset VM_LOCKED--no need to unlock, and
336 * return nr_pages so these don't get counted against task's
337 * locked limit. huge pages are already counted against
338 * locked vm limit.
339 */
340 make_pages_present(start, end);
341
342no_mlock:
343 vma->vm_flags &= ~VM_LOCKED; /* and don't come back! */
Rik van Rielba470de2008-10-18 20:26:50 -0700344 return nr_pages; /* error or pages NOT mlocked */
Nick Pigginb291f002008-10-18 20:26:44 -0700345}
346
347
348/*
Rik van Rielba470de2008-10-18 20:26:50 -0700349 * munlock_vma_pages_range() - munlock all pages in the vma range.'
350 * @vma - vma containing range to be munlock()ed.
351 * @start - start address in @vma of the range
352 * @end - end of range in @vma.
353 *
354 * For mremap(), munmap() and exit().
355 *
356 * Called with @vma VM_LOCKED.
357 *
358 * Returns with VM_LOCKED cleared. Callers must be prepared to
359 * deal with this.
360 *
361 * We don't save and restore VM_LOCKED here because pages are
362 * still on lru. In unmap path, pages might be scanned by reclaim
363 * and re-mlocked by try_to_{munlock|unmap} before we unmap and
364 * free them. This will result in freeing mlocked pages.
Nick Pigginb291f002008-10-18 20:26:44 -0700365 */
Rik van Rielba470de2008-10-18 20:26:50 -0700366void munlock_vma_pages_range(struct vm_area_struct *vma,
367 unsigned long start, unsigned long end)
Nick Pigginb291f002008-10-18 20:26:44 -0700368{
369 vma->vm_flags &= ~VM_LOCKED;
Rik van Rielba470de2008-10-18 20:26:50 -0700370 __mlock_vma_pages_range(vma, start, end, 0);
Nick Pigginb291f002008-10-18 20:26:44 -0700371}
372
373/*
374 * mlock_fixup - handle mlock[all]/munlock[all] requests.
375 *
376 * Filters out "special" vmas -- VM_LOCKED never gets set for these, and
377 * munlock is a no-op. However, for some special vmas, we go ahead and
378 * populate the ptes via make_pages_present().
379 *
380 * For vmas that pass the filters, merge/split as appropriate.
381 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
383 unsigned long start, unsigned long end, unsigned int newflags)
384{
Nick Pigginb291f002008-10-18 20:26:44 -0700385 struct mm_struct *mm = vma->vm_mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 pgoff_t pgoff;
Nick Pigginb291f002008-10-18 20:26:44 -0700387 int nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 int ret = 0;
Nick Pigginb291f002008-10-18 20:26:44 -0700389 int lock = newflags & VM_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390
Nick Pigginb291f002008-10-18 20:26:44 -0700391 if (newflags == vma->vm_flags ||
392 (vma->vm_flags & (VM_IO | VM_PFNMAP)))
393 goto out; /* don't set VM_LOCKED, don't count */
394
395 if ((vma->vm_flags & (VM_DONTEXPAND | VM_RESERVED)) ||
396 is_vm_hugetlb_page(vma) ||
397 vma == get_gate_vma(current)) {
398 if (lock)
399 make_pages_present(start, end);
400 goto out; /* don't set VM_LOCKED, don't count */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 }
402
403 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
404 *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,
405 vma->vm_file, pgoff, vma_policy(vma));
406 if (*prev) {
407 vma = *prev;
408 goto success;
409 }
410
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 if (start != vma->vm_start) {
412 ret = split_vma(mm, vma, start, 1);
413 if (ret)
414 goto out;
415 }
416
417 if (end != vma->vm_end) {
418 ret = split_vma(mm, vma, end, 0);
419 if (ret)
420 goto out;
421 }
422
423success:
424 /*
Nick Pigginb291f002008-10-18 20:26:44 -0700425 * Keep track of amount of locked VM.
426 */
427 nr_pages = (end - start) >> PAGE_SHIFT;
428 if (!lock)
429 nr_pages = -nr_pages;
430 mm->locked_vm += nr_pages;
431
432 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433 * vm_flags is protected by the mmap_sem held in write mode.
434 * It's okay if try_to_unmap_one unmaps a page just after we
Nick Pigginb291f002008-10-18 20:26:44 -0700435 * set VM_LOCKED, __mlock_vma_pages_range will bring it back.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 */
437 vma->vm_flags = newflags;
438
Nick Pigginb291f002008-10-18 20:26:44 -0700439 if (lock) {
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700440 /*
441 * mmap_sem is currently held for write. Downgrade the write
442 * lock to a read lock so that other faults, mmap scans, ...
443 * while we fault in all pages.
444 */
445 downgrade_write(&mm->mmap_sem);
446
Rik van Rielba470de2008-10-18 20:26:50 -0700447 ret = __mlock_vma_pages_range(vma, start, end, 1);
Lee Schermerhorn9978ad52008-10-18 20:26:56 -0700448
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700449 /*
450 * Need to reacquire mmap sem in write mode, as our callers
451 * expect this. We have no support for atomically upgrading
452 * a sem to write, so we need to check for ranges while sem
453 * is unlocked.
454 */
455 up_read(&mm->mmap_sem);
456 /* vma can change or disappear */
457 down_write(&mm->mmap_sem);
458 *prev = find_vma(mm, start);
459 /* non-NULL *prev must contain @start, but need to check @end */
460 if (!(*prev) || end > (*prev)->vm_end)
Rik van Rielba470de2008-10-18 20:26:50 -0700461 ret = -ENOMEM;
Lee Schermerhorn9978ad52008-10-18 20:26:56 -0700462 else if (ret > 0) {
463 mm->locked_vm -= ret;
464 ret = 0;
465 } else
466 ret = __mlock_posix_error_return(ret); /* translate if needed */
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700467 } else {
468 /*
469 * TODO: for unlocking, pages will already be resident, so
470 * we don't need to wait for allocations/reclaim/pagein, ...
471 * However, unlocking a very large region can still take a
472 * while. Should we downgrade the semaphore for both lock
473 * AND unlock ?
474 */
Rik van Rielba470de2008-10-18 20:26:50 -0700475 __mlock_vma_pages_range(vma, start, end, 0);
Lee Schermerhorn8edb08c2008-10-18 20:26:49 -0700476 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478out:
Nick Pigginb291f002008-10-18 20:26:44 -0700479 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 return ret;
481}
482
483static int do_mlock(unsigned long start, size_t len, int on)
484{
485 unsigned long nstart, end, tmp;
486 struct vm_area_struct * vma, * prev;
487 int error;
488
489 len = PAGE_ALIGN(len);
490 end = start + len;
491 if (end < start)
492 return -EINVAL;
493 if (end == start)
494 return 0;
495 vma = find_vma_prev(current->mm, start, &prev);
496 if (!vma || vma->vm_start > start)
497 return -ENOMEM;
498
499 if (start > vma->vm_start)
500 prev = vma;
501
502 for (nstart = start ; ; ) {
503 unsigned int newflags;
504
505 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
506
507 newflags = vma->vm_flags | VM_LOCKED;
508 if (!on)
509 newflags &= ~VM_LOCKED;
510
511 tmp = vma->vm_end;
512 if (tmp > end)
513 tmp = end;
514 error = mlock_fixup(vma, &prev, nstart, tmp, newflags);
515 if (error)
516 break;
517 nstart = tmp;
518 if (nstart < prev->vm_end)
519 nstart = prev->vm_end;
520 if (nstart >= end)
521 break;
522
523 vma = prev->vm_next;
524 if (!vma || vma->vm_start != nstart) {
525 error = -ENOMEM;
526 break;
527 }
528 }
529 return error;
530}
531
532asmlinkage long sys_mlock(unsigned long start, size_t len)
533{
534 unsigned long locked;
535 unsigned long lock_limit;
536 int error = -ENOMEM;
537
538 if (!can_do_mlock())
539 return -EPERM;
540
KOSAKI Motohiro8891d6d2008-11-12 13:26:53 -0800541 lru_add_drain_all(); /* flush pagevec */
542
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 down_write(&current->mm->mmap_sem);
544 len = PAGE_ALIGN(len + (start & ~PAGE_MASK));
545 start &= PAGE_MASK;
546
547 locked = len >> PAGE_SHIFT;
548 locked += current->mm->locked_vm;
549
550 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
551 lock_limit >>= PAGE_SHIFT;
552
553 /* check against resource limits */
554 if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
555 error = do_mlock(start, len, 1);
556 up_write(&current->mm->mmap_sem);
557 return error;
558}
559
560asmlinkage long sys_munlock(unsigned long start, size_t len)
561{
562 int ret;
563
564 down_write(&current->mm->mmap_sem);
565 len = PAGE_ALIGN(len + (start & ~PAGE_MASK));
566 start &= PAGE_MASK;
567 ret = do_mlock(start, len, 0);
568 up_write(&current->mm->mmap_sem);
569 return ret;
570}
571
572static int do_mlockall(int flags)
573{
574 struct vm_area_struct * vma, * prev = NULL;
575 unsigned int def_flags = 0;
576
577 if (flags & MCL_FUTURE)
578 def_flags = VM_LOCKED;
579 current->mm->def_flags = def_flags;
580 if (flags == MCL_FUTURE)
581 goto out;
582
583 for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
584 unsigned int newflags;
585
586 newflags = vma->vm_flags | VM_LOCKED;
587 if (!(flags & MCL_CURRENT))
588 newflags &= ~VM_LOCKED;
589
590 /* Ignore errors */
591 mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
592 }
593out:
594 return 0;
595}
596
597asmlinkage long sys_mlockall(int flags)
598{
599 unsigned long lock_limit;
600 int ret = -EINVAL;
601
602 if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE)))
603 goto out;
604
605 ret = -EPERM;
606 if (!can_do_mlock())
607 goto out;
608
KOSAKI Motohiro8891d6d2008-11-12 13:26:53 -0800609 lru_add_drain_all(); /* flush pagevec */
610
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 down_write(&current->mm->mmap_sem);
612
613 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
614 lock_limit >>= PAGE_SHIFT;
615
616 ret = -ENOMEM;
617 if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
618 capable(CAP_IPC_LOCK))
619 ret = do_mlockall(flags);
620 up_write(&current->mm->mmap_sem);
621out:
622 return ret;
623}
624
625asmlinkage long sys_munlockall(void)
626{
627 int ret;
628
629 down_write(&current->mm->mmap_sem);
630 ret = do_mlockall(0);
631 up_write(&current->mm->mmap_sem);
632 return ret;
633}
634
635/*
636 * Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB
637 * shm segments) get accounted against the user_struct instead.
638 */
639static DEFINE_SPINLOCK(shmlock_user_lock);
640
641int user_shm_lock(size_t size, struct user_struct *user)
642{
643 unsigned long lock_limit, locked;
644 int allowed = 0;
645
646 locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
647 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
Herbert van den Bergh5ed44a42007-07-15 23:38:25 -0700648 if (lock_limit == RLIM_INFINITY)
649 allowed = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 lock_limit >>= PAGE_SHIFT;
651 spin_lock(&shmlock_user_lock);
Herbert van den Bergh5ed44a42007-07-15 23:38:25 -0700652 if (!allowed &&
653 locked + user->locked_shm > lock_limit && !capable(CAP_IPC_LOCK))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 goto out;
655 get_uid(user);
656 user->locked_shm += locked;
657 allowed = 1;
658out:
659 spin_unlock(&shmlock_user_lock);
660 return allowed;
661}
662
663void user_shm_unlock(size_t size, struct user_struct *user)
664{
665 spin_lock(&shmlock_user_lock);
666 user->locked_shm -= (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
667 spin_unlock(&shmlock_user_lock);
668 free_uid(user);
669}
Markus Metzgerc5dee612008-12-19 15:17:02 +0100670
671void *alloc_locked_buffer(size_t size)
672{
673 unsigned long rlim, vm, pgsz;
674 void *buffer = NULL;
675
676 pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
677
678 down_write(&current->mm->mmap_sem);
679
680 rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
681 vm = current->mm->total_vm + pgsz;
682 if (rlim < vm)
683 goto out;
684
685 rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
686 vm = current->mm->locked_vm + pgsz;
687 if (rlim < vm)
688 goto out;
689
690 buffer = kzalloc(size, GFP_KERNEL);
691 if (!buffer)
692 goto out;
693
694 current->mm->total_vm += pgsz;
695 current->mm->locked_vm += pgsz;
696
697 out:
698 up_write(&current->mm->mmap_sem);
699 return buffer;
700}
701
702void free_locked_buffer(void *buffer, size_t size)
703{
704 unsigned long pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
705
706 down_write(&current->mm->mmap_sem);
707
708 current->mm->total_vm -= pgsz;
709 current->mm->locked_vm -= pgsz;
710
711 up_write(&current->mm->mmap_sem);
712
713 kfree(buffer);
714}