blob: a402f8fdc68e94888ea177104524085c9f490fd5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/madvise.c
3 *
4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig
6 */
7
8#include <linux/mman.h>
9#include <linux/pagemap.h>
10#include <linux/syscalls.h>
Prasanna Meda05b74382005-06-21 17:14:37 -070011#include <linux/mempolicy.h>
Andi Kleenafcf9382009-12-16 12:20:00 +010012#include <linux/page-isolation.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/hugetlb.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070014#include <linux/falloc.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040015#include <linux/sched.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070016#include <linux/ksm.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070017#include <linux/fs.h>
Andy Lutomirski9ab42332012-07-05 16:00:11 -070018#include <linux/file.h>
Shaohua Li1998cc02013-02-22 16:32:31 -080019#include <linux/blkdev.h>
20#include <linux/swap.h>
21#include <linux/swapops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23/*
Nick Piggin0a27a142007-05-06 14:49:53 -070024 * Any behaviour which results in changes to the vma->vm_flags needs to
25 * take mmap_sem for writing. Others, which simply traverse vmas, need
26 * to only take it for reading.
27 */
28static int madvise_need_mmap_write(int behavior)
29{
30 switch (behavior) {
31 case MADV_REMOVE:
32 case MADV_WILLNEED:
33 case MADV_DONTNEED:
34 return 0;
35 default:
36 /* be safe, default to 1. list exceptions explicitly */
37 return 1;
38 }
39}
40
41/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 * We can potentially split a vm area into separate
43 * areas, each area with its own behavior.
44 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -070045static long madvise_behavior(struct vm_area_struct *vma,
Prasanna Meda05b74382005-06-21 17:14:37 -070046 struct vm_area_struct **prev,
47 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -070048{
Vladimir Cernovec9bed92013-09-11 14:20:15 -070049 struct mm_struct *mm = vma->vm_mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 int error = 0;
Prasanna Meda05b74382005-06-21 17:14:37 -070051 pgoff_t pgoff;
Hugh Dickins3866ea92009-09-21 17:01:52 -070052 unsigned long new_flags = vma->vm_flags;
Prasanna Medae798c6e2005-06-21 17:14:36 -070053
54 switch (behavior) {
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080055 case MADV_NORMAL:
56 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
57 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070058 case MADV_SEQUENTIAL:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080059 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070060 break;
61 case MADV_RANDOM:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080062 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070063 break;
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080064 case MADV_DONTFORK:
65 new_flags |= VM_DONTCOPY;
66 break;
67 case MADV_DOFORK:
Hugh Dickins3866ea92009-09-21 17:01:52 -070068 if (vma->vm_flags & VM_IO) {
69 error = -EINVAL;
70 goto out;
71 }
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080072 new_flags &= ~VM_DONTCOPY;
Prasanna Medae798c6e2005-06-21 17:14:36 -070073 break;
Jason Baronaccb61f2012-03-23 15:02:51 -070074 case MADV_DONTDUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070075 new_flags |= VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070076 break;
77 case MADV_DODUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070078 if (new_flags & VM_SPECIAL) {
79 error = -EINVAL;
80 goto out;
81 }
82 new_flags &= ~VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070083 break;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070084 case MADV_MERGEABLE:
85 case MADV_UNMERGEABLE:
86 error = ksm_madvise(vma, start, end, behavior, &new_flags);
87 if (error)
88 goto out;
89 break;
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080090 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -080091 case MADV_NOHUGEPAGE:
Andrea Arcangeli60ab3242011-01-13 15:47:18 -080092 error = hugepage_madvise(vma, &new_flags, behavior);
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080093 if (error)
94 goto out;
95 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070096 }
97
Prasanna Meda05b74382005-06-21 17:14:37 -070098 if (new_flags == vma->vm_flags) {
99 *prev = vma;
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700100 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700101 }
102
103 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
104 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
105 vma->vm_file, pgoff, vma_policy(vma));
106 if (*prev) {
107 vma = *prev;
108 goto success;
109 }
110
111 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
113 if (start != vma->vm_start) {
114 error = split_vma(mm, vma, start, 1);
115 if (error)
116 goto out;
117 }
118
119 if (end != vma->vm_end) {
120 error = split_vma(mm, vma, end, 0);
121 if (error)
122 goto out;
123 }
124
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700125success:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 /*
127 * vm_flags is protected by the mmap_sem held in write mode.
128 */
Prasanna Medae798c6e2005-06-21 17:14:36 -0700129 vma->vm_flags = new_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130
131out:
132 if (error == -ENOMEM)
133 error = -EAGAIN;
134 return error;
135}
136
Shaohua Li1998cc02013-02-22 16:32:31 -0800137#ifdef CONFIG_SWAP
138static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
139 unsigned long end, struct mm_walk *walk)
140{
141 pte_t *orig_pte;
142 struct vm_area_struct *vma = walk->private;
143 unsigned long index;
144
145 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
146 return 0;
147
148 for (index = start; index != end; index += PAGE_SIZE) {
149 pte_t pte;
150 swp_entry_t entry;
151 struct page *page;
152 spinlock_t *ptl;
153
154 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
155 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
156 pte_unmap_unlock(orig_pte, ptl);
157
158 if (pte_present(pte) || pte_none(pte) || pte_file(pte))
159 continue;
160 entry = pte_to_swp_entry(pte);
161 if (unlikely(non_swap_entry(entry)))
162 continue;
163
164 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
165 vma, index);
166 if (page)
167 page_cache_release(page);
168 }
169
170 return 0;
171}
172
173static void force_swapin_readahead(struct vm_area_struct *vma,
174 unsigned long start, unsigned long end)
175{
176 struct mm_walk walk = {
177 .mm = vma->vm_mm,
178 .pmd_entry = swapin_walk_pmd_entry,
179 .private = vma,
180 };
181
182 walk_page_range(start, end, &walk);
183
184 lru_add_drain(); /* Push any new pages onto the LRU now */
185}
186
187static void force_shm_swapin_readahead(struct vm_area_struct *vma,
188 unsigned long start, unsigned long end,
189 struct address_space *mapping)
190{
191 pgoff_t index;
192 struct page *page;
193 swp_entry_t swap;
194
195 for (; start < end; start += PAGE_SIZE) {
196 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
197
Johannes Weiner55231e52014-05-22 11:54:17 -0700198 page = find_get_entry(mapping, index);
Shaohua Li1998cc02013-02-22 16:32:31 -0800199 if (!radix_tree_exceptional_entry(page)) {
200 if (page)
201 page_cache_release(page);
202 continue;
203 }
204 swap = radix_to_swp_entry(page);
205 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
206 NULL, 0);
207 if (page)
208 page_cache_release(page);
209 }
210
211 lru_add_drain(); /* Push any new pages onto the LRU now */
212}
213#endif /* CONFIG_SWAP */
214
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215/*
216 * Schedule all required I/O operations. Do not wait for completion.
217 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700218static long madvise_willneed(struct vm_area_struct *vma,
219 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 unsigned long start, unsigned long end)
221{
222 struct file *file = vma->vm_file;
223
Shaohua Li1998cc02013-02-22 16:32:31 -0800224#ifdef CONFIG_SWAP
225 if (!file || mapping_cap_swap_backed(file->f_mapping)) {
226 *prev = vma;
227 if (!file)
228 force_swapin_readahead(vma, start, end);
229 else
230 force_shm_swapin_readahead(vma, start, end,
231 file->f_mapping);
232 return 0;
233 }
234#endif
235
Suzuki1bef4002005-10-11 08:29:06 -0700236 if (!file)
237 return -EBADF;
238
Nick Piggin70688e42008-04-28 02:13:02 -0700239 if (file->f_mapping->a_ops->get_xip_mem) {
Carsten Ottefe77ba62005-06-23 22:05:29 -0700240 /* no bad return value, but ignore advice */
241 return 0;
242 }
243
Prasanna Meda05b74382005-06-21 17:14:37 -0700244 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
246 if (end > vma->vm_end)
247 end = vma->vm_end;
248 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
249
Wu Fengguangf7e839d2009-06-16 15:31:20 -0700250 force_page_cache_readahead(file->f_mapping, file, start, end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 return 0;
252}
253
254/*
255 * Application no longer needs these pages. If the pages are dirty,
256 * it's OK to just throw them away. The app will be more careful about
257 * data it wants to keep. Be sure to free swap resources too. The
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700258 * zap_page_range call sets things up for shrink_active_list to actually free
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 * these pages later if no one else has touched them in the meantime,
260 * although we could add these pages to a global reuse list for
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700261 * shrink_active_list to pick up before reclaiming other pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 *
263 * NB: This interface discards data rather than pushes it out to swap,
264 * as some implementations do. This has performance implications for
265 * applications like large transactional databases which want to discard
266 * pages in anonymous maps after committing to backing store the data
267 * that was kept in them. There is no reason to write this data out to
268 * the swap area if the application is discarding it.
269 *
270 * An interface that causes the system to free clean pages and flush
271 * dirty pages is already available as msync(MS_INVALIDATE).
272 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700273static long madvise_dontneed(struct vm_area_struct *vma,
274 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 unsigned long start, unsigned long end)
276{
Prasanna Meda05b74382005-06-21 17:14:37 -0700277 *prev = vma;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800278 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 return -EINVAL;
280
281 if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
282 struct zap_details details = {
283 .nonlinear_vma = vma,
284 .last_index = ULONG_MAX,
285 };
286 zap_page_range(vma, start, end - start, &details);
287 } else
288 zap_page_range(vma, start, end - start, NULL);
289 return 0;
290}
291
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800292/*
293 * Application wants to free up the pages and associated backing store.
294 * This is effectively punching a hole into the middle of a file.
295 *
296 * NOTE: Currently, only shmfs/tmpfs is supported for this operation.
297 * Other filesystems return -ENOSYS.
298 */
299static long madvise_remove(struct vm_area_struct *vma,
Nick Piggin00e9fa22007-03-16 13:38:10 -0800300 struct vm_area_struct **prev,
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800301 unsigned long start, unsigned long end)
302{
Hugh Dickins3f31d072012-05-29 15:06:40 -0700303 loff_t offset;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700304 int error;
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700305 struct file *f;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800306
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700307 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
Nick Piggin00e9fa22007-03-16 13:38:10 -0800308
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800309 if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
310 return -EINVAL;
311
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700312 f = vma->vm_file;
313
314 if (!f || !f->f_mapping || !f->f_mapping->host) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800315 return -EINVAL;
316 }
317
Hugh Dickins69cf0fa2006-04-17 22:46:32 +0100318 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
319 return -EACCES;
320
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800321 offset = (loff_t)(start - vma->vm_start)
322 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700323
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700324 /*
325 * Filesystem's fallocate may need to take i_mutex. We need to
326 * explicitly grab a reference because the vma (and hence the
327 * vma's reference to the file) can go away as soon as we drop
328 * mmap_sem.
329 */
330 get_file(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700331 up_read(&current->mm->mmap_sem);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700332 error = do_fallocate(f,
Hugh Dickins3f31d072012-05-29 15:06:40 -0700333 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
334 offset, end - start);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700335 fput(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700336 down_read(&current->mm->mmap_sem);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700337 return error;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800338}
339
Andi Kleen9893e492009-09-16 11:50:17 +0200340#ifdef CONFIG_MEMORY_FAILURE
341/*
342 * Error injection support for memory error handling.
343 */
Andi Kleenafcf9382009-12-16 12:20:00 +0100344static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
Andi Kleen9893e492009-09-16 11:50:17 +0200345{
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700346 struct page *p;
Andi Kleen9893e492009-09-16 11:50:17 +0200347 if (!capable(CAP_SYS_ADMIN))
348 return -EPERM;
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700349 for (; start < end; start += PAGE_SIZE <<
350 compound_order(compound_head(p))) {
Andrew Morton325c4ef2013-09-11 14:23:03 -0700351 int ret;
352
353 ret = get_user_pages_fast(start, 1, 0, &p);
Andi Kleen9893e492009-09-16 11:50:17 +0200354 if (ret != 1)
355 return ret;
Andrew Morton325c4ef2013-09-11 14:23:03 -0700356
Wanpeng Li29b4eed2013-09-11 14:22:59 -0700357 if (PageHWPoison(p)) {
358 put_page(p);
359 continue;
360 }
Andi Kleenafcf9382009-12-16 12:20:00 +0100361 if (bhv == MADV_SOFT_OFFLINE) {
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700362 pr_info("Soft offlining page %#lx at %#lx\n",
Andi Kleenafcf9382009-12-16 12:20:00 +0100363 page_to_pfn(p), start);
364 ret = soft_offline_page(p, MF_COUNT_INCREASED);
365 if (ret)
Wanpeng Li83024232013-09-11 14:23:02 -0700366 return ret;
Andi Kleenafcf9382009-12-16 12:20:00 +0100367 continue;
368 }
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700369 pr_info("Injecting memory failure for page %#lx at %#lx\n",
Andi Kleen9893e492009-09-16 11:50:17 +0200370 page_to_pfn(p), start);
371 /* Ignore return value for now */
Tony Luckcd42f4a2011-12-15 10:48:12 -0800372 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
Andi Kleen9893e492009-09-16 11:50:17 +0200373 }
Andrew Morton325c4ef2013-09-11 14:23:03 -0700374 return 0;
Andi Kleen9893e492009-09-16 11:50:17 +0200375}
376#endif
377
suzuki165cd402005-07-27 11:43:59 -0700378static long
379madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
380 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 switch (behavior) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800383 case MADV_REMOVE:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700384 return madvise_remove(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 case MADV_WILLNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700386 return madvise_willneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 case MADV_DONTNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700388 return madvise_dontneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 default:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700390 return madvise_behavior(vma, prev, start, end, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392}
393
Nick Piggin75927af2009-06-16 15:32:38 -0700394static int
395madvise_behavior_valid(int behavior)
396{
397 switch (behavior) {
398 case MADV_DOFORK:
399 case MADV_DONTFORK:
400 case MADV_NORMAL:
401 case MADV_SEQUENTIAL:
402 case MADV_RANDOM:
403 case MADV_REMOVE:
404 case MADV_WILLNEED:
405 case MADV_DONTNEED:
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700406#ifdef CONFIG_KSM
407 case MADV_MERGEABLE:
408 case MADV_UNMERGEABLE:
409#endif
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800410#ifdef CONFIG_TRANSPARENT_HUGEPAGE
411 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -0800412 case MADV_NOHUGEPAGE:
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800413#endif
Jason Baronaccb61f2012-03-23 15:02:51 -0700414 case MADV_DONTDUMP:
415 case MADV_DODUMP:
Nick Piggin75927af2009-06-16 15:32:38 -0700416 return 1;
417
418 default:
419 return 0;
420 }
421}
Hugh Dickins3866ea92009-09-21 17:01:52 -0700422
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423/*
424 * The madvise(2) system call.
425 *
426 * Applications can use madvise() to advise the kernel how it should
427 * handle paging I/O in this VM area. The idea is to help the kernel
428 * use appropriate read-ahead and caching techniques. The information
429 * provided is advisory only, and can be safely disregarded by the
430 * kernel without affecting the correct operation of the application.
431 *
432 * behavior values:
433 * MADV_NORMAL - the default behavior is to read clusters. This
434 * results in some read-ahead and read-behind.
435 * MADV_RANDOM - the system should read the minimum amount of data
436 * on any access, since it is unlikely that the appli-
437 * cation will need more than what it asks for.
438 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
439 * once, so they can be aggressively read ahead, and
440 * can be freed soon after they are accessed.
441 * MADV_WILLNEED - the application is notifying the system to read
442 * some pages ahead.
443 * MADV_DONTNEED - the application is finished with the given range,
444 * so the kernel can free resources associated with it.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800445 * MADV_REMOVE - the application wants to free up the given range of
446 * pages and associated backing store.
Hugh Dickins3866ea92009-09-21 17:01:52 -0700447 * MADV_DONTFORK - omit this area from child's address space when forking:
448 * typically, to avoid COWing pages pinned by get_user_pages().
449 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700450 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
451 * this area with pages of identical content from other such areas.
452 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 *
454 * return values:
455 * zero - success
456 * -EINVAL - start + len < 0, start is not page-aligned,
457 * "behavior" is not a valid value, or application
458 * is attempting to release locked or shared pages.
459 * -ENOMEM - addresses in the specified range are not currently
460 * mapped, or are outside the AS of the process.
461 * -EIO - an I/O error occurred while paging in data.
462 * -EBADF - map exists, but area maps something that isn't a file.
463 * -EAGAIN - a kernel resource was temporarily unavailable.
464 */
Heiko Carstens3480b252009-01-14 14:14:16 +0100465SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466{
Prasanna Meda05b74382005-06-21 17:14:37 -0700467 unsigned long end, tmp;
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700468 struct vm_area_struct *vma, *prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 int unmapped_error = 0;
470 int error = -EINVAL;
Jason Baronf7977792007-07-15 23:38:21 -0700471 int write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 size_t len;
Shaohua Li1998cc02013-02-22 16:32:31 -0800473 struct blk_plug plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
Andi Kleen9893e492009-09-16 11:50:17 +0200475#ifdef CONFIG_MEMORY_FAILURE
Andi Kleenafcf9382009-12-16 12:20:00 +0100476 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
477 return madvise_hwpoison(behavior, start, start+len_in);
Andi Kleen9893e492009-09-16 11:50:17 +0200478#endif
Nick Piggin75927af2009-06-16 15:32:38 -0700479 if (!madvise_behavior_valid(behavior))
480 return error;
481
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700482 if (start & ~PAGE_MASK)
483 return error;
484 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
485
486 /* Check to see whether len was rounded up from small -ve to zero */
487 if (len_in && !len)
488 return error;
489
490 end = start + len;
491 if (end < start)
492 return error;
493
494 error = 0;
495 if (end == start)
496 return error;
497
Jason Baronf7977792007-07-15 23:38:21 -0700498 write = madvise_need_mmap_write(behavior);
499 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700500 down_write(&current->mm->mmap_sem);
501 else
502 down_read(&current->mm->mmap_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 /*
505 * If the interval [start,end) covers some unmapped address
506 * ranges, just ignore them, but return -ENOMEM at the end.
Prasanna Meda05b74382005-06-21 17:14:37 -0700507 * - different from the way of handling in mlock etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 */
Prasanna Meda05b74382005-06-21 17:14:37 -0700509 vma = find_vma_prev(current->mm, start, &prev);
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700510 if (vma && start > vma->vm_start)
511 prev = vma;
512
Shaohua Li1998cc02013-02-22 16:32:31 -0800513 blk_start_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 for (;;) {
515 /* Still start < end. */
516 error = -ENOMEM;
517 if (!vma)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700518 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519
Prasanna Meda05b74382005-06-21 17:14:37 -0700520 /* Here start < (end|vma->vm_end). */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 if (start < vma->vm_start) {
522 unmapped_error = -ENOMEM;
523 start = vma->vm_start;
Prasanna Meda05b74382005-06-21 17:14:37 -0700524 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700525 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 }
527
Prasanna Meda05b74382005-06-21 17:14:37 -0700528 /* Here vma->vm_start <= start < (end|vma->vm_end) */
529 tmp = vma->vm_end;
530 if (end < tmp)
531 tmp = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532
Prasanna Meda05b74382005-06-21 17:14:37 -0700533 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
534 error = madvise_vma(vma, &prev, start, tmp, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 if (error)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700536 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700537 start = tmp;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700538 if (prev && start < prev->vm_end)
Prasanna Meda05b74382005-06-21 17:14:37 -0700539 start = prev->vm_end;
540 error = unmapped_error;
541 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700542 goto out;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700543 if (prev)
544 vma = prev->vm_next;
545 else /* madvise_remove dropped mmap_sem */
546 vma = find_vma(current->mm, start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548out:
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700549 blk_finish_plug(&plug);
Jason Baronf7977792007-07-15 23:38:21 -0700550 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700551 up_write(&current->mm->mmap_sem);
552 else
553 up_read(&current->mm->mmap_sem);
554
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 return error;
556}