blob: 0938b30da4abbb91aa01e7884c6c236728403dee [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/madvise.c
3 *
4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig
6 */
7
8#include <linux/mman.h>
9#include <linux/pagemap.h>
10#include <linux/syscalls.h>
Prasanna Meda05b74382005-06-21 17:14:37 -070011#include <linux/mempolicy.h>
Andi Kleenafcf9382009-12-16 12:20:00 +010012#include <linux/page-isolation.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/hugetlb.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070014#include <linux/falloc.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040015#include <linux/sched.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070016#include <linux/ksm.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070017#include <linux/fs.h>
Andy Lutomirski9ab42332012-07-05 16:00:11 -070018#include <linux/file.h>
Shaohua Li1998cc02013-02-22 16:32:31 -080019#include <linux/blkdev.h>
20#include <linux/swap.h>
21#include <linux/swapops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23/*
Nick Piggin0a27a142007-05-06 14:49:53 -070024 * Any behaviour which results in changes to the vma->vm_flags needs to
25 * take mmap_sem for writing. Others, which simply traverse vmas, need
26 * to only take it for reading.
27 */
28static int madvise_need_mmap_write(int behavior)
29{
30 switch (behavior) {
31 case MADV_REMOVE:
32 case MADV_WILLNEED:
33 case MADV_DONTNEED:
34 return 0;
35 default:
36 /* be safe, default to 1. list exceptions explicitly */
37 return 1;
38 }
39}
40
41/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 * We can potentially split a vm area into separate
43 * areas, each area with its own behavior.
44 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -070045static long madvise_behavior(struct vm_area_struct *vma,
Prasanna Meda05b74382005-06-21 17:14:37 -070046 struct vm_area_struct **prev,
47 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -070048{
Vladimir Cernovec9bed92013-09-11 14:20:15 -070049 struct mm_struct *mm = vma->vm_mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 int error = 0;
Prasanna Meda05b74382005-06-21 17:14:37 -070051 pgoff_t pgoff;
Hugh Dickins3866ea92009-09-21 17:01:52 -070052 unsigned long new_flags = vma->vm_flags;
Prasanna Medae798c6e2005-06-21 17:14:36 -070053
54 switch (behavior) {
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080055 case MADV_NORMAL:
56 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
57 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070058 case MADV_SEQUENTIAL:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080059 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070060 break;
61 case MADV_RANDOM:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080062 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070063 break;
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080064 case MADV_DONTFORK:
65 new_flags |= VM_DONTCOPY;
66 break;
67 case MADV_DOFORK:
Hugh Dickins3866ea92009-09-21 17:01:52 -070068 if (vma->vm_flags & VM_IO) {
69 error = -EINVAL;
70 goto out;
71 }
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080072 new_flags &= ~VM_DONTCOPY;
Prasanna Medae798c6e2005-06-21 17:14:36 -070073 break;
Jason Baronaccb61f2012-03-23 15:02:51 -070074 case MADV_DONTDUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070075 new_flags |= VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070076 break;
77 case MADV_DODUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070078 if (new_flags & VM_SPECIAL) {
79 error = -EINVAL;
80 goto out;
81 }
82 new_flags &= ~VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070083 break;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070084 case MADV_MERGEABLE:
85 case MADV_UNMERGEABLE:
86 error = ksm_madvise(vma, start, end, behavior, &new_flags);
87 if (error)
88 goto out;
89 break;
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080090 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -080091 case MADV_NOHUGEPAGE:
Andrea Arcangeli60ab3242011-01-13 15:47:18 -080092 error = hugepage_madvise(vma, &new_flags, behavior);
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080093 if (error)
94 goto out;
95 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070096 }
97
Prasanna Meda05b74382005-06-21 17:14:37 -070098 if (new_flags == vma->vm_flags) {
99 *prev = vma;
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700100 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700101 }
102
103 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
104 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
105 vma->vm_file, pgoff, vma_policy(vma));
106 if (*prev) {
107 vma = *prev;
108 goto success;
109 }
110
111 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
113 if (start != vma->vm_start) {
114 error = split_vma(mm, vma, start, 1);
115 if (error)
116 goto out;
117 }
118
119 if (end != vma->vm_end) {
120 error = split_vma(mm, vma, end, 0);
121 if (error)
122 goto out;
123 }
124
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700125success:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 /*
127 * vm_flags is protected by the mmap_sem held in write mode.
128 */
Prasanna Medae798c6e2005-06-21 17:14:36 -0700129 vma->vm_flags = new_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130
131out:
132 if (error == -ENOMEM)
133 error = -EAGAIN;
134 return error;
135}
136
Shaohua Li1998cc02013-02-22 16:32:31 -0800137#ifdef CONFIG_SWAP
138static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
139 unsigned long end, struct mm_walk *walk)
140{
141 pte_t *orig_pte;
142 struct vm_area_struct *vma = walk->private;
143 unsigned long index;
144
145 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
146 return 0;
147
148 for (index = start; index != end; index += PAGE_SIZE) {
149 pte_t pte;
150 swp_entry_t entry;
151 struct page *page;
152 spinlock_t *ptl;
153
154 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
155 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
156 pte_unmap_unlock(orig_pte, ptl);
157
158 if (pte_present(pte) || pte_none(pte) || pte_file(pte))
159 continue;
160 entry = pte_to_swp_entry(pte);
161 if (unlikely(non_swap_entry(entry)))
162 continue;
163
164 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
165 vma, index);
166 if (page)
167 page_cache_release(page);
168 }
169
170 return 0;
171}
172
173static void force_swapin_readahead(struct vm_area_struct *vma,
174 unsigned long start, unsigned long end)
175{
176 struct mm_walk walk = {
177 .mm = vma->vm_mm,
178 .pmd_entry = swapin_walk_pmd_entry,
179 .private = vma,
180 };
181
182 walk_page_range(start, end, &walk);
183
184 lru_add_drain(); /* Push any new pages onto the LRU now */
185}
186
187static void force_shm_swapin_readahead(struct vm_area_struct *vma,
188 unsigned long start, unsigned long end,
189 struct address_space *mapping)
190{
191 pgoff_t index;
192 struct page *page;
193 swp_entry_t swap;
194
195 for (; start < end; start += PAGE_SIZE) {
196 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
197
Johannes Weiner55231e52014-05-22 11:54:17 -0700198 page = find_get_entry(mapping, index);
Shaohua Li1998cc02013-02-22 16:32:31 -0800199 if (!radix_tree_exceptional_entry(page)) {
200 if (page)
201 page_cache_release(page);
202 continue;
203 }
204 swap = radix_to_swp_entry(page);
205 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
206 NULL, 0);
207 if (page)
208 page_cache_release(page);
209 }
210
211 lru_add_drain(); /* Push any new pages onto the LRU now */
212}
213#endif /* CONFIG_SWAP */
214
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215/*
216 * Schedule all required I/O operations. Do not wait for completion.
217 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700218static long madvise_willneed(struct vm_area_struct *vma,
219 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 unsigned long start, unsigned long end)
221{
222 struct file *file = vma->vm_file;
223
Shaohua Li1998cc02013-02-22 16:32:31 -0800224#ifdef CONFIG_SWAP
225 if (!file || mapping_cap_swap_backed(file->f_mapping)) {
226 *prev = vma;
227 if (!file)
228 force_swapin_readahead(vma, start, end);
229 else
230 force_shm_swapin_readahead(vma, start, end,
231 file->f_mapping);
232 return 0;
233 }
234#endif
235
Suzuki1bef4002005-10-11 08:29:06 -0700236 if (!file)
237 return -EBADF;
238
Nick Piggin70688e42008-04-28 02:13:02 -0700239 if (file->f_mapping->a_ops->get_xip_mem) {
Carsten Ottefe77ba62005-06-23 22:05:29 -0700240 /* no bad return value, but ignore advice */
241 return 0;
242 }
243
Prasanna Meda05b74382005-06-21 17:14:37 -0700244 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
246 if (end > vma->vm_end)
247 end = vma->vm_end;
248 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
249
Wu Fengguangf7e839d2009-06-16 15:31:20 -0700250 force_page_cache_readahead(file->f_mapping, file, start, end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 return 0;
252}
253
254/*
255 * Application no longer needs these pages. If the pages are dirty,
256 * it's OK to just throw them away. The app will be more careful about
257 * data it wants to keep. Be sure to free swap resources too. The
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700258 * zap_page_range call sets things up for shrink_active_list to actually free
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 * these pages later if no one else has touched them in the meantime,
260 * although we could add these pages to a global reuse list for
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700261 * shrink_active_list to pick up before reclaiming other pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 *
263 * NB: This interface discards data rather than pushes it out to swap,
264 * as some implementations do. This has performance implications for
265 * applications like large transactional databases which want to discard
266 * pages in anonymous maps after committing to backing store the data
267 * that was kept in them. There is no reason to write this data out to
268 * the swap area if the application is discarding it.
269 *
270 * An interface that causes the system to free clean pages and flush
271 * dirty pages is already available as msync(MS_INVALIDATE).
272 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700273static long madvise_dontneed(struct vm_area_struct *vma,
274 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 unsigned long start, unsigned long end)
276{
Prasanna Meda05b74382005-06-21 17:14:37 -0700277 *prev = vma;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800278 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 return -EINVAL;
280
281 if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
282 struct zap_details details = {
283 .nonlinear_vma = vma,
284 .last_index = ULONG_MAX,
285 };
286 zap_page_range(vma, start, end - start, &details);
287 } else
288 zap_page_range(vma, start, end - start, NULL);
289 return 0;
290}
291
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800292/*
293 * Application wants to free up the pages and associated backing store.
294 * This is effectively punching a hole into the middle of a file.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800295 */
296static long madvise_remove(struct vm_area_struct *vma,
Nick Piggin00e9fa22007-03-16 13:38:10 -0800297 struct vm_area_struct **prev,
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800298 unsigned long start, unsigned long end)
299{
Hugh Dickins3f31d072012-05-29 15:06:40 -0700300 loff_t offset;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700301 int error;
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700302 struct file *f;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800303
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700304 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
Nick Piggin00e9fa22007-03-16 13:38:10 -0800305
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800306 if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
307 return -EINVAL;
308
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700309 f = vma->vm_file;
310
311 if (!f || !f->f_mapping || !f->f_mapping->host) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800312 return -EINVAL;
313 }
314
Hugh Dickins69cf0fa2006-04-17 22:46:32 +0100315 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
316 return -EACCES;
317
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800318 offset = (loff_t)(start - vma->vm_start)
319 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700320
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700321 /*
322 * Filesystem's fallocate may need to take i_mutex. We need to
323 * explicitly grab a reference because the vma (and hence the
324 * vma's reference to the file) can go away as soon as we drop
325 * mmap_sem.
326 */
327 get_file(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700328 up_read(&current->mm->mmap_sem);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700329 error = do_fallocate(f,
Hugh Dickins3f31d072012-05-29 15:06:40 -0700330 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
331 offset, end - start);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700332 fput(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700333 down_read(&current->mm->mmap_sem);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700334 return error;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800335}
336
Andi Kleen9893e492009-09-16 11:50:17 +0200337#ifdef CONFIG_MEMORY_FAILURE
338/*
339 * Error injection support for memory error handling.
340 */
Andi Kleenafcf9382009-12-16 12:20:00 +0100341static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
Andi Kleen9893e492009-09-16 11:50:17 +0200342{
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700343 struct page *p;
Andi Kleen9893e492009-09-16 11:50:17 +0200344 if (!capable(CAP_SYS_ADMIN))
345 return -EPERM;
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700346 for (; start < end; start += PAGE_SIZE <<
347 compound_order(compound_head(p))) {
Andrew Morton325c4ef2013-09-11 14:23:03 -0700348 int ret;
349
350 ret = get_user_pages_fast(start, 1, 0, &p);
Andi Kleen9893e492009-09-16 11:50:17 +0200351 if (ret != 1)
352 return ret;
Andrew Morton325c4ef2013-09-11 14:23:03 -0700353
Wanpeng Li29b4eed2013-09-11 14:22:59 -0700354 if (PageHWPoison(p)) {
355 put_page(p);
356 continue;
357 }
Andi Kleenafcf9382009-12-16 12:20:00 +0100358 if (bhv == MADV_SOFT_OFFLINE) {
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700359 pr_info("Soft offlining page %#lx at %#lx\n",
Andi Kleenafcf9382009-12-16 12:20:00 +0100360 page_to_pfn(p), start);
361 ret = soft_offline_page(p, MF_COUNT_INCREASED);
362 if (ret)
Wanpeng Li83024232013-09-11 14:23:02 -0700363 return ret;
Andi Kleenafcf9382009-12-16 12:20:00 +0100364 continue;
365 }
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700366 pr_info("Injecting memory failure for page %#lx at %#lx\n",
Andi Kleen9893e492009-09-16 11:50:17 +0200367 page_to_pfn(p), start);
368 /* Ignore return value for now */
Tony Luckcd42f4a2011-12-15 10:48:12 -0800369 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
Andi Kleen9893e492009-09-16 11:50:17 +0200370 }
Andrew Morton325c4ef2013-09-11 14:23:03 -0700371 return 0;
Andi Kleen9893e492009-09-16 11:50:17 +0200372}
373#endif
374
suzuki165cd402005-07-27 11:43:59 -0700375static long
376madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
377 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 switch (behavior) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800380 case MADV_REMOVE:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700381 return madvise_remove(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 case MADV_WILLNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700383 return madvise_willneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 case MADV_DONTNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700385 return madvise_dontneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 default:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700387 return madvise_behavior(vma, prev, start, end, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389}
390
Nick Piggin75927af2009-06-16 15:32:38 -0700391static int
392madvise_behavior_valid(int behavior)
393{
394 switch (behavior) {
395 case MADV_DOFORK:
396 case MADV_DONTFORK:
397 case MADV_NORMAL:
398 case MADV_SEQUENTIAL:
399 case MADV_RANDOM:
400 case MADV_REMOVE:
401 case MADV_WILLNEED:
402 case MADV_DONTNEED:
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700403#ifdef CONFIG_KSM
404 case MADV_MERGEABLE:
405 case MADV_UNMERGEABLE:
406#endif
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800407#ifdef CONFIG_TRANSPARENT_HUGEPAGE
408 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -0800409 case MADV_NOHUGEPAGE:
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800410#endif
Jason Baronaccb61f2012-03-23 15:02:51 -0700411 case MADV_DONTDUMP:
412 case MADV_DODUMP:
Nick Piggin75927af2009-06-16 15:32:38 -0700413 return 1;
414
415 default:
416 return 0;
417 }
418}
Hugh Dickins3866ea92009-09-21 17:01:52 -0700419
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420/*
421 * The madvise(2) system call.
422 *
423 * Applications can use madvise() to advise the kernel how it should
424 * handle paging I/O in this VM area. The idea is to help the kernel
425 * use appropriate read-ahead and caching techniques. The information
426 * provided is advisory only, and can be safely disregarded by the
427 * kernel without affecting the correct operation of the application.
428 *
429 * behavior values:
430 * MADV_NORMAL - the default behavior is to read clusters. This
431 * results in some read-ahead and read-behind.
432 * MADV_RANDOM - the system should read the minimum amount of data
433 * on any access, since it is unlikely that the appli-
434 * cation will need more than what it asks for.
435 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
436 * once, so they can be aggressively read ahead, and
437 * can be freed soon after they are accessed.
438 * MADV_WILLNEED - the application is notifying the system to read
439 * some pages ahead.
440 * MADV_DONTNEED - the application is finished with the given range,
441 * so the kernel can free resources associated with it.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800442 * MADV_REMOVE - the application wants to free up the given range of
443 * pages and associated backing store.
Hugh Dickins3866ea92009-09-21 17:01:52 -0700444 * MADV_DONTFORK - omit this area from child's address space when forking:
445 * typically, to avoid COWing pages pinned by get_user_pages().
446 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700447 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
448 * this area with pages of identical content from other such areas.
449 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 *
451 * return values:
452 * zero - success
453 * -EINVAL - start + len < 0, start is not page-aligned,
454 * "behavior" is not a valid value, or application
455 * is attempting to release locked or shared pages.
456 * -ENOMEM - addresses in the specified range are not currently
457 * mapped, or are outside the AS of the process.
458 * -EIO - an I/O error occurred while paging in data.
459 * -EBADF - map exists, but area maps something that isn't a file.
460 * -EAGAIN - a kernel resource was temporarily unavailable.
461 */
Heiko Carstens3480b252009-01-14 14:14:16 +0100462SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463{
Prasanna Meda05b74382005-06-21 17:14:37 -0700464 unsigned long end, tmp;
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700465 struct vm_area_struct *vma, *prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 int unmapped_error = 0;
467 int error = -EINVAL;
Jason Baronf7977792007-07-15 23:38:21 -0700468 int write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 size_t len;
Shaohua Li1998cc02013-02-22 16:32:31 -0800470 struct blk_plug plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471
Andi Kleen9893e492009-09-16 11:50:17 +0200472#ifdef CONFIG_MEMORY_FAILURE
Andi Kleenafcf9382009-12-16 12:20:00 +0100473 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
474 return madvise_hwpoison(behavior, start, start+len_in);
Andi Kleen9893e492009-09-16 11:50:17 +0200475#endif
Nick Piggin75927af2009-06-16 15:32:38 -0700476 if (!madvise_behavior_valid(behavior))
477 return error;
478
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700479 if (start & ~PAGE_MASK)
480 return error;
481 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
482
483 /* Check to see whether len was rounded up from small -ve to zero */
484 if (len_in && !len)
485 return error;
486
487 end = start + len;
488 if (end < start)
489 return error;
490
491 error = 0;
492 if (end == start)
493 return error;
494
Jason Baronf7977792007-07-15 23:38:21 -0700495 write = madvise_need_mmap_write(behavior);
496 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700497 down_write(&current->mm->mmap_sem);
498 else
499 down_read(&current->mm->mmap_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 /*
502 * If the interval [start,end) covers some unmapped address
503 * ranges, just ignore them, but return -ENOMEM at the end.
Prasanna Meda05b74382005-06-21 17:14:37 -0700504 * - different from the way of handling in mlock etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 */
Prasanna Meda05b74382005-06-21 17:14:37 -0700506 vma = find_vma_prev(current->mm, start, &prev);
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700507 if (vma && start > vma->vm_start)
508 prev = vma;
509
Shaohua Li1998cc02013-02-22 16:32:31 -0800510 blk_start_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 for (;;) {
512 /* Still start < end. */
513 error = -ENOMEM;
514 if (!vma)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700515 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516
Prasanna Meda05b74382005-06-21 17:14:37 -0700517 /* Here start < (end|vma->vm_end). */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 if (start < vma->vm_start) {
519 unmapped_error = -ENOMEM;
520 start = vma->vm_start;
Prasanna Meda05b74382005-06-21 17:14:37 -0700521 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700522 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 }
524
Prasanna Meda05b74382005-06-21 17:14:37 -0700525 /* Here vma->vm_start <= start < (end|vma->vm_end) */
526 tmp = vma->vm_end;
527 if (end < tmp)
528 tmp = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529
Prasanna Meda05b74382005-06-21 17:14:37 -0700530 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
531 error = madvise_vma(vma, &prev, start, tmp, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 if (error)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700533 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700534 start = tmp;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700535 if (prev && start < prev->vm_end)
Prasanna Meda05b74382005-06-21 17:14:37 -0700536 start = prev->vm_end;
537 error = unmapped_error;
538 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700539 goto out;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700540 if (prev)
541 vma = prev->vm_next;
542 else /* madvise_remove dropped mmap_sem */
543 vma = find_vma(current->mm, start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545out:
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700546 blk_finish_plug(&plug);
Jason Baronf7977792007-07-15 23:38:21 -0700547 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700548 up_write(&current->mm->mmap_sem);
549 else
550 up_read(&current->mm->mmap_sem);
551
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 return error;
553}