blob: 7055883e6e250f2914ed5c0a7814400e4945c9d1 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/madvise.c
3 *
4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig
6 */
7
8#include <linux/mman.h>
9#include <linux/pagemap.h>
10#include <linux/syscalls.h>
Prasanna Meda05b74382005-06-21 17:14:37 -070011#include <linux/mempolicy.h>
Andi Kleenafcf9382009-12-16 12:20:00 +010012#include <linux/page-isolation.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/hugetlb.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070014#include <linux/falloc.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040015#include <linux/sched.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070016#include <linux/ksm.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070017#include <linux/fs.h>
Andy Lutomirski9ab42332012-07-05 16:00:11 -070018#include <linux/file.h>
Shaohua Li1998cc02013-02-22 16:32:31 -080019#include <linux/blkdev.h>
20#include <linux/swap.h>
21#include <linux/swapops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23/*
Nick Piggin0a27a142007-05-06 14:49:53 -070024 * Any behaviour which results in changes to the vma->vm_flags needs to
25 * take mmap_sem for writing. Others, which simply traverse vmas, need
26 * to only take it for reading.
27 */
28static int madvise_need_mmap_write(int behavior)
29{
30 switch (behavior) {
31 case MADV_REMOVE:
32 case MADV_WILLNEED:
33 case MADV_DONTNEED:
34 return 0;
35 default:
36 /* be safe, default to 1. list exceptions explicitly */
37 return 1;
38 }
39}
40
41/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 * We can potentially split a vm area into separate
43 * areas, each area with its own behavior.
44 */
Prasanna Meda05b74382005-06-21 17:14:37 -070045static long madvise_behavior(struct vm_area_struct * vma,
46 struct vm_area_struct **prev,
47 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -070048{
49 struct mm_struct * mm = vma->vm_mm;
50 int error = 0;
Prasanna Meda05b74382005-06-21 17:14:37 -070051 pgoff_t pgoff;
Hugh Dickins3866ea92009-09-21 17:01:52 -070052 unsigned long new_flags = vma->vm_flags;
Prasanna Medae798c6e2005-06-21 17:14:36 -070053
54 switch (behavior) {
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080055 case MADV_NORMAL:
56 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
57 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070058 case MADV_SEQUENTIAL:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080059 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070060 break;
61 case MADV_RANDOM:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080062 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070063 break;
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080064 case MADV_DONTFORK:
65 new_flags |= VM_DONTCOPY;
66 break;
67 case MADV_DOFORK:
Hugh Dickins3866ea92009-09-21 17:01:52 -070068 if (vma->vm_flags & VM_IO) {
69 error = -EINVAL;
70 goto out;
71 }
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080072 new_flags &= ~VM_DONTCOPY;
Prasanna Medae798c6e2005-06-21 17:14:36 -070073 break;
Jason Baronaccb61f2012-03-23 15:02:51 -070074 case MADV_DONTDUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070075 new_flags |= VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070076 break;
77 case MADV_DODUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070078 if (new_flags & VM_SPECIAL) {
79 error = -EINVAL;
80 goto out;
81 }
82 new_flags &= ~VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070083 break;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070084 case MADV_MERGEABLE:
85 case MADV_UNMERGEABLE:
86 error = ksm_madvise(vma, start, end, behavior, &new_flags);
87 if (error)
88 goto out;
89 break;
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080090 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -080091 case MADV_NOHUGEPAGE:
Andrea Arcangeli60ab3242011-01-13 15:47:18 -080092 error = hugepage_madvise(vma, &new_flags, behavior);
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080093 if (error)
94 goto out;
95 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070096 }
97
Prasanna Meda05b74382005-06-21 17:14:37 -070098 if (new_flags == vma->vm_flags) {
99 *prev = vma;
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700100 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700101 }
102
103 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
104 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
105 vma->vm_file, pgoff, vma_policy(vma));
106 if (*prev) {
107 vma = *prev;
108 goto success;
109 }
110
111 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
113 if (start != vma->vm_start) {
114 error = split_vma(mm, vma, start, 1);
115 if (error)
116 goto out;
117 }
118
119 if (end != vma->vm_end) {
120 error = split_vma(mm, vma, end, 0);
121 if (error)
122 goto out;
123 }
124
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700125success:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 /*
127 * vm_flags is protected by the mmap_sem held in write mode.
128 */
Prasanna Medae798c6e2005-06-21 17:14:36 -0700129 vma->vm_flags = new_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130
131out:
132 if (error == -ENOMEM)
133 error = -EAGAIN;
134 return error;
135}
136
Shaohua Li1998cc02013-02-22 16:32:31 -0800137#ifdef CONFIG_SWAP
138static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
139 unsigned long end, struct mm_walk *walk)
140{
141 pte_t *orig_pte;
142 struct vm_area_struct *vma = walk->private;
143 unsigned long index;
144
145 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
146 return 0;
147
148 for (index = start; index != end; index += PAGE_SIZE) {
149 pte_t pte;
150 swp_entry_t entry;
151 struct page *page;
152 spinlock_t *ptl;
153
154 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
155 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
156 pte_unmap_unlock(orig_pte, ptl);
157
158 if (pte_present(pte) || pte_none(pte) || pte_file(pte))
159 continue;
160 entry = pte_to_swp_entry(pte);
161 if (unlikely(non_swap_entry(entry)))
162 continue;
163
164 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
165 vma, index);
166 if (page)
167 page_cache_release(page);
168 }
169
170 return 0;
171}
172
173static void force_swapin_readahead(struct vm_area_struct *vma,
174 unsigned long start, unsigned long end)
175{
176 struct mm_walk walk = {
177 .mm = vma->vm_mm,
178 .pmd_entry = swapin_walk_pmd_entry,
179 .private = vma,
180 };
181
182 walk_page_range(start, end, &walk);
183
184 lru_add_drain(); /* Push any new pages onto the LRU now */
185}
186
187static void force_shm_swapin_readahead(struct vm_area_struct *vma,
188 unsigned long start, unsigned long end,
189 struct address_space *mapping)
190{
191 pgoff_t index;
192 struct page *page;
193 swp_entry_t swap;
194
195 for (; start < end; start += PAGE_SIZE) {
196 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
197
198 page = find_get_page(mapping, index);
199 if (!radix_tree_exceptional_entry(page)) {
200 if (page)
201 page_cache_release(page);
202 continue;
203 }
204 swap = radix_to_swp_entry(page);
205 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
206 NULL, 0);
207 if (page)
208 page_cache_release(page);
209 }
210
211 lru_add_drain(); /* Push any new pages onto the LRU now */
212}
213#endif /* CONFIG_SWAP */
214
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215/*
216 * Schedule all required I/O operations. Do not wait for completion.
217 */
218static long madvise_willneed(struct vm_area_struct * vma,
Prasanna Meda05b74382005-06-21 17:14:37 -0700219 struct vm_area_struct ** prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 unsigned long start, unsigned long end)
221{
222 struct file *file = vma->vm_file;
223
Shaohua Li1998cc02013-02-22 16:32:31 -0800224#ifdef CONFIG_SWAP
225 if (!file || mapping_cap_swap_backed(file->f_mapping)) {
226 *prev = vma;
227 if (!file)
228 force_swapin_readahead(vma, start, end);
229 else
230 force_shm_swapin_readahead(vma, start, end,
231 file->f_mapping);
232 return 0;
233 }
234#endif
235
Suzuki1bef4002005-10-11 08:29:06 -0700236 if (!file)
237 return -EBADF;
238
Nick Piggin70688e42008-04-28 02:13:02 -0700239 if (file->f_mapping->a_ops->get_xip_mem) {
Carsten Ottefe77ba62005-06-23 22:05:29 -0700240 /* no bad return value, but ignore advice */
241 return 0;
242 }
243
Prasanna Meda05b74382005-06-21 17:14:37 -0700244 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
246 if (end > vma->vm_end)
247 end = vma->vm_end;
248 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
249
Wu Fengguangf7e839d2009-06-16 15:31:20 -0700250 force_page_cache_readahead(file->f_mapping, file, start, end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 return 0;
252}
253
254/*
255 * Application no longer needs these pages. If the pages are dirty,
256 * it's OK to just throw them away. The app will be more careful about
257 * data it wants to keep. Be sure to free swap resources too. The
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700258 * zap_page_range call sets things up for shrink_active_list to actually free
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 * these pages later if no one else has touched them in the meantime,
260 * although we could add these pages to a global reuse list for
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700261 * shrink_active_list to pick up before reclaiming other pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 *
263 * NB: This interface discards data rather than pushes it out to swap,
264 * as some implementations do. This has performance implications for
265 * applications like large transactional databases which want to discard
266 * pages in anonymous maps after committing to backing store the data
267 * that was kept in them. There is no reason to write this data out to
268 * the swap area if the application is discarding it.
269 *
270 * An interface that causes the system to free clean pages and flush
271 * dirty pages is already available as msync(MS_INVALIDATE).
272 */
273static long madvise_dontneed(struct vm_area_struct * vma,
Prasanna Meda05b74382005-06-21 17:14:37 -0700274 struct vm_area_struct ** prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 unsigned long start, unsigned long end)
276{
Prasanna Meda05b74382005-06-21 17:14:37 -0700277 *prev = vma;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800278 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 return -EINVAL;
280
281 if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
282 struct zap_details details = {
283 .nonlinear_vma = vma,
284 .last_index = ULONG_MAX,
285 };
286 zap_page_range(vma, start, end - start, &details);
287 } else
288 zap_page_range(vma, start, end - start, NULL);
289 return 0;
290}
291
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800292/*
293 * Application wants to free up the pages and associated backing store.
294 * This is effectively punching a hole into the middle of a file.
295 *
296 * NOTE: Currently, only shmfs/tmpfs is supported for this operation.
297 * Other filesystems return -ENOSYS.
298 */
299static long madvise_remove(struct vm_area_struct *vma,
Nick Piggin00e9fa22007-03-16 13:38:10 -0800300 struct vm_area_struct **prev,
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800301 unsigned long start, unsigned long end)
302{
Hugh Dickins3f31d072012-05-29 15:06:40 -0700303 loff_t offset;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700304 int error;
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700305 struct file *f;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800306
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700307 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
Nick Piggin00e9fa22007-03-16 13:38:10 -0800308
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800309 if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
310 return -EINVAL;
311
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700312 f = vma->vm_file;
313
314 if (!f || !f->f_mapping || !f->f_mapping->host) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800315 return -EINVAL;
316 }
317
Hugh Dickins69cf0fa2006-04-17 22:46:32 +0100318 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
319 return -EACCES;
320
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800321 offset = (loff_t)(start - vma->vm_start)
322 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700323
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700324 /*
325 * Filesystem's fallocate may need to take i_mutex. We need to
326 * explicitly grab a reference because the vma (and hence the
327 * vma's reference to the file) can go away as soon as we drop
328 * mmap_sem.
329 */
330 get_file(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700331 up_read(&current->mm->mmap_sem);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700332 error = do_fallocate(f,
Hugh Dickins3f31d072012-05-29 15:06:40 -0700333 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
334 offset, end - start);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700335 fput(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700336 down_read(&current->mm->mmap_sem);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700337 return error;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800338}
339
Andi Kleen9893e492009-09-16 11:50:17 +0200340#ifdef CONFIG_MEMORY_FAILURE
341/*
342 * Error injection support for memory error handling.
343 */
Andi Kleenafcf9382009-12-16 12:20:00 +0100344static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
Andi Kleen9893e492009-09-16 11:50:17 +0200345{
346 int ret = 0;
347
348 if (!capable(CAP_SYS_ADMIN))
349 return -EPERM;
350 for (; start < end; start += PAGE_SIZE) {
351 struct page *p;
Andi Kleend15f1072009-12-16 12:20:00 +0100352 int ret = get_user_pages_fast(start, 1, 0, &p);
Andi Kleen9893e492009-09-16 11:50:17 +0200353 if (ret != 1)
354 return ret;
Andi Kleenafcf9382009-12-16 12:20:00 +0100355 if (bhv == MADV_SOFT_OFFLINE) {
356 printk(KERN_INFO "Soft offlining page %lx at %lx\n",
357 page_to_pfn(p), start);
358 ret = soft_offline_page(p, MF_COUNT_INCREASED);
359 if (ret)
360 break;
361 continue;
362 }
Andi Kleen9893e492009-09-16 11:50:17 +0200363 printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n",
364 page_to_pfn(p), start);
365 /* Ignore return value for now */
Tony Luckcd42f4a2011-12-15 10:48:12 -0800366 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
Andi Kleen9893e492009-09-16 11:50:17 +0200367 }
368 return ret;
369}
370#endif
371
suzuki165cd402005-07-27 11:43:59 -0700372static long
373madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
374 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 switch (behavior) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800377 case MADV_REMOVE:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700378 return madvise_remove(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 case MADV_WILLNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700380 return madvise_willneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 case MADV_DONTNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700382 return madvise_dontneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 default:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700384 return madvise_behavior(vma, prev, start, end, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386}
387
Nick Piggin75927af2009-06-16 15:32:38 -0700388static int
389madvise_behavior_valid(int behavior)
390{
391 switch (behavior) {
392 case MADV_DOFORK:
393 case MADV_DONTFORK:
394 case MADV_NORMAL:
395 case MADV_SEQUENTIAL:
396 case MADV_RANDOM:
397 case MADV_REMOVE:
398 case MADV_WILLNEED:
399 case MADV_DONTNEED:
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700400#ifdef CONFIG_KSM
401 case MADV_MERGEABLE:
402 case MADV_UNMERGEABLE:
403#endif
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800404#ifdef CONFIG_TRANSPARENT_HUGEPAGE
405 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -0800406 case MADV_NOHUGEPAGE:
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800407#endif
Jason Baronaccb61f2012-03-23 15:02:51 -0700408 case MADV_DONTDUMP:
409 case MADV_DODUMP:
Nick Piggin75927af2009-06-16 15:32:38 -0700410 return 1;
411
412 default:
413 return 0;
414 }
415}
Hugh Dickins3866ea92009-09-21 17:01:52 -0700416
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417/*
418 * The madvise(2) system call.
419 *
420 * Applications can use madvise() to advise the kernel how it should
421 * handle paging I/O in this VM area. The idea is to help the kernel
422 * use appropriate read-ahead and caching techniques. The information
423 * provided is advisory only, and can be safely disregarded by the
424 * kernel without affecting the correct operation of the application.
425 *
426 * behavior values:
427 * MADV_NORMAL - the default behavior is to read clusters. This
428 * results in some read-ahead and read-behind.
429 * MADV_RANDOM - the system should read the minimum amount of data
430 * on any access, since it is unlikely that the appli-
431 * cation will need more than what it asks for.
432 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
433 * once, so they can be aggressively read ahead, and
434 * can be freed soon after they are accessed.
435 * MADV_WILLNEED - the application is notifying the system to read
436 * some pages ahead.
437 * MADV_DONTNEED - the application is finished with the given range,
438 * so the kernel can free resources associated with it.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800439 * MADV_REMOVE - the application wants to free up the given range of
440 * pages and associated backing store.
Hugh Dickins3866ea92009-09-21 17:01:52 -0700441 * MADV_DONTFORK - omit this area from child's address space when forking:
442 * typically, to avoid COWing pages pinned by get_user_pages().
443 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700444 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
445 * this area with pages of identical content from other such areas.
446 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 *
448 * return values:
449 * zero - success
450 * -EINVAL - start + len < 0, start is not page-aligned,
451 * "behavior" is not a valid value, or application
452 * is attempting to release locked or shared pages.
453 * -ENOMEM - addresses in the specified range are not currently
454 * mapped, or are outside the AS of the process.
455 * -EIO - an I/O error occurred while paging in data.
456 * -EBADF - map exists, but area maps something that isn't a file.
457 * -EAGAIN - a kernel resource was temporarily unavailable.
458 */
Heiko Carstens3480b252009-01-14 14:14:16 +0100459SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460{
Prasanna Meda05b74382005-06-21 17:14:37 -0700461 unsigned long end, tmp;
462 struct vm_area_struct * vma, *prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463 int unmapped_error = 0;
464 int error = -EINVAL;
Jason Baronf7977792007-07-15 23:38:21 -0700465 int write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 size_t len;
Shaohua Li1998cc02013-02-22 16:32:31 -0800467 struct blk_plug plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468
Andi Kleen9893e492009-09-16 11:50:17 +0200469#ifdef CONFIG_MEMORY_FAILURE
Andi Kleenafcf9382009-12-16 12:20:00 +0100470 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
471 return madvise_hwpoison(behavior, start, start+len_in);
Andi Kleen9893e492009-09-16 11:50:17 +0200472#endif
Nick Piggin75927af2009-06-16 15:32:38 -0700473 if (!madvise_behavior_valid(behavior))
474 return error;
475
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700476 if (start & ~PAGE_MASK)
477 return error;
478 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
479
480 /* Check to see whether len was rounded up from small -ve to zero */
481 if (len_in && !len)
482 return error;
483
484 end = start + len;
485 if (end < start)
486 return error;
487
488 error = 0;
489 if (end == start)
490 return error;
491
Jason Baronf7977792007-07-15 23:38:21 -0700492 write = madvise_need_mmap_write(behavior);
493 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700494 down_write(&current->mm->mmap_sem);
495 else
496 down_read(&current->mm->mmap_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 /*
499 * If the interval [start,end) covers some unmapped address
500 * ranges, just ignore them, but return -ENOMEM at the end.
Prasanna Meda05b74382005-06-21 17:14:37 -0700501 * - different from the way of handling in mlock etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 */
Prasanna Meda05b74382005-06-21 17:14:37 -0700503 vma = find_vma_prev(current->mm, start, &prev);
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700504 if (vma && start > vma->vm_start)
505 prev = vma;
506
Shaohua Li1998cc02013-02-22 16:32:31 -0800507 blk_start_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 for (;;) {
509 /* Still start < end. */
510 error = -ENOMEM;
511 if (!vma)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700512 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513
Prasanna Meda05b74382005-06-21 17:14:37 -0700514 /* Here start < (end|vma->vm_end). */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 if (start < vma->vm_start) {
516 unmapped_error = -ENOMEM;
517 start = vma->vm_start;
Prasanna Meda05b74382005-06-21 17:14:37 -0700518 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700519 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 }
521
Prasanna Meda05b74382005-06-21 17:14:37 -0700522 /* Here vma->vm_start <= start < (end|vma->vm_end) */
523 tmp = vma->vm_end;
524 if (end < tmp)
525 tmp = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526
Prasanna Meda05b74382005-06-21 17:14:37 -0700527 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
528 error = madvise_vma(vma, &prev, start, tmp, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 if (error)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700530 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700531 start = tmp;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700532 if (prev && start < prev->vm_end)
Prasanna Meda05b74382005-06-21 17:14:37 -0700533 start = prev->vm_end;
534 error = unmapped_error;
535 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700536 goto out;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700537 if (prev)
538 vma = prev->vm_next;
539 else /* madvise_remove dropped mmap_sem */
540 vma = find_vma(current->mm, start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542out:
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700543 blk_finish_plug(&plug);
Jason Baronf7977792007-07-15 23:38:21 -0700544 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700545 up_write(&current->mm->mmap_sem);
546 else
547 up_read(&current->mm->mmap_sem);
548
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 return error;
550}