blob: c889fcbb530e98d8779ef75750e1fde08bf786cf [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/madvise.c
3 *
4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig
6 */
7
8#include <linux/mman.h>
9#include <linux/pagemap.h>
10#include <linux/syscalls.h>
Prasanna Meda05b74382005-06-21 17:14:37 -070011#include <linux/mempolicy.h>
Andi Kleenafcf9382009-12-16 12:20:00 +010012#include <linux/page-isolation.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/hugetlb.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070014#include <linux/falloc.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040015#include <linux/sched.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070016#include <linux/ksm.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070017#include <linux/fs.h>
Andy Lutomirski9ab42332012-07-05 16:00:11 -070018#include <linux/file.h>
Shaohua Li1998cc02013-02-22 16:32:31 -080019#include <linux/blkdev.h>
Tejun Heo66114ca2015-05-22 17:13:32 -040020#include <linux/backing-dev.h>
Shaohua Li1998cc02013-02-22 16:32:31 -080021#include <linux/swap.h>
22#include <linux/swapops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24/*
Nick Piggin0a27a142007-05-06 14:49:53 -070025 * Any behaviour which results in changes to the vma->vm_flags needs to
26 * take mmap_sem for writing. Others, which simply traverse vmas, need
27 * to only take it for reading.
28 */
29static int madvise_need_mmap_write(int behavior)
30{
31 switch (behavior) {
32 case MADV_REMOVE:
33 case MADV_WILLNEED:
34 case MADV_DONTNEED:
35 return 0;
36 default:
37 /* be safe, default to 1. list exceptions explicitly */
38 return 1;
39 }
40}
41
42/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070043 * We can potentially split a vm area into separate
44 * areas, each area with its own behavior.
45 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -070046static long madvise_behavior(struct vm_area_struct *vma,
Prasanna Meda05b74382005-06-21 17:14:37 -070047 struct vm_area_struct **prev,
48 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -070049{
Vladimir Cernovec9bed92013-09-11 14:20:15 -070050 struct mm_struct *mm = vma->vm_mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 int error = 0;
Prasanna Meda05b74382005-06-21 17:14:37 -070052 pgoff_t pgoff;
Hugh Dickins3866ea92009-09-21 17:01:52 -070053 unsigned long new_flags = vma->vm_flags;
Prasanna Medae798c6e2005-06-21 17:14:36 -070054
55 switch (behavior) {
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080056 case MADV_NORMAL:
57 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
58 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070059 case MADV_SEQUENTIAL:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080060 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070061 break;
62 case MADV_RANDOM:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080063 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070064 break;
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080065 case MADV_DONTFORK:
66 new_flags |= VM_DONTCOPY;
67 break;
68 case MADV_DOFORK:
Hugh Dickins3866ea92009-09-21 17:01:52 -070069 if (vma->vm_flags & VM_IO) {
70 error = -EINVAL;
71 goto out;
72 }
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080073 new_flags &= ~VM_DONTCOPY;
Prasanna Medae798c6e2005-06-21 17:14:36 -070074 break;
Jason Baronaccb61f2012-03-23 15:02:51 -070075 case MADV_DONTDUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070076 new_flags |= VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070077 break;
78 case MADV_DODUMP:
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -070079 if (new_flags & VM_SPECIAL) {
80 error = -EINVAL;
81 goto out;
82 }
83 new_flags &= ~VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070084 break;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070085 case MADV_MERGEABLE:
86 case MADV_UNMERGEABLE:
87 error = ksm_madvise(vma, start, end, behavior, &new_flags);
88 if (error)
89 goto out;
90 break;
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080091 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -080092 case MADV_NOHUGEPAGE:
Andrea Arcangeli60ab3242011-01-13 15:47:18 -080093 error = hugepage_madvise(vma, &new_flags, behavior);
Andrea Arcangeli0af4e982011-01-13 15:46:55 -080094 if (error)
95 goto out;
96 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070097 }
98
Prasanna Meda05b74382005-06-21 17:14:37 -070099 if (new_flags == vma->vm_flags) {
100 *prev = vma;
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700101 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700102 }
103
104 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
105 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
Andrea Arcangeli19a809a2015-09-04 15:46:24 -0700106 vma->vm_file, pgoff, vma_policy(vma),
107 vma->vm_userfaultfd_ctx);
Prasanna Meda05b74382005-06-21 17:14:37 -0700108 if (*prev) {
109 vma = *prev;
110 goto success;
111 }
112
113 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
115 if (start != vma->vm_start) {
116 error = split_vma(mm, vma, start, 1);
117 if (error)
118 goto out;
119 }
120
121 if (end != vma->vm_end) {
122 error = split_vma(mm, vma, end, 0);
123 if (error)
124 goto out;
125 }
126
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700127success:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 /*
129 * vm_flags is protected by the mmap_sem held in write mode.
130 */
Prasanna Medae798c6e2005-06-21 17:14:36 -0700131 vma->vm_flags = new_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132
133out:
134 if (error == -ENOMEM)
135 error = -EAGAIN;
136 return error;
137}
138
Shaohua Li1998cc02013-02-22 16:32:31 -0800139#ifdef CONFIG_SWAP
140static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
141 unsigned long end, struct mm_walk *walk)
142{
143 pte_t *orig_pte;
144 struct vm_area_struct *vma = walk->private;
145 unsigned long index;
146
147 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
148 return 0;
149
150 for (index = start; index != end; index += PAGE_SIZE) {
151 pte_t pte;
152 swp_entry_t entry;
153 struct page *page;
154 spinlock_t *ptl;
155
156 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
157 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
158 pte_unmap_unlock(orig_pte, ptl);
159
Kirill A. Shutemov0661a332015-02-10 14:10:04 -0800160 if (pte_present(pte) || pte_none(pte))
Shaohua Li1998cc02013-02-22 16:32:31 -0800161 continue;
162 entry = pte_to_swp_entry(pte);
163 if (unlikely(non_swap_entry(entry)))
164 continue;
165
166 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
167 vma, index);
168 if (page)
169 page_cache_release(page);
170 }
171
172 return 0;
173}
174
175static void force_swapin_readahead(struct vm_area_struct *vma,
176 unsigned long start, unsigned long end)
177{
178 struct mm_walk walk = {
179 .mm = vma->vm_mm,
180 .pmd_entry = swapin_walk_pmd_entry,
181 .private = vma,
182 };
183
184 walk_page_range(start, end, &walk);
185
186 lru_add_drain(); /* Push any new pages onto the LRU now */
187}
188
189static void force_shm_swapin_readahead(struct vm_area_struct *vma,
190 unsigned long start, unsigned long end,
191 struct address_space *mapping)
192{
193 pgoff_t index;
194 struct page *page;
195 swp_entry_t swap;
196
197 for (; start < end; start += PAGE_SIZE) {
198 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
199
Johannes Weiner55231e52014-05-22 11:54:17 -0700200 page = find_get_entry(mapping, index);
Shaohua Li1998cc02013-02-22 16:32:31 -0800201 if (!radix_tree_exceptional_entry(page)) {
202 if (page)
203 page_cache_release(page);
204 continue;
205 }
206 swap = radix_to_swp_entry(page);
207 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
208 NULL, 0);
209 if (page)
210 page_cache_release(page);
211 }
212
213 lru_add_drain(); /* Push any new pages onto the LRU now */
214}
215#endif /* CONFIG_SWAP */
216
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217/*
218 * Schedule all required I/O operations. Do not wait for completion.
219 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700220static long madvise_willneed(struct vm_area_struct *vma,
221 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222 unsigned long start, unsigned long end)
223{
224 struct file *file = vma->vm_file;
225
Shaohua Li1998cc02013-02-22 16:32:31 -0800226#ifdef CONFIG_SWAP
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100227 if (!file) {
Shaohua Li1998cc02013-02-22 16:32:31 -0800228 *prev = vma;
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100229 force_swapin_readahead(vma, start, end);
Shaohua Li1998cc02013-02-22 16:32:31 -0800230 return 0;
231 }
Shaohua Li1998cc02013-02-22 16:32:31 -0800232
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100233 if (shmem_mapping(file->f_mapping)) {
234 *prev = vma;
235 force_shm_swapin_readahead(vma, start, end,
236 file->f_mapping);
237 return 0;
238 }
239#else
Suzuki1bef4002005-10-11 08:29:06 -0700240 if (!file)
241 return -EBADF;
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100242#endif
Suzuki1bef4002005-10-11 08:29:06 -0700243
Matthew Wilcoxe748dcd2015-02-16 15:59:12 -0800244 if (IS_DAX(file_inode(file))) {
Carsten Ottefe77ba62005-06-23 22:05:29 -0700245 /* no bad return value, but ignore advice */
246 return 0;
247 }
248
Prasanna Meda05b74382005-06-21 17:14:37 -0700249 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
251 if (end > vma->vm_end)
252 end = vma->vm_end;
253 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
254
Wu Fengguangf7e839d2009-06-16 15:31:20 -0700255 force_page_cache_readahead(file->f_mapping, file, start, end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 return 0;
257}
258
259/*
260 * Application no longer needs these pages. If the pages are dirty,
261 * it's OK to just throw them away. The app will be more careful about
262 * data it wants to keep. Be sure to free swap resources too. The
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700263 * zap_page_range call sets things up for shrink_active_list to actually free
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 * these pages later if no one else has touched them in the meantime,
265 * although we could add these pages to a global reuse list for
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700266 * shrink_active_list to pick up before reclaiming other pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 *
268 * NB: This interface discards data rather than pushes it out to swap,
269 * as some implementations do. This has performance implications for
270 * applications like large transactional databases which want to discard
271 * pages in anonymous maps after committing to backing store the data
272 * that was kept in them. There is no reason to write this data out to
273 * the swap area if the application is discarding it.
274 *
275 * An interface that causes the system to free clean pages and flush
276 * dirty pages is already available as msync(MS_INVALIDATE).
277 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700278static long madvise_dontneed(struct vm_area_struct *vma,
279 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 unsigned long start, unsigned long end)
281{
Prasanna Meda05b74382005-06-21 17:14:37 -0700282 *prev = vma;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800283 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 return -EINVAL;
285
Kirill A. Shutemov8a5f14a2015-02-10 14:09:49 -0800286 zap_page_range(vma, start, end - start, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 return 0;
288}
289
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800290/*
291 * Application wants to free up the pages and associated backing store.
292 * This is effectively punching a hole into the middle of a file.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800293 */
294static long madvise_remove(struct vm_area_struct *vma,
Nick Piggin00e9fa22007-03-16 13:38:10 -0800295 struct vm_area_struct **prev,
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800296 unsigned long start, unsigned long end)
297{
Hugh Dickins3f31d072012-05-29 15:06:40 -0700298 loff_t offset;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700299 int error;
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700300 struct file *f;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800301
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700302 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
Nick Piggin00e9fa22007-03-16 13:38:10 -0800303
Mike Kravetz72079ba2015-09-08 15:01:57 -0700304 if (vma->vm_flags & VM_LOCKED)
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800305 return -EINVAL;
306
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700307 f = vma->vm_file;
308
309 if (!f || !f->f_mapping || !f->f_mapping->host) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800310 return -EINVAL;
311 }
312
Hugh Dickins69cf0fa2006-04-17 22:46:32 +0100313 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
314 return -EACCES;
315
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800316 offset = (loff_t)(start - vma->vm_start)
317 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700318
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700319 /*
320 * Filesystem's fallocate may need to take i_mutex. We need to
321 * explicitly grab a reference because the vma (and hence the
322 * vma's reference to the file) can go away as soon as we drop
323 * mmap_sem.
324 */
325 get_file(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700326 up_read(&current->mm->mmap_sem);
Anna Schumaker72c72bd2014-11-07 14:44:25 -0500327 error = vfs_fallocate(f,
Hugh Dickins3f31d072012-05-29 15:06:40 -0700328 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
329 offset, end - start);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700330 fput(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700331 down_read(&current->mm->mmap_sem);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700332 return error;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800333}
334
Andi Kleen9893e492009-09-16 11:50:17 +0200335#ifdef CONFIG_MEMORY_FAILURE
336/*
337 * Error injection support for memory error handling.
338 */
Andi Kleenafcf9382009-12-16 12:20:00 +0100339static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
Andi Kleen9893e492009-09-16 11:50:17 +0200340{
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700341 struct page *p;
Andi Kleen9893e492009-09-16 11:50:17 +0200342 if (!capable(CAP_SYS_ADMIN))
343 return -EPERM;
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700344 for (; start < end; start += PAGE_SIZE <<
345 compound_order(compound_head(p))) {
Andrew Morton325c4ef2013-09-11 14:23:03 -0700346 int ret;
347
348 ret = get_user_pages_fast(start, 1, 0, &p);
Andi Kleen9893e492009-09-16 11:50:17 +0200349 if (ret != 1)
350 return ret;
Andrew Morton325c4ef2013-09-11 14:23:03 -0700351
Wanpeng Li29b4eed2013-09-11 14:22:59 -0700352 if (PageHWPoison(p)) {
353 put_page(p);
354 continue;
355 }
Andi Kleenafcf9382009-12-16 12:20:00 +0100356 if (bhv == MADV_SOFT_OFFLINE) {
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700357 pr_info("Soft offlining page %#lx at %#lx\n",
Andi Kleenafcf9382009-12-16 12:20:00 +0100358 page_to_pfn(p), start);
359 ret = soft_offline_page(p, MF_COUNT_INCREASED);
360 if (ret)
Wanpeng Li83024232013-09-11 14:23:02 -0700361 return ret;
Andi Kleenafcf9382009-12-16 12:20:00 +0100362 continue;
363 }
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700364 pr_info("Injecting memory failure for page %#lx at %#lx\n",
Andi Kleen9893e492009-09-16 11:50:17 +0200365 page_to_pfn(p), start);
366 /* Ignore return value for now */
Tony Luckcd42f4a2011-12-15 10:48:12 -0800367 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
Andi Kleen9893e492009-09-16 11:50:17 +0200368 }
Andrew Morton325c4ef2013-09-11 14:23:03 -0700369 return 0;
Andi Kleen9893e492009-09-16 11:50:17 +0200370}
371#endif
372
suzuki165cd402005-07-27 11:43:59 -0700373static long
374madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
375 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 switch (behavior) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800378 case MADV_REMOVE:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700379 return madvise_remove(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 case MADV_WILLNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700381 return madvise_willneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 case MADV_DONTNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700383 return madvise_dontneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 default:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700385 return madvise_behavior(vma, prev, start, end, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387}
388
Nicholas Krause1ecef9e2015-09-04 15:48:24 -0700389static bool
Nick Piggin75927af2009-06-16 15:32:38 -0700390madvise_behavior_valid(int behavior)
391{
392 switch (behavior) {
393 case MADV_DOFORK:
394 case MADV_DONTFORK:
395 case MADV_NORMAL:
396 case MADV_SEQUENTIAL:
397 case MADV_RANDOM:
398 case MADV_REMOVE:
399 case MADV_WILLNEED:
400 case MADV_DONTNEED:
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700401#ifdef CONFIG_KSM
402 case MADV_MERGEABLE:
403 case MADV_UNMERGEABLE:
404#endif
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800405#ifdef CONFIG_TRANSPARENT_HUGEPAGE
406 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -0800407 case MADV_NOHUGEPAGE:
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800408#endif
Jason Baronaccb61f2012-03-23 15:02:51 -0700409 case MADV_DONTDUMP:
410 case MADV_DODUMP:
Nicholas Krause1ecef9e2015-09-04 15:48:24 -0700411 return true;
Nick Piggin75927af2009-06-16 15:32:38 -0700412
413 default:
Nicholas Krause1ecef9e2015-09-04 15:48:24 -0700414 return false;
Nick Piggin75927af2009-06-16 15:32:38 -0700415 }
416}
Hugh Dickins3866ea92009-09-21 17:01:52 -0700417
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418/*
419 * The madvise(2) system call.
420 *
421 * Applications can use madvise() to advise the kernel how it should
422 * handle paging I/O in this VM area. The idea is to help the kernel
423 * use appropriate read-ahead and caching techniques. The information
424 * provided is advisory only, and can be safely disregarded by the
425 * kernel without affecting the correct operation of the application.
426 *
427 * behavior values:
428 * MADV_NORMAL - the default behavior is to read clusters. This
429 * results in some read-ahead and read-behind.
430 * MADV_RANDOM - the system should read the minimum amount of data
431 * on any access, since it is unlikely that the appli-
432 * cation will need more than what it asks for.
433 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
434 * once, so they can be aggressively read ahead, and
435 * can be freed soon after they are accessed.
436 * MADV_WILLNEED - the application is notifying the system to read
437 * some pages ahead.
438 * MADV_DONTNEED - the application is finished with the given range,
439 * so the kernel can free resources associated with it.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800440 * MADV_REMOVE - the application wants to free up the given range of
441 * pages and associated backing store.
Hugh Dickins3866ea92009-09-21 17:01:52 -0700442 * MADV_DONTFORK - omit this area from child's address space when forking:
443 * typically, to avoid COWing pages pinned by get_user_pages().
444 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700445 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
446 * this area with pages of identical content from other such areas.
447 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 *
449 * return values:
450 * zero - success
451 * -EINVAL - start + len < 0, start is not page-aligned,
452 * "behavior" is not a valid value, or application
453 * is attempting to release locked or shared pages.
454 * -ENOMEM - addresses in the specified range are not currently
455 * mapped, or are outside the AS of the process.
456 * -EIO - an I/O error occurred while paging in data.
457 * -EBADF - map exists, but area maps something that isn't a file.
458 * -EAGAIN - a kernel resource was temporarily unavailable.
459 */
Heiko Carstens3480b252009-01-14 14:14:16 +0100460SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461{
Prasanna Meda05b74382005-06-21 17:14:37 -0700462 unsigned long end, tmp;
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700463 struct vm_area_struct *vma, *prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 int unmapped_error = 0;
465 int error = -EINVAL;
Jason Baronf7977792007-07-15 23:38:21 -0700466 int write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 size_t len;
Shaohua Li1998cc02013-02-22 16:32:31 -0800468 struct blk_plug plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
Andi Kleen9893e492009-09-16 11:50:17 +0200470#ifdef CONFIG_MEMORY_FAILURE
Andi Kleenafcf9382009-12-16 12:20:00 +0100471 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
472 return madvise_hwpoison(behavior, start, start+len_in);
Andi Kleen9893e492009-09-16 11:50:17 +0200473#endif
Nick Piggin75927af2009-06-16 15:32:38 -0700474 if (!madvise_behavior_valid(behavior))
475 return error;
476
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700477 if (start & ~PAGE_MASK)
478 return error;
479 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
480
481 /* Check to see whether len was rounded up from small -ve to zero */
482 if (len_in && !len)
483 return error;
484
485 end = start + len;
486 if (end < start)
487 return error;
488
489 error = 0;
490 if (end == start)
491 return error;
492
Jason Baronf7977792007-07-15 23:38:21 -0700493 write = madvise_need_mmap_write(behavior);
494 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700495 down_write(&current->mm->mmap_sem);
496 else
497 down_read(&current->mm->mmap_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 /*
500 * If the interval [start,end) covers some unmapped address
501 * ranges, just ignore them, but return -ENOMEM at the end.
Prasanna Meda05b74382005-06-21 17:14:37 -0700502 * - different from the way of handling in mlock etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 */
Prasanna Meda05b74382005-06-21 17:14:37 -0700504 vma = find_vma_prev(current->mm, start, &prev);
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700505 if (vma && start > vma->vm_start)
506 prev = vma;
507
Shaohua Li1998cc02013-02-22 16:32:31 -0800508 blk_start_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 for (;;) {
510 /* Still start < end. */
511 error = -ENOMEM;
512 if (!vma)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700513 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514
Prasanna Meda05b74382005-06-21 17:14:37 -0700515 /* Here start < (end|vma->vm_end). */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 if (start < vma->vm_start) {
517 unmapped_error = -ENOMEM;
518 start = vma->vm_start;
Prasanna Meda05b74382005-06-21 17:14:37 -0700519 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700520 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 }
522
Prasanna Meda05b74382005-06-21 17:14:37 -0700523 /* Here vma->vm_start <= start < (end|vma->vm_end) */
524 tmp = vma->vm_end;
525 if (end < tmp)
526 tmp = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527
Prasanna Meda05b74382005-06-21 17:14:37 -0700528 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
529 error = madvise_vma(vma, &prev, start, tmp, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 if (error)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700531 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700532 start = tmp;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700533 if (prev && start < prev->vm_end)
Prasanna Meda05b74382005-06-21 17:14:37 -0700534 start = prev->vm_end;
535 error = unmapped_error;
536 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700537 goto out;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700538 if (prev)
539 vma = prev->vm_next;
540 else /* madvise_remove dropped mmap_sem */
541 vma = find_vma(current->mm, start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543out:
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700544 blk_finish_plug(&plug);
Jason Baronf7977792007-07-15 23:38:21 -0700545 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700546 up_write(&current->mm->mmap_sem);
547 else
548 up_read(&current->mm->mmap_sem);
549
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 return error;
551}