blob: a3af058f68e4d9f434337d0dcd6127d5d8c6d039 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * mm/mprotect.c
3 *
4 * (C) Copyright 1994 Linus Torvalds
5 * (C) Copyright 2002 Christoph Hellwig
6 *
Alan Cox046c6882009-01-05 14:06:29 +00007 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
9 */
10
11#include <linux/mm.h>
12#include <linux/hugetlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/shm.h>
14#include <linux/mman.h>
15#include <linux/fs.h>
16#include <linux/highmem.h>
17#include <linux/security.h>
18#include <linux/mempolicy.h>
19#include <linux/personality.h>
20#include <linux/syscalls.h>
Christoph Lameter06972122006-06-23 02:03:35 -070021#include <linux/swap.h>
22#include <linux/swapops.h>
Andrea Arcangelicddb8a52008-07-28 15:46:29 -070023#include <linux/mmu_notifier.h>
KOSAKI Motohiro64cdd542009-01-06 14:39:16 -080024#include <linux/migrate.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020025#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <asm/uaccess.h>
27#include <asm/pgtable.h>
28#include <asm/cacheflush.h>
29#include <asm/tlbflush.h>
30
Venki Pallipadi1c12c4c2008-05-14 16:05:51 -070031#ifndef pgprot_modify
32static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
33{
34 return newprot;
35}
36#endif
37
Mel Gorman4b10e7d2012-10-25 14:16:32 +020038static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Peter Zijlstrac1e60982006-09-25 23:30:59 -070039 unsigned long addr, unsigned long end, pgprot_t newprot,
Mel Gorman9532fec2012-11-15 01:24:32 +000040 int dirty_accountable, int prot_numa, bool *ret_all_same_node)
Linus Torvalds1da177e2005-04-16 15:20:36 -070041{
Mel Gorman4b10e7d2012-10-25 14:16:32 +020042 struct mm_struct *mm = vma->vm_mm;
Christoph Lameter06972122006-06-23 02:03:35 -070043 pte_t *pte, oldpte;
Hugh Dickins705e87c2005-10-29 18:16:27 -070044 spinlock_t *ptl;
Peter Zijlstra7da4d642012-11-19 03:14:23 +010045 unsigned long pages = 0;
Mel Gorman9532fec2012-11-15 01:24:32 +000046 bool all_same_node = true;
47 int last_nid = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
Hugh Dickins705e87c2005-10-29 18:16:27 -070049 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
Zachary Amsden6606c3e2006-09-30 23:29:33 -070050 arch_enter_lazy_mmu_mode();
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 do {
Christoph Lameter06972122006-06-23 02:03:35 -070052 oldpte = *pte;
53 if (pte_present(oldpte)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 pte_t ptent;
Mel Gorman4b10e7d2012-10-25 14:16:32 +020055 bool updated = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
Jeremy Fitzhardinge1ea07042008-06-16 04:30:00 -070057 ptent = ptep_modify_prot_start(mm, addr, pte);
Mel Gorman4b10e7d2012-10-25 14:16:32 +020058 if (!prot_numa) {
59 ptent = pte_modify(ptent, newprot);
60 updated = true;
61 } else {
62 struct page *page;
63
64 page = vm_normal_page(vma, addr, oldpte);
65 if (page) {
Mel Gorman9532fec2012-11-15 01:24:32 +000066 int this_nid = page_to_nid(page);
67 if (last_nid == -1)
68 last_nid = this_nid;
69 if (last_nid != this_nid)
70 all_same_node = false;
71
Mel Gorman4b10e7d2012-10-25 14:16:32 +020072 /* only check non-shared pages */
73 if (!pte_numa(oldpte) &&
74 page_mapcount(page) == 1) {
75 ptent = pte_mknuma(ptent);
76 updated = true;
77 }
78 }
79 }
Jeremy Fitzhardinge1ea07042008-06-16 04:30:00 -070080
Peter Zijlstrac1e60982006-09-25 23:30:59 -070081 /*
82 * Avoid taking write faults for pages we know to be
83 * dirty.
84 */
Mel Gorman4b10e7d2012-10-25 14:16:32 +020085 if (dirty_accountable && pte_dirty(ptent)) {
Peter Zijlstrac1e60982006-09-25 23:30:59 -070086 ptent = pte_mkwrite(ptent);
Mel Gorman4b10e7d2012-10-25 14:16:32 +020087 updated = true;
88 }
Jeremy Fitzhardinge1ea07042008-06-16 04:30:00 -070089
Mel Gorman4b10e7d2012-10-25 14:16:32 +020090 if (updated)
91 pages++;
Jeremy Fitzhardinge1ea07042008-06-16 04:30:00 -070092 ptep_modify_prot_commit(mm, addr, pte, ptent);
Konstantin Khlebnikovce1744f2012-03-21 16:33:59 -070093 } else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) {
Christoph Lameter06972122006-06-23 02:03:35 -070094 swp_entry_t entry = pte_to_swp_entry(oldpte);
95
96 if (is_write_migration_entry(entry)) {
Cyrill Gorcunovc3d16e12013-10-16 13:46:51 -070097 pte_t newpte;
Christoph Lameter06972122006-06-23 02:03:35 -070098 /*
99 * A protection check is difficult so
100 * just be safe and disable write
101 */
102 make_migration_entry_read(&entry);
Cyrill Gorcunovc3d16e12013-10-16 13:46:51 -0700103 newpte = swp_entry_to_pte(entry);
104 if (pte_swp_soft_dirty(oldpte))
105 newpte = pte_swp_mksoft_dirty(newpte);
106 set_pte_at(mm, addr, pte, newpte);
Christoph Lameter06972122006-06-23 02:03:35 -0700107 }
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100108 pages++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109 }
110 } while (pte++, addr += PAGE_SIZE, addr != end);
Zachary Amsden6606c3e2006-09-30 23:29:33 -0700111 arch_leave_lazy_mmu_mode();
Hugh Dickins705e87c2005-10-29 18:16:27 -0700112 pte_unmap_unlock(pte - 1, ptl);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100113
Mel Gorman9532fec2012-11-15 01:24:32 +0000114 *ret_all_same_node = all_same_node;
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100115 return pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116}
117
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200118#ifdef CONFIG_NUMA_BALANCING
119static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr,
Andrew Morton7d12efa2012-12-18 14:23:17 -0800120 pmd_t *pmd)
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200121{
122 spin_lock(&mm->page_table_lock);
123 set_pmd_at(mm, addr & PMD_MASK, pmd, pmd_mknuma(*pmd));
124 spin_unlock(&mm->page_table_lock);
125}
126#else
127static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr,
Andrew Morton7d12efa2012-12-18 14:23:17 -0800128 pmd_t *pmd)
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200129{
130 BUG();
131}
132#endif /* CONFIG_NUMA_BALANCING */
133
Andrew Morton7d12efa2012-12-18 14:23:17 -0800134static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
135 pud_t *pud, unsigned long addr, unsigned long end,
136 pgprot_t newprot, int dirty_accountable, int prot_numa)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137{
138 pmd_t *pmd;
139 unsigned long next;
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100140 unsigned long pages = 0;
Mel Gorman9532fec2012-11-15 01:24:32 +0000141 bool all_same_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
143 pmd = pmd_offset(pud, addr);
144 do {
145 next = pmd_addr_end(addr, end);
Johannes Weinercd7548a2011-01-13 15:47:04 -0800146 if (pmd_trans_huge(*pmd)) {
147 if (next - addr != HPAGE_PMD_SIZE)
Kirill A. Shutemove1803772012-12-12 13:50:59 -0800148 split_huge_page_pmd(vma, addr, pmd);
Andrew Morton7d12efa2012-12-18 14:23:17 -0800149 else if (change_huge_pmd(vma, pmd, addr, newprot,
150 prot_numa)) {
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100151 pages += HPAGE_PMD_NR;
Johannes Weinercd7548a2011-01-13 15:47:04 -0800152 continue;
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100153 }
Johannes Weinercd7548a2011-01-13 15:47:04 -0800154 /* fall through */
155 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 if (pmd_none_or_clear_bad(pmd))
157 continue;
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200158 pages += change_pte_range(vma, pmd, addr, next, newprot,
Mel Gorman9532fec2012-11-15 01:24:32 +0000159 dirty_accountable, prot_numa, &all_same_node);
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200160
Mel Gorman9532fec2012-11-15 01:24:32 +0000161 /*
162 * If we are changing protections for NUMA hinting faults then
163 * set pmd_numa if the examined pages were all on the same
164 * node. This allows a regular PMD to be handled as one fault
165 * and effectively batches the taking of the PTL
166 */
167 if (prot_numa && all_same_node)
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200168 change_pmd_protnuma(vma->vm_mm, addr, pmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 } while (pmd++, addr = next, addr != end);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100170
171 return pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172}
173
Andrew Morton7d12efa2012-12-18 14:23:17 -0800174static inline unsigned long change_pud_range(struct vm_area_struct *vma,
175 pgd_t *pgd, unsigned long addr, unsigned long end,
176 pgprot_t newprot, int dirty_accountable, int prot_numa)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177{
178 pud_t *pud;
179 unsigned long next;
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100180 unsigned long pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
182 pud = pud_offset(pgd, addr);
183 do {
184 next = pud_addr_end(addr, end);
185 if (pud_none_or_clear_bad(pud))
186 continue;
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100187 pages += change_pmd_range(vma, pud, addr, next, newprot,
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200188 dirty_accountable, prot_numa);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 } while (pud++, addr = next, addr != end);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100190
191 return pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192}
193
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100194static unsigned long change_protection_range(struct vm_area_struct *vma,
Peter Zijlstrac1e60982006-09-25 23:30:59 -0700195 unsigned long addr, unsigned long end, pgprot_t newprot,
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200196 int dirty_accountable, int prot_numa)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197{
198 struct mm_struct *mm = vma->vm_mm;
199 pgd_t *pgd;
200 unsigned long next;
201 unsigned long start = addr;
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100202 unsigned long pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203
204 BUG_ON(addr >= end);
205 pgd = pgd_offset(mm, addr);
206 flush_cache_range(vma, addr, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 do {
208 next = pgd_addr_end(addr, end);
209 if (pgd_none_or_clear_bad(pgd))
210 continue;
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100211 pages += change_pud_range(vma, pgd, addr, next, newprot,
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200212 dirty_accountable, prot_numa);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 } while (pgd++, addr = next, addr != end);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100214
Ingo Molnar1233d582012-11-19 03:14:24 +0100215 /* Only flush the TLB if we actually modified any entries: */
216 if (pages)
217 flush_tlb_range(vma, start, end);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100218
219 return pages;
220}
221
222unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
223 unsigned long end, pgprot_t newprot,
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200224 int dirty_accountable, int prot_numa)
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100225{
226 struct mm_struct *mm = vma->vm_mm;
227 unsigned long pages;
228
229 mmu_notifier_invalidate_range_start(mm, start, end);
230 if (is_vm_hugetlb_page(vma))
231 pages = hugetlb_change_protection(vma, start, end, newprot);
232 else
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200233 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100234 mmu_notifier_invalidate_range_end(mm, start, end);
235
236 return pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237}
238
Ollie Wildb6a2fea2007-07-19 01:48:16 -0700239int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
241 unsigned long start, unsigned long end, unsigned long newflags)
242{
243 struct mm_struct *mm = vma->vm_mm;
244 unsigned long oldflags = vma->vm_flags;
245 long nrpages = (end - start) >> PAGE_SHIFT;
246 unsigned long charged = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 pgoff_t pgoff;
248 int error;
Peter Zijlstrac1e60982006-09-25 23:30:59 -0700249 int dirty_accountable = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250
251 if (newflags == oldflags) {
252 *pprev = vma;
253 return 0;
254 }
255
256 /*
257 * If we make a private mapping writable we increase our commit;
258 * but (without finer accounting) cannot reduce our commit if we
Mel Gorman5a6fe122009-02-10 14:02:27 +0000259 * make it unwritable again. hugetlb mapping were accounted for
260 * even if read-only so there is no need to account for them here
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 */
262 if (newflags & VM_WRITE) {
Mel Gorman5a6fe122009-02-10 14:02:27 +0000263 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
Andy Whitcroftcdfd4322008-07-23 21:27:28 -0700264 VM_SHARED|VM_NORESERVE))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 charged = nrpages;
Al Viro191c5422012-02-13 03:58:52 +0000266 if (security_vm_enough_memory_mm(mm, charged))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 return -ENOMEM;
268 newflags |= VM_ACCOUNT;
269 }
270 }
271
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 /*
273 * First try to merge with previous and/or next vma.
274 */
275 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
276 *pprev = vma_merge(mm, *pprev, start, end, newflags,
277 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
278 if (*pprev) {
279 vma = *pprev;
280 goto success;
281 }
282
283 *pprev = vma;
284
285 if (start != vma->vm_start) {
286 error = split_vma(mm, vma, start, 1);
287 if (error)
288 goto fail;
289 }
290
291 if (end != vma->vm_end) {
292 error = split_vma(mm, vma, end, 0);
293 if (error)
294 goto fail;
295 }
296
297success:
298 /*
299 * vm_flags and vm_page_prot are protected by the mmap_sem
300 * held in write mode.
301 */
302 vma->vm_flags = newflags;
Venki Pallipadi1c12c4c2008-05-14 16:05:51 -0700303 vma->vm_page_prot = pgprot_modify(vma->vm_page_prot,
304 vm_get_page_prot(newflags));
305
Peter Zijlstrac1e60982006-09-25 23:30:59 -0700306 if (vma_wants_writenotify(vma)) {
Hugh Dickins1ddd4392007-10-22 20:45:12 -0700307 vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED);
Peter Zijlstrac1e60982006-09-25 23:30:59 -0700308 dirty_accountable = 1;
309 }
Peter Zijlstrad08b3852006-09-25 23:30:57 -0700310
Andrew Morton7d12efa2012-12-18 14:23:17 -0800311 change_protection(vma, start, end, vma->vm_page_prot,
312 dirty_accountable, 0);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100313
Hugh Dickinsab50b8e2005-10-29 18:15:56 -0700314 vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
315 vm_stat_account(mm, newflags, vma->vm_file, nrpages);
Pekka Enberg63bfd732010-11-08 21:29:07 +0200316 perf_event_mmap(vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 return 0;
318
319fail:
320 vm_unacct_memory(charged);
321 return error;
322}
323
Heiko Carstens6a6160a2009-01-14 14:14:15 +0100324SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
325 unsigned long, prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326{
327 unsigned long vm_flags, nstart, end, tmp, reqprot;
328 struct vm_area_struct *vma, *prev;
329 int error = -EINVAL;
330 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
331 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
332 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
333 return -EINVAL;
334
335 if (start & ~PAGE_MASK)
336 return -EINVAL;
337 if (!len)
338 return 0;
339 len = PAGE_ALIGN(len);
340 end = start + len;
341 if (end <= start)
342 return -ENOMEM;
Dave Kleikampb845f312008-07-08 00:28:51 +1000343 if (!arch_validate_prot(prot))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 return -EINVAL;
345
346 reqprot = prot;
347 /*
348 * Does the application expect PROT_READ to imply PROT_EXEC:
349 */
Hua Zhongb344e052006-06-23 02:03:23 -0700350 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 prot |= PROT_EXEC;
352
353 vm_flags = calc_vm_prot_bits(prot);
354
355 down_write(&current->mm->mmap_sem);
356
Linus Torvalds097d5912012-03-06 18:23:36 -0800357 vma = find_vma(current->mm, start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 error = -ENOMEM;
359 if (!vma)
360 goto out;
Linus Torvalds097d5912012-03-06 18:23:36 -0800361 prev = vma->vm_prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 if (unlikely(grows & PROT_GROWSDOWN)) {
363 if (vma->vm_start >= end)
364 goto out;
365 start = vma->vm_start;
366 error = -EINVAL;
367 if (!(vma->vm_flags & VM_GROWSDOWN))
368 goto out;
Andrew Morton7d12efa2012-12-18 14:23:17 -0800369 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 if (vma->vm_start > start)
371 goto out;
372 if (unlikely(grows & PROT_GROWSUP)) {
373 end = vma->vm_end;
374 error = -EINVAL;
375 if (!(vma->vm_flags & VM_GROWSUP))
376 goto out;
377 }
378 }
379 if (start > vma->vm_start)
380 prev = vma;
381
382 for (nstart = start ; ; ) {
383 unsigned long newflags;
384
Andrew Morton7d12efa2012-12-18 14:23:17 -0800385 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
Andrew Morton7d12efa2012-12-18 14:23:17 -0800387 newflags = vm_flags;
388 newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389
Paolo 'Blaisorblade' Giarrusso7e2cff42005-09-21 09:55:39 -0700390 /* newflags >> 4 shift VM_MAY% in place of VM_% */
391 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 error = -EACCES;
393 goto out;
394 }
395
396 error = security_file_mprotect(vma, reqprot, prot);
397 if (error)
398 goto out;
399
400 tmp = vma->vm_end;
401 if (tmp > end)
402 tmp = end;
403 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
404 if (error)
405 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 nstart = tmp;
407
408 if (nstart < prev->vm_end)
409 nstart = prev->vm_end;
410 if (nstart >= end)
411 goto out;
412
413 vma = prev->vm_next;
414 if (!vma || vma->vm_start != nstart) {
415 error = -ENOMEM;
416 goto out;
417 }
418 }
419out:
420 up_write(&current->mm->mmap_sem);
421 return error;
422}