blob: 23a3e415ac2ce592bc0e7e21f3deb957e89dff7f [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Matt Mackalle6473092008-02-04 22:29:01 -08002#include <linux/mm.h>
3#include <linux/highmem.h>
4#include <linux/sched.h>
Naoya Horiguchid33b9f42009-12-14 17:59:59 -08005#include <linux/hugetlb.h>
Matt Mackalle6473092008-02-04 22:29:01 -08006
7static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -07008 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -08009{
10 pte_t *pte;
11 int err = 0;
12
13 pte = pte_offset_map(pmd, addr);
Johannes Weiner556637c2008-04-28 02:11:47 -070014 for (;;) {
Dave Hansen21650092008-06-12 15:21:47 -070015 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080016 if (err)
17 break;
Johannes Weiner556637c2008-04-28 02:11:47 -070018 addr += PAGE_SIZE;
19 if (addr == end)
20 break;
21 pte++;
22 }
Matt Mackalle6473092008-02-04 22:29:01 -080023
24 pte_unmap(pte);
25 return err;
26}
27
28static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -070029 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -080030{
31 pmd_t *pmd;
32 unsigned long next;
33 int err = 0;
34
35 pmd = pmd_offset(pud, addr);
36 do {
Dave Hansen03319322011-03-22 16:32:56 -070037again:
Matt Mackalle6473092008-02-04 22:29:01 -080038 next = pmd_addr_end(addr, end);
Naoya Horiguchi48684a62015-02-11 15:28:06 -080039 if (pmd_none(*pmd) || !walk->vma) {
Matt Mackalle6473092008-02-04 22:29:01 -080040 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -070041 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080042 if (err)
43 break;
44 continue;
45 }
Dave Hansen03319322011-03-22 16:32:56 -070046 /*
47 * This implies that each ->pmd_entry() handler
48 * needs to know about pmd_trans_huge() pmds
49 */
Matt Mackalle6473092008-02-04 22:29:01 -080050 if (walk->pmd_entry)
Dave Hansen21650092008-06-12 15:21:47 -070051 err = walk->pmd_entry(pmd, addr, next, walk);
Dave Hansen03319322011-03-22 16:32:56 -070052 if (err)
53 break;
54
55 /*
56 * Check this here so we only break down trans_huge
57 * pages when we _need_ to
58 */
59 if (!walk->pte_entry)
60 continue;
61
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -080062 split_huge_pmd(walk->vma, pmd, addr);
Naoya Horiguchifafaa422015-02-11 15:27:37 -080063 if (pmd_trans_unstable(pmd))
Dave Hansen03319322011-03-22 16:32:56 -070064 goto again;
65 err = walk_pte_range(pmd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080066 if (err)
67 break;
68 } while (pmd++, addr = next, addr != end);
69
70 return err;
71}
72
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +030073static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -070074 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -080075{
76 pud_t *pud;
77 unsigned long next;
78 int err = 0;
79
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +030080 pud = pud_offset(p4d, addr);
Matt Mackalle6473092008-02-04 22:29:01 -080081 do {
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -080082 again:
Matt Mackalle6473092008-02-04 22:29:01 -080083 next = pud_addr_end(addr, end);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -080084 if (pud_none(*pud) || !walk->vma) {
Matt Mackalle6473092008-02-04 22:29:01 -080085 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -070086 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080087 if (err)
88 break;
89 continue;
90 }
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -080091
92 if (walk->pud_entry) {
93 spinlock_t *ptl = pud_trans_huge_lock(pud, walk->vma);
94
95 if (ptl) {
96 err = walk->pud_entry(pud, addr, next, walk);
97 spin_unlock(ptl);
98 if (err)
99 break;
100 continue;
101 }
102 }
103
104 split_huge_pud(walk->vma, pud, addr);
105 if (pud_none(*pud))
106 goto again;
107
Naoya Horiguchi0b1fbfe2015-02-11 15:27:34 -0800108 if (walk->pmd_entry || walk->pte_entry)
Dave Hansen21650092008-06-12 15:21:47 -0700109 err = walk_pmd_range(pud, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800110 if (err)
111 break;
112 } while (pud++, addr = next, addr != end);
113
114 return err;
115}
116
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300117static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
118 struct mm_walk *walk)
119{
120 p4d_t *p4d;
121 unsigned long next;
122 int err = 0;
123
124 p4d = p4d_offset(pgd, addr);
125 do {
126 next = p4d_addr_end(addr, end);
127 if (p4d_none_or_clear_bad(p4d)) {
128 if (walk->pte_hole)
129 err = walk->pte_hole(addr, next, walk);
130 if (err)
131 break;
132 continue;
133 }
134 if (walk->pmd_entry || walk->pte_entry)
135 err = walk_pud_range(p4d, addr, next, walk);
136 if (err)
137 break;
138 } while (p4d++, addr = next, addr != end);
139
140 return err;
141}
142
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800143static int walk_pgd_range(unsigned long addr, unsigned long end,
144 struct mm_walk *walk)
145{
146 pgd_t *pgd;
147 unsigned long next;
148 int err = 0;
149
150 pgd = pgd_offset(walk->mm, addr);
151 do {
152 next = pgd_addr_end(addr, end);
153 if (pgd_none_or_clear_bad(pgd)) {
154 if (walk->pte_hole)
155 err = walk->pte_hole(addr, next, walk);
156 if (err)
157 break;
158 continue;
159 }
160 if (walk->pmd_entry || walk->pte_entry)
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300161 err = walk_p4d_range(pgd, addr, next, walk);
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800162 if (err)
163 break;
164 } while (pgd++, addr = next, addr != end);
165
166 return err;
167}
168
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700169#ifdef CONFIG_HUGETLB_PAGE
170static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr,
171 unsigned long end)
172{
173 unsigned long boundary = (addr & huge_page_mask(h)) + huge_page_size(h);
174 return boundary < end ? boundary : end;
175}
176
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800177static int walk_hugetlb_range(unsigned long addr, unsigned long end,
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700178 struct mm_walk *walk)
179{
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800180 struct vm_area_struct *vma = walk->vma;
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700181 struct hstate *h = hstate_vma(vma);
182 unsigned long next;
183 unsigned long hmask = huge_page_mask(h);
Punit Agrawal7868a202017-07-06 15:39:42 -0700184 unsigned long sz = huge_page_size(h);
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700185 pte_t *pte;
186 int err = 0;
187
188 do {
189 next = hugetlb_entry_end(h, addr, end);
Punit Agrawal7868a202017-07-06 15:39:42 -0700190 pte = huge_pte_offset(walk->mm, addr & hmask, sz);
Jann Horn373c4552017-11-14 01:03:44 +0100191
192 if (pte)
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700193 err = walk->hugetlb_entry(pte, hmask, addr, next, walk);
Jann Horn373c4552017-11-14 01:03:44 +0100194 else if (walk->pte_hole)
195 err = walk->pte_hole(addr, next, walk);
196
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700197 if (err)
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800198 break;
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700199 } while (addr = next, addr != end);
200
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800201 return err;
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700202}
KOSAKI Motohiro6c6d5282011-07-25 17:12:09 -0700203
KOSAKI Motohiro6c6d5282011-07-25 17:12:09 -0700204#else /* CONFIG_HUGETLB_PAGE */
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800205static int walk_hugetlb_range(unsigned long addr, unsigned long end,
KOSAKI Motohiro6c6d5282011-07-25 17:12:09 -0700206 struct mm_walk *walk)
207{
208 return 0;
209}
210
211#endif /* CONFIG_HUGETLB_PAGE */
212
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800213/*
214 * Decide whether we really walk over the current vma on [@start, @end)
215 * or skip it via the returned value. Return 0 if we do walk over the
216 * current vma, and return 1 if we skip the vma. Negative values means
217 * error, where we abort the current walk.
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800218 */
219static int walk_page_test(unsigned long start, unsigned long end,
220 struct mm_walk *walk)
221{
222 struct vm_area_struct *vma = walk->vma;
KOSAKI Motohiro6c6d5282011-07-25 17:12:09 -0700223
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800224 if (walk->test_walk)
225 return walk->test_walk(start, end, walk);
226
227 /*
Naoya Horiguchi48684a62015-02-11 15:28:06 -0800228 * vma(VM_PFNMAP) doesn't have any valid struct pages behind VM_PFNMAP
229 * range, so we don't walk over it as we do for normal vmas. However,
230 * Some callers are interested in handling hole range and they don't
231 * want to just ignore any single address range. Such users certainly
232 * define their ->pte_hole() callbacks, so let's delegate them to handle
233 * vma(VM_PFNMAP).
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800234 */
Naoya Horiguchi48684a62015-02-11 15:28:06 -0800235 if (vma->vm_flags & VM_PFNMAP) {
236 int err = 1;
237 if (walk->pte_hole)
238 err = walk->pte_hole(start, end, walk);
239 return err ? err : 1;
240 }
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800241 return 0;
242}
243
244static int __walk_page_range(unsigned long start, unsigned long end,
245 struct mm_walk *walk)
246{
247 int err = 0;
248 struct vm_area_struct *vma = walk->vma;
249
250 if (vma && is_vm_hugetlb_page(vma)) {
251 if (walk->hugetlb_entry)
252 err = walk_hugetlb_range(start, end, walk);
253 } else
254 err = walk_pgd_range(start, end, walk);
255
256 return err;
257}
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700258
Matt Mackalle6473092008-02-04 22:29:01 -0800259/**
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800260 * walk_page_range - walk page table with caller specific callbacks
Matt Mackalle6473092008-02-04 22:29:01 -0800261 *
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800262 * Recursively walk the page table tree of the process represented by @walk->mm
263 * within the virtual address range [@start, @end). During walking, we can do
264 * some caller-specific works for each entry, by setting up pmd_entry(),
265 * pte_entry(), and/or hugetlb_entry(). If you don't set up for some of these
266 * callbacks, the associated entries/pages are just ignored.
267 * The return values of these callbacks are commonly defined like below:
268 * - 0 : succeeded to handle the current entry, and if you don't reach the
269 * end address yet, continue to walk.
270 * - >0 : succeeded to handle the current entry, and return to the caller
271 * with caller specific value.
272 * - <0 : failed to handle the current entry, and return to the caller
273 * with error code.
Matt Mackalle6473092008-02-04 22:29:01 -0800274 *
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800275 * Before starting to walk page table, some callers want to check whether
276 * they really want to walk over the current vma, typically by checking
277 * its vm_flags. walk_page_test() and @walk->test_walk() are used for this
278 * purpose.
Matt Mackalle6473092008-02-04 22:29:01 -0800279 *
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800280 * struct mm_walk keeps current values of some common data like vma and pmd,
281 * which are useful for the access from callbacks. If you want to pass some
282 * caller-specific data to callbacks, @walk->private should be helpful.
Matt Mackalle6473092008-02-04 22:29:01 -0800283 *
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800284 * Locking:
285 * Callers of walk_page_range() and walk_page_vma() should hold
286 * @walk->mm->mmap_sem, because these function traverse vma list and/or
287 * access to vma's data.
Matt Mackalle6473092008-02-04 22:29:01 -0800288 */
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800289int walk_page_range(unsigned long start, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -0700290 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -0800291{
Matt Mackalle6473092008-02-04 22:29:01 -0800292 int err = 0;
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800293 unsigned long next;
294 struct vm_area_struct *vma;
Matt Mackalle6473092008-02-04 22:29:01 -0800295
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800296 if (start >= end)
297 return -EINVAL;
Matt Mackalle6473092008-02-04 22:29:01 -0800298
Dave Hansen21650092008-06-12 15:21:47 -0700299 if (!walk->mm)
300 return -EINVAL;
301
Sasha Levin96dad672014-10-09 15:28:39 -0700302 VM_BUG_ON_MM(!rwsem_is_locked(&walk->mm->mmap_sem), walk->mm);
Cliff Wickmana9ff7852013-05-24 15:55:36 -0700303
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800304 vma = find_vma(walk->mm, start);
Matt Mackalle6473092008-02-04 22:29:01 -0800305 do {
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800306 if (!vma) { /* after the last vma */
307 walk->vma = NULL;
308 next = end;
309 } else if (start < vma->vm_start) { /* outside vma */
310 walk->vma = NULL;
311 next = min(end, vma->vm_start);
312 } else { /* inside vma */
313 walk->vma = vma;
314 next = min(end, vma->vm_end);
315 vma = vma->vm_next;
David Sterba5f0af702010-11-24 12:57:10 -0800316
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800317 err = walk_page_test(start, next, walk);
Naoya Horiguchif6837392015-03-25 15:55:14 -0700318 if (err > 0) {
319 /*
320 * positive return values are purely for
321 * controlling the pagewalk, so should never
322 * be passed to the callers.
323 */
324 err = 0;
Cliff Wickmana9ff7852013-05-24 15:55:36 -0700325 continue;
Naoya Horiguchif6837392015-03-25 15:55:14 -0700326 }
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800327 if (err < 0)
Matt Mackalle6473092008-02-04 22:29:01 -0800328 break;
Matt Mackalle6473092008-02-04 22:29:01 -0800329 }
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800330 if (walk->vma || walk->pte_hole)
331 err = __walk_page_range(start, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800332 if (err)
333 break;
Naoya Horiguchifafaa422015-02-11 15:27:37 -0800334 } while (start = next, start < end);
Matt Mackalle6473092008-02-04 22:29:01 -0800335 return err;
336}
Naoya Horiguchi900fc5f2015-02-11 15:27:40 -0800337
338int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk)
339{
340 int err;
341
342 if (!walk->mm)
343 return -EINVAL;
344
345 VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_sem));
346 VM_BUG_ON(!vma);
347 walk->vma = vma;
348 err = walk_page_test(vma->vm_start, vma->vm_end, walk);
349 if (err > 0)
350 return 0;
351 if (err < 0)
352 return err;
353 return __walk_page_range(vma->vm_start, vma->vm_end, walk);
354}