blob: d5878bed78412542b82c03022756db62c9d73d63 [file] [log] [blame]
Matt Mackalle6473092008-02-04 22:29:01 -08001#include <linux/mm.h>
2#include <linux/highmem.h>
3#include <linux/sched.h>
4
5static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -07006 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -08007{
8 pte_t *pte;
9 int err = 0;
10
11 pte = pte_offset_map(pmd, addr);
Johannes Weiner556637c2008-04-28 02:11:47 -070012 for (;;) {
Dave Hansen21650092008-06-12 15:21:47 -070013 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080014 if (err)
15 break;
Johannes Weiner556637c2008-04-28 02:11:47 -070016 addr += PAGE_SIZE;
17 if (addr == end)
18 break;
19 pte++;
20 }
Matt Mackalle6473092008-02-04 22:29:01 -080021
22 pte_unmap(pte);
23 return err;
24}
25
26static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -070027 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -080028{
29 pmd_t *pmd;
30 unsigned long next;
31 int err = 0;
32
33 pmd = pmd_offset(pud, addr);
34 do {
35 next = pmd_addr_end(addr, end);
36 if (pmd_none_or_clear_bad(pmd)) {
37 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -070038 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080039 if (err)
40 break;
41 continue;
42 }
43 if (walk->pmd_entry)
Dave Hansen21650092008-06-12 15:21:47 -070044 err = walk->pmd_entry(pmd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080045 if (!err && walk->pte_entry)
Dave Hansen21650092008-06-12 15:21:47 -070046 err = walk_pte_range(pmd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080047 if (err)
48 break;
49 } while (pmd++, addr = next, addr != end);
50
51 return err;
52}
53
54static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -070055 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -080056{
57 pud_t *pud;
58 unsigned long next;
59 int err = 0;
60
61 pud = pud_offset(pgd, addr);
62 do {
63 next = pud_addr_end(addr, end);
64 if (pud_none_or_clear_bad(pud)) {
65 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -070066 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080067 if (err)
68 break;
69 continue;
70 }
71 if (walk->pud_entry)
Dave Hansen21650092008-06-12 15:21:47 -070072 err = walk->pud_entry(pud, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080073 if (!err && (walk->pmd_entry || walk->pte_entry))
Dave Hansen21650092008-06-12 15:21:47 -070074 err = walk_pmd_range(pud, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080075 if (err)
76 break;
77 } while (pud++, addr = next, addr != end);
78
79 return err;
80}
81
82/**
83 * walk_page_range - walk a memory map's page tables with a callback
Randy Dunlap76824862008-03-19 17:00:40 -070084 * @mm: memory map to walk
85 * @addr: starting address
86 * @end: ending address
87 * @walk: set of callbacks to invoke for each level of the tree
Matt Mackalle6473092008-02-04 22:29:01 -080088 *
89 * Recursively walk the page table for the memory area in a VMA,
90 * calling supplied callbacks. Callbacks are called in-order (first
91 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
92 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
93 *
Dave Hansen21650092008-06-12 15:21:47 -070094 * Each callback receives an entry pointer and the start and end of the
95 * associated range, and a copy of the original mm_walk for access to
96 * the ->private or ->mm fields.
Matt Mackalle6473092008-02-04 22:29:01 -080097 *
98 * No locks are taken, but the bottom level iterator will map PTE
99 * directories from highmem if necessary.
100 *
101 * If any callback returns a non-zero value, the walk is aborted and
102 * the return value is propagated back to the caller. Otherwise 0 is returned.
103 */
Dave Hansen21650092008-06-12 15:21:47 -0700104int walk_page_range(unsigned long addr, unsigned long end,
105 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -0800106{
107 pgd_t *pgd;
108 unsigned long next;
109 int err = 0;
110
111 if (addr >= end)
112 return err;
113
Dave Hansen21650092008-06-12 15:21:47 -0700114 if (!walk->mm)
115 return -EINVAL;
116
117 pgd = pgd_offset(walk->mm, addr);
Matt Mackalle6473092008-02-04 22:29:01 -0800118 do {
119 next = pgd_addr_end(addr, end);
120 if (pgd_none_or_clear_bad(pgd)) {
121 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -0700122 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800123 if (err)
124 break;
125 continue;
126 }
127 if (walk->pgd_entry)
Dave Hansen21650092008-06-12 15:21:47 -0700128 err = walk->pgd_entry(pgd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800129 if (!err &&
130 (walk->pud_entry || walk->pmd_entry || walk->pte_entry))
Dave Hansen21650092008-06-12 15:21:47 -0700131 err = walk_pud_range(pgd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800132 if (err)
133 break;
134 } while (pgd++, addr = next, addr != end);
135
136 return err;
137}