blob: 7b47a57b66462dae3d9ae42929a1bba543d2c290 [file] [log] [blame]
Matt Mackalle6473092008-02-04 22:29:01 -08001#include <linux/mm.h>
2#include <linux/highmem.h>
3#include <linux/sched.h>
Naoya Horiguchid33b9f42009-12-14 17:59:59 -08004#include <linux/hugetlb.h>
Matt Mackalle6473092008-02-04 22:29:01 -08005
6static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -07007 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -08008{
9 pte_t *pte;
10 int err = 0;
11
12 pte = pte_offset_map(pmd, addr);
Johannes Weiner556637c2008-04-28 02:11:47 -070013 for (;;) {
Dave Hansen21650092008-06-12 15:21:47 -070014 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080015 if (err)
16 break;
Johannes Weiner556637c2008-04-28 02:11:47 -070017 addr += PAGE_SIZE;
18 if (addr == end)
19 break;
20 pte++;
21 }
Matt Mackalle6473092008-02-04 22:29:01 -080022
23 pte_unmap(pte);
24 return err;
25}
26
27static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -070028 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -080029{
30 pmd_t *pmd;
31 unsigned long next;
32 int err = 0;
33
34 pmd = pmd_offset(pud, addr);
35 do {
36 next = pmd_addr_end(addr, end);
37 if (pmd_none_or_clear_bad(pmd)) {
38 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -070039 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080040 if (err)
41 break;
42 continue;
43 }
44 if (walk->pmd_entry)
Dave Hansen21650092008-06-12 15:21:47 -070045 err = walk->pmd_entry(pmd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080046 if (!err && walk->pte_entry)
Dave Hansen21650092008-06-12 15:21:47 -070047 err = walk_pte_range(pmd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080048 if (err)
49 break;
50 } while (pmd++, addr = next, addr != end);
51
52 return err;
53}
54
55static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -070056 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -080057{
58 pud_t *pud;
59 unsigned long next;
60 int err = 0;
61
62 pud = pud_offset(pgd, addr);
63 do {
64 next = pud_addr_end(addr, end);
65 if (pud_none_or_clear_bad(pud)) {
66 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -070067 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080068 if (err)
69 break;
70 continue;
71 }
72 if (walk->pud_entry)
Dave Hansen21650092008-06-12 15:21:47 -070073 err = walk->pud_entry(pud, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080074 if (!err && (walk->pmd_entry || walk->pte_entry))
Dave Hansen21650092008-06-12 15:21:47 -070075 err = walk_pmd_range(pud, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -080076 if (err)
77 break;
78 } while (pud++, addr = next, addr != end);
79
80 return err;
81}
82
83/**
84 * walk_page_range - walk a memory map's page tables with a callback
Randy Dunlap76824862008-03-19 17:00:40 -070085 * @mm: memory map to walk
86 * @addr: starting address
87 * @end: ending address
88 * @walk: set of callbacks to invoke for each level of the tree
Matt Mackalle6473092008-02-04 22:29:01 -080089 *
90 * Recursively walk the page table for the memory area in a VMA,
91 * calling supplied callbacks. Callbacks are called in-order (first
92 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
93 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
94 *
Dave Hansen21650092008-06-12 15:21:47 -070095 * Each callback receives an entry pointer and the start and end of the
96 * associated range, and a copy of the original mm_walk for access to
97 * the ->private or ->mm fields.
Matt Mackalle6473092008-02-04 22:29:01 -080098 *
99 * No locks are taken, but the bottom level iterator will map PTE
100 * directories from highmem if necessary.
101 *
102 * If any callback returns a non-zero value, the walk is aborted and
103 * the return value is propagated back to the caller. Otherwise 0 is returned.
104 */
Dave Hansen21650092008-06-12 15:21:47 -0700105int walk_page_range(unsigned long addr, unsigned long end,
106 struct mm_walk *walk)
Matt Mackalle6473092008-02-04 22:29:01 -0800107{
108 pgd_t *pgd;
109 unsigned long next;
110 int err = 0;
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800111 struct vm_area_struct *vma;
Matt Mackalle6473092008-02-04 22:29:01 -0800112
113 if (addr >= end)
114 return err;
115
Dave Hansen21650092008-06-12 15:21:47 -0700116 if (!walk->mm)
117 return -EINVAL;
118
119 pgd = pgd_offset(walk->mm, addr);
Matt Mackalle6473092008-02-04 22:29:01 -0800120 do {
121 next = pgd_addr_end(addr, end);
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800122
Naoya Horiguchi5dc37642009-12-14 18:00:01 -0800123 /*
124 * handle hugetlb vma individually because pagetable walk for
125 * the hugetlb page is dependent on the architecture and
126 * we can't handled it in the same manner as non-huge pages.
127 */
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800128 vma = find_vma(walk->mm, addr);
Naoya Horiguchi5dc37642009-12-14 18:00:01 -0800129#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800130 if (vma && is_vm_hugetlb_page(vma)) {
Naoya Horiguchi5dc37642009-12-14 18:00:01 -0800131 pte_t *pte;
132 struct hstate *hs;
133
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800134 if (vma->vm_end < next)
135 next = vma->vm_end;
Naoya Horiguchi5dc37642009-12-14 18:00:01 -0800136 hs = hstate_vma(vma);
137 pte = huge_pte_offset(walk->mm,
138 addr & huge_page_mask(hs));
139 if (pte && !huge_pte_none(huge_ptep_get(pte))
140 && walk->hugetlb_entry)
141 err = walk->hugetlb_entry(pte, addr,
142 next, walk);
143 if (err)
144 break;
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800145 continue;
146 }
Naoya Horiguchi5dc37642009-12-14 18:00:01 -0800147#endif
Matt Mackalle6473092008-02-04 22:29:01 -0800148 if (pgd_none_or_clear_bad(pgd)) {
149 if (walk->pte_hole)
Dave Hansen21650092008-06-12 15:21:47 -0700150 err = walk->pte_hole(addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800151 if (err)
152 break;
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800153 pgd++;
Matt Mackalle6473092008-02-04 22:29:01 -0800154 continue;
155 }
156 if (walk->pgd_entry)
Dave Hansen21650092008-06-12 15:21:47 -0700157 err = walk->pgd_entry(pgd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800158 if (!err &&
159 (walk->pud_entry || walk->pmd_entry || walk->pte_entry))
Dave Hansen21650092008-06-12 15:21:47 -0700160 err = walk_pud_range(pgd, addr, next, walk);
Matt Mackalle6473092008-02-04 22:29:01 -0800161 if (err)
162 break;
Naoya Horiguchid33b9f42009-12-14 17:59:59 -0800163 pgd++;
164 } while (addr = next, addr != end);
Matt Mackalle6473092008-02-04 22:29:01 -0800165
166 return err;
167}