blob: fa0cfbd551e185f83f75adecd7a0ecd6a03788cc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/i386/mm/pgtable.c
3 */
4
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/sched.h>
6#include <linux/kernel.h>
7#include <linux/errno.h>
8#include <linux/mm.h>
9#include <linux/swap.h>
10#include <linux/smp.h>
11#include <linux/highmem.h>
12#include <linux/slab.h>
13#include <linux/pagemap.h>
14#include <linux/spinlock.h>
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -070015#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
17#include <asm/system.h>
18#include <asm/pgtable.h>
19#include <asm/pgalloc.h>
20#include <asm/fixmap.h>
21#include <asm/e820.h>
22#include <asm/tlb.h>
23#include <asm/tlbflush.h>
24
25void show_mem(void)
26{
27 int total = 0, reserved = 0;
28 int shared = 0, cached = 0;
29 int highmem = 0;
30 struct page *page;
31 pg_data_t *pgdat;
32 unsigned long i;
Dave Hansen208d54e2005-10-29 18:16:52 -070033 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
Christophe Lucasf90e7182005-06-25 14:59:24 -070035 printk(KERN_INFO "Mem-info:\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 show_free_areas();
Christophe Lucasf90e7182005-06-25 14:59:24 -070037 printk(KERN_INFO "Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
KAMEZAWA Hiroyukiec936fc2006-03-27 01:15:59 -080038 for_each_online_pgdat(pgdat) {
Dave Hansen208d54e2005-10-29 18:16:52 -070039 pgdat_resize_lock(pgdat, &flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 for (i = 0; i < pgdat->node_spanned_pages; ++i) {
Dave Hansen408fde82005-06-23 00:07:37 -070041 page = pgdat_page_nr(pgdat, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 total++;
43 if (PageHighMem(page))
44 highmem++;
45 if (PageReserved(page))
46 reserved++;
47 else if (PageSwapCache(page))
48 cached++;
49 else if (page_count(page))
50 shared += page_count(page) - 1;
51 }
Dave Hansen208d54e2005-10-29 18:16:52 -070052 pgdat_resize_unlock(pgdat, &flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070053 }
Christophe Lucasf90e7182005-06-25 14:59:24 -070054 printk(KERN_INFO "%d pages of RAM\n", total);
55 printk(KERN_INFO "%d pages of HIGHMEM\n", highmem);
56 printk(KERN_INFO "%d reserved pages\n", reserved);
57 printk(KERN_INFO "%d pages shared\n", shared);
58 printk(KERN_INFO "%d pages swap cached\n", cached);
Martin J. Bligh6f4e1e52005-06-23 00:08:08 -070059
Christoph Lameterb1e7a8f2006-06-30 01:55:39 -070060 printk(KERN_INFO "%lu pages dirty\n", global_page_state(NR_FILE_DIRTY));
Christoph Lameterce866b32006-06-30 01:55:40 -070061 printk(KERN_INFO "%lu pages writeback\n",
62 global_page_state(NR_WRITEBACK));
Christoph Lameter65ba55f2006-06-30 01:55:34 -070063 printk(KERN_INFO "%lu pages mapped\n", global_page_state(NR_FILE_MAPPED));
Christoph Lameter972d1a72006-09-25 23:31:51 -070064 printk(KERN_INFO "%lu pages slab\n",
65 global_page_state(NR_SLAB_RECLAIMABLE) +
66 global_page_state(NR_SLAB_UNRECLAIMABLE));
Christoph Lameterdf849a12006-06-30 01:55:38 -070067 printk(KERN_INFO "%lu pages pagetables\n",
68 global_page_state(NR_PAGETABLE));
Linus Torvalds1da177e2005-04-16 15:20:36 -070069}
70
71/*
72 * Associate a virtual page frame with a given physical page frame
73 * and protection flags for that frame.
74 */
75static void set_pte_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
76{
77 pgd_t *pgd;
78 pud_t *pud;
79 pmd_t *pmd;
80 pte_t *pte;
81
82 pgd = swapper_pg_dir + pgd_index(vaddr);
83 if (pgd_none(*pgd)) {
84 BUG();
85 return;
86 }
87 pud = pud_offset(pgd, vaddr);
88 if (pud_none(*pud)) {
89 BUG();
90 return;
91 }
92 pmd = pmd_offset(pud, vaddr);
93 if (pmd_none(*pmd)) {
94 BUG();
95 return;
96 }
97 pte = pte_offset_kernel(pmd, vaddr);
Jan Beulichb0bfece2006-12-07 02:14:09 +010098 if (pgprot_val(flags))
99 /* <pfn,flags> stored as-is, to permit clearing entries */
100 set_pte(pte, pfn_pte(pfn, flags));
101 else
102 pte_clear(&init_mm, vaddr, pte);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
104 /*
105 * It's enough to flush this one mapping.
106 * (PGE mappings get flushed as well)
107 */
108 __flush_tlb_one(vaddr);
109}
110
111/*
112 * Associate a large virtual page frame with a given physical page frame
113 * and protection flags for that frame. pfn is for the base of the page,
114 * vaddr is what the page gets mapped to - both must be properly aligned.
115 * The pmd must already be instantiated. Assumes PAE mode.
116 */
117void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
118{
119 pgd_t *pgd;
120 pud_t *pud;
121 pmd_t *pmd;
122
123 if (vaddr & (PMD_SIZE-1)) { /* vaddr is misaligned */
Christophe Lucasf90e7182005-06-25 14:59:24 -0700124 printk(KERN_WARNING "set_pmd_pfn: vaddr misaligned\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125 return; /* BUG(); */
126 }
127 if (pfn & (PTRS_PER_PTE-1)) { /* pfn is misaligned */
Christophe Lucasf90e7182005-06-25 14:59:24 -0700128 printk(KERN_WARNING "set_pmd_pfn: pfn misaligned\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 return; /* BUG(); */
130 }
131 pgd = swapper_pg_dir + pgd_index(vaddr);
132 if (pgd_none(*pgd)) {
Christophe Lucasf90e7182005-06-25 14:59:24 -0700133 printk(KERN_WARNING "set_pmd_pfn: pgd_none\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 return; /* BUG(); */
135 }
136 pud = pud_offset(pgd, vaddr);
137 pmd = pmd_offset(pud, vaddr);
138 set_pmd(pmd, pfn_pmd(pfn, flags));
139 /*
140 * It's enough to flush this one mapping.
141 * (PGE mappings get flushed as well)
142 */
143 __flush_tlb_one(vaddr);
144}
145
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700146static int fixmaps;
147#ifndef CONFIG_COMPAT_VDSO
148unsigned long __FIXADDR_TOP = 0xfffff000;
149EXPORT_SYMBOL(__FIXADDR_TOP);
150#endif
151
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
153{
154 unsigned long address = __fix_to_virt(idx);
155
156 if (idx >= __end_of_fixed_addresses) {
157 BUG();
158 return;
159 }
160 set_pte_pfn(address, phys >> PAGE_SHIFT, flags);
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700161 fixmaps++;
162}
163
164/**
165 * reserve_top_address - reserves a hole in the top of kernel address space
166 * @reserve - size of hole to reserve
167 *
168 * Can be used to relocate the fixmap area and poke a hole in the top
169 * of kernel address space to make room for a hypervisor.
170 */
171void reserve_top_address(unsigned long reserve)
172{
173 BUG_ON(fixmaps > 0);
Zachary Amsden7ce0bcf2007-02-13 13:26:21 +0100174 printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
175 (int)-reserve);
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700176#ifdef CONFIG_COMPAT_VDSO
177 BUG_ON(reserve != 0);
178#else
179 __FIXADDR_TOP = -reserve - PAGE_SIZE;
180 __VMALLOC_RESERVE += reserve;
181#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182}
183
184pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
185{
186 return (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
187}
188
189struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
190{
191 struct page *pte;
192
193#ifdef CONFIG_HIGHPTE
194 pte = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM|__GFP_REPEAT|__GFP_ZERO, 0);
195#else
196 pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
197#endif
198 return pte;
199}
200
Christoph Lametere18b8902006-12-06 20:33:20 -0800201void pmd_ctor(void *pmd, struct kmem_cache *cache, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202{
203 memset(pmd, 0, PTRS_PER_PMD*sizeof(pmd_t));
204}
205
206/*
207 * List of all pgd's needed for non-PAE so it can invalidate entries
208 * in both cached and uncached pgd's; not needed for PAE since the
209 * kernel pmd is shared. If PAE were not to share the pmd a similar
210 * tactic would be needed. This is essentially codepath-based locking
211 * against pageattr.c; it is the unique case in which a valid change
212 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
213 * vmalloc faults work because attached pagetables are never freed.
214 * The locking scheme was chosen on the basis of manfred's
215 * recommendations and having no core impact whatsoever.
216 * -- wli
217 */
218DEFINE_SPINLOCK(pgd_lock);
219struct page *pgd_list;
220
221static inline void pgd_list_add(pgd_t *pgd)
222{
223 struct page *page = virt_to_page(pgd);
224 page->index = (unsigned long)pgd_list;
225 if (pgd_list)
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700226 set_page_private(pgd_list, (unsigned long)&page->index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 pgd_list = page;
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700228 set_page_private(page, (unsigned long)&pgd_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229}
230
231static inline void pgd_list_del(pgd_t *pgd)
232{
233 struct page *next, **pprev, *page = virt_to_page(pgd);
234 next = (struct page *)page->index;
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700235 pprev = (struct page **)page_private(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 *pprev = next;
237 if (next)
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700238 set_page_private(next, (unsigned long)pprev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239}
240
Christoph Lametere18b8902006-12-06 20:33:20 -0800241void pgd_ctor(void *pgd, struct kmem_cache *cache, unsigned long unused)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242{
243 unsigned long flags;
244
Zachary Amsdend7271b12005-09-03 15:56:50 -0700245 if (PTRS_PER_PMD == 1) {
246 memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 spin_lock_irqsave(&pgd_lock, flags);
Zachary Amsdend7271b12005-09-03 15:56:50 -0700248 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
Zachary Amsdend7271b12005-09-03 15:56:50 -0700250 clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 swapper_pg_dir + USER_PTRS_PER_PGD,
Zachary Amsdend7271b12005-09-03 15:56:50 -0700252 KERNEL_PGD_PTRS);
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100253
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 if (PTRS_PER_PMD > 1)
255 return;
256
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100257 /* must happen under lock */
258 paravirt_alloc_pd_clone(__pa(pgd) >> PAGE_SHIFT,
259 __pa(swapper_pg_dir) >> PAGE_SHIFT,
260 USER_PTRS_PER_PGD, PTRS_PER_PGD - USER_PTRS_PER_PGD);
261
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 pgd_list_add(pgd);
263 spin_unlock_irqrestore(&pgd_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264}
265
266/* never called when PTRS_PER_PMD > 1 */
Christoph Lametere18b8902006-12-06 20:33:20 -0800267void pgd_dtor(void *pgd, struct kmem_cache *cache, unsigned long unused)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268{
269 unsigned long flags; /* can be called from interrupt context */
270
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100271 paravirt_release_pd(__pa(pgd) >> PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 spin_lock_irqsave(&pgd_lock, flags);
273 pgd_list_del(pgd);
274 spin_unlock_irqrestore(&pgd_lock, flags);
275}
276
277pgd_t *pgd_alloc(struct mm_struct *mm)
278{
279 int i;
280 pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);
281
282 if (PTRS_PER_PMD == 1 || !pgd)
283 return pgd;
284
285 for (i = 0; i < USER_PTRS_PER_PGD; ++i) {
286 pmd_t *pmd = kmem_cache_alloc(pmd_cache, GFP_KERNEL);
287 if (!pmd)
288 goto out_oom;
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100289 paravirt_alloc_pd(__pa(pmd) >> PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 set_pgd(&pgd[i], __pgd(1 + __pa(pmd)));
291 }
292 return pgd;
293
294out_oom:
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100295 for (i--; i >= 0; i--) {
296 pgd_t pgdent = pgd[i];
297 void* pmd = (void *)__va(pgd_val(pgdent)-1);
298 paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
299 kmem_cache_free(pmd_cache, pmd);
300 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 kmem_cache_free(pgd_cache, pgd);
302 return NULL;
303}
304
305void pgd_free(pgd_t *pgd)
306{
307 int i;
308
309 /* in the PAE case user pgd entries are overwritten before usage */
310 if (PTRS_PER_PMD > 1)
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100311 for (i = 0; i < USER_PTRS_PER_PGD; ++i) {
312 pgd_t pgdent = pgd[i];
313 void* pmd = (void *)__va(pgd_val(pgdent)-1);
314 paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
315 kmem_cache_free(pmd_cache, pmd);
316 }
Hugh Dickinse0da3822005-04-19 13:29:15 -0700317 /* in the non-PAE case, free_pgtables() clears user pgd entries */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 kmem_cache_free(pgd_cache, pgd);
319}