blob: a1ad4cc423a7c79701a4461828af635e8cde9d3b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86_64/mm/init.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
7 */
8
9#include <linux/config.h>
10#include <linux/signal.h>
11#include <linux/sched.h>
12#include <linux/kernel.h>
13#include <linux/errno.h>
14#include <linux/string.h>
15#include <linux/types.h>
16#include <linux/ptrace.h>
17#include <linux/mman.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/smp.h>
21#include <linux/init.h>
22#include <linux/pagemap.h>
23#include <linux/bootmem.h>
24#include <linux/proc_fs.h>
25
26#include <asm/processor.h>
27#include <asm/system.h>
28#include <asm/uaccess.h>
29#include <asm/pgtable.h>
30#include <asm/pgalloc.h>
31#include <asm/dma.h>
32#include <asm/fixmap.h>
33#include <asm/e820.h>
34#include <asm/apic.h>
35#include <asm/tlb.h>
36#include <asm/mmu_context.h>
37#include <asm/proto.h>
38#include <asm/smp.h>
39
40#ifndef Dprintk
41#define Dprintk(x...)
42#endif
43
44#ifdef CONFIG_GART_IOMMU
45extern int swiotlb;
46#endif
47
48extern char _stext[];
49
50DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
51
52/*
53 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
54 * physical space so we can cache the place of the first one and move
55 * around without checking the pgd every time.
56 */
57
58void show_mem(void)
59{
Andi Kleene92343c2005-09-12 18:49:24 +020060 long i, total = 0, reserved = 0;
61 long shared = 0, cached = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 pg_data_t *pgdat;
63 struct page *page;
64
Andi Kleene92343c2005-09-12 18:49:24 +020065 printk(KERN_INFO "Mem-info:\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 show_free_areas();
Andi Kleene92343c2005-09-12 18:49:24 +020067 printk(KERN_INFO "Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
69 for_each_pgdat(pgdat) {
70 for (i = 0; i < pgdat->node_spanned_pages; ++i) {
71 page = pfn_to_page(pgdat->node_start_pfn + i);
72 total++;
Andi Kleene92343c2005-09-12 18:49:24 +020073 if (PageReserved(page))
74 reserved++;
75 else if (PageSwapCache(page))
76 cached++;
77 else if (page_count(page))
78 shared += page_count(page) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 }
80 }
Andi Kleene92343c2005-09-12 18:49:24 +020081 printk(KERN_INFO "%lu pages of RAM\n", total);
82 printk(KERN_INFO "%lu reserved pages\n",reserved);
83 printk(KERN_INFO "%lu pages shared\n",shared);
84 printk(KERN_INFO "%lu pages swap cached\n",cached);
Linus Torvalds1da177e2005-04-16 15:20:36 -070085}
86
87/* References to section boundaries */
88
89extern char _text, _etext, _edata, __bss_start, _end[];
90extern char __init_begin, __init_end;
91
92int after_bootmem;
93
94static void *spp_getpage(void)
95{
96 void *ptr;
97 if (after_bootmem)
98 ptr = (void *) get_zeroed_page(GFP_ATOMIC);
99 else
100 ptr = alloc_bootmem_pages(PAGE_SIZE);
101 if (!ptr || ((unsigned long)ptr & ~PAGE_MASK))
102 panic("set_pte_phys: cannot allocate page data %s\n", after_bootmem?"after bootmem":"");
103
104 Dprintk("spp_getpage %p\n", ptr);
105 return ptr;
106}
107
108static void set_pte_phys(unsigned long vaddr,
109 unsigned long phys, pgprot_t prot)
110{
111 pgd_t *pgd;
112 pud_t *pud;
113 pmd_t *pmd;
114 pte_t *pte, new_pte;
115
116 Dprintk("set_pte_phys %lx to %lx\n", vaddr, phys);
117
118 pgd = pgd_offset_k(vaddr);
119 if (pgd_none(*pgd)) {
120 printk("PGD FIXMAP MISSING, it should be setup in head.S!\n");
121 return;
122 }
123 pud = pud_offset(pgd, vaddr);
124 if (pud_none(*pud)) {
125 pmd = (pmd_t *) spp_getpage();
126 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
127 if (pmd != pmd_offset(pud, 0)) {
128 printk("PAGETABLE BUG #01! %p <-> %p\n", pmd, pmd_offset(pud,0));
129 return;
130 }
131 }
132 pmd = pmd_offset(pud, vaddr);
133 if (pmd_none(*pmd)) {
134 pte = (pte_t *) spp_getpage();
135 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE | _PAGE_USER));
136 if (pte != pte_offset_kernel(pmd, 0)) {
137 printk("PAGETABLE BUG #02!\n");
138 return;
139 }
140 }
141 new_pte = pfn_pte(phys >> PAGE_SHIFT, prot);
142
143 pte = pte_offset_kernel(pmd, vaddr);
144 if (!pte_none(*pte) &&
145 pte_val(*pte) != (pte_val(new_pte) & __supported_pte_mask))
146 pte_ERROR(*pte);
147 set_pte(pte, new_pte);
148
149 /*
150 * It's enough to flush this one mapping.
151 * (PGE mappings get flushed as well)
152 */
153 __flush_tlb_one(vaddr);
154}
155
156/* NOTE: this is meant to be run only at boot */
157void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
158{
159 unsigned long address = __fix_to_virt(idx);
160
161 if (idx >= __end_of_fixed_addresses) {
162 printk("Invalid __set_fixmap\n");
163 return;
164 }
165 set_pte_phys(address, phys, prot);
166}
167
168unsigned long __initdata table_start, table_end;
169
170extern pmd_t temp_boot_pmds[];
171
172static struct temp_map {
173 pmd_t *pmd;
174 void *address;
175 int allocated;
176} temp_mappings[] __initdata = {
177 { &temp_boot_pmds[0], (void *)(40UL * 1024 * 1024) },
178 { &temp_boot_pmds[1], (void *)(42UL * 1024 * 1024) },
179 {}
180};
181
182static __init void *alloc_low_page(int *index, unsigned long *phys)
183{
184 struct temp_map *ti;
185 int i;
186 unsigned long pfn = table_end++, paddr;
187 void *adr;
188
189 if (pfn >= end_pfn)
190 panic("alloc_low_page: ran out of memory");
191 for (i = 0; temp_mappings[i].allocated; i++) {
192 if (!temp_mappings[i].pmd)
193 panic("alloc_low_page: ran out of temp mappings");
194 }
195 ti = &temp_mappings[i];
196 paddr = (pfn << PAGE_SHIFT) & PMD_MASK;
197 set_pmd(ti->pmd, __pmd(paddr | _KERNPG_TABLE | _PAGE_PSE));
198 ti->allocated = 1;
199 __flush_tlb();
200 adr = ti->address + ((pfn << PAGE_SHIFT) & ~PMD_MASK);
201 *index = i;
202 *phys = pfn * PAGE_SIZE;
203 return adr;
204}
205
206static __init void unmap_low_page(int i)
207{
208 struct temp_map *ti = &temp_mappings[i];
209 set_pmd(ti->pmd, __pmd(0));
210 ti->allocated = 0;
211}
212
213static void __init phys_pud_init(pud_t *pud, unsigned long address, unsigned long end)
214{
215 long i, j;
216
217 i = pud_index(address);
218 pud = pud + i;
219 for (; i < PTRS_PER_PUD; pud++, i++) {
220 int map;
221 unsigned long paddr, pmd_phys;
222 pmd_t *pmd;
223
224 paddr = address + i*PUD_SIZE;
225 if (paddr >= end) {
226 for (; i < PTRS_PER_PUD; i++, pud++)
227 set_pud(pud, __pud(0));
228 break;
229 }
230
231 if (!e820_mapped(paddr, paddr+PUD_SIZE, 0)) {
232 set_pud(pud, __pud(0));
233 continue;
234 }
235
236 pmd = alloc_low_page(&map, &pmd_phys);
237 set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE));
238 for (j = 0; j < PTRS_PER_PMD; pmd++, j++, paddr += PMD_SIZE) {
239 unsigned long pe;
240
241 if (paddr >= end) {
242 for (; j < PTRS_PER_PMD; j++, pmd++)
243 set_pmd(pmd, __pmd(0));
244 break;
245 }
246 pe = _PAGE_NX|_PAGE_PSE | _KERNPG_TABLE | _PAGE_GLOBAL | paddr;
247 pe &= __supported_pte_mask;
248 set_pmd(pmd, __pmd(pe));
249 }
250 unmap_low_page(map);
251 }
252 __flush_tlb();
253}
254
255static void __init find_early_table_space(unsigned long end)
256{
257 unsigned long puds, pmds, tables;
258
259 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
260 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
261 tables = round_up(puds * sizeof(pud_t), PAGE_SIZE) +
262 round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
263
264 table_start = find_e820_area(0x8000, __pa_symbol(&_text), tables);
265 if (table_start == -1UL)
266 panic("Cannot find space for the kernel page tables");
267
268 table_start >>= PAGE_SHIFT;
269 table_end = table_start;
270}
271
272/* Setup the direct mapping of the physical memory at PAGE_OFFSET.
273 This runs before bootmem is initialized and gets pages directly from the
274 physical memory. To access them they are temporarily mapped. */
275void __init init_memory_mapping(unsigned long start, unsigned long end)
276{
277 unsigned long next;
278
279 Dprintk("init_memory_mapping\n");
280
281 /*
282 * Find space for the kernel direct mapping tables.
283 * Later we should allocate these tables in the local node of the memory
284 * mapped. Unfortunately this is done currently before the nodes are
285 * discovered.
286 */
287 find_early_table_space(end);
288
289 start = (unsigned long)__va(start);
290 end = (unsigned long)__va(end);
291
292 for (; start < end; start = next) {
293 int map;
294 unsigned long pud_phys;
295 pud_t *pud = alloc_low_page(&map, &pud_phys);
296 next = start + PGDIR_SIZE;
297 if (next > end)
298 next = end;
299 phys_pud_init(pud, __pa(start), __pa(next));
300 set_pgd(pgd_offset_k(start), mk_kernel_pgd(pud_phys));
301 unmap_low_page(map);
302 }
303
304 asm volatile("movq %%cr4,%0" : "=r" (mmu_cr4_features));
305 __flush_tlb_all();
306 early_printk("kernel direct mapping tables upto %lx @ %lx-%lx\n", end,
307 table_start<<PAGE_SHIFT,
308 table_end<<PAGE_SHIFT);
309}
310
311extern struct x8664_pda cpu_pda[NR_CPUS];
312
313/* Assumes all CPUs still execute in init_mm */
314void zap_low_mappings(void)
315{
316 pgd_t *pgd = pgd_offset_k(0UL);
317 pgd_clear(pgd);
318 flush_tlb_all();
319}
320
Andi Kleena2f1b422005-11-05 17:25:53 +0100321/* Compute zone sizes for the DMA and DMA32 zones in a node. */
322__init void
323size_zones(unsigned long *z, unsigned long *h,
324 unsigned long start_pfn, unsigned long end_pfn)
325{
326 int i;
327 unsigned long w;
328
329 for (i = 0; i < MAX_NR_ZONES; i++)
330 z[i] = 0;
331
332 if (start_pfn < MAX_DMA_PFN)
333 z[ZONE_DMA] = MAX_DMA_PFN - start_pfn;
334 if (start_pfn < MAX_DMA32_PFN) {
335 unsigned long dma32_pfn = MAX_DMA32_PFN;
336 if (dma32_pfn > end_pfn)
337 dma32_pfn = end_pfn;
338 z[ZONE_DMA32] = dma32_pfn - start_pfn;
339 }
340 z[ZONE_NORMAL] = end_pfn - start_pfn;
341
342 /* Remove lower zones from higher ones. */
343 w = 0;
344 for (i = 0; i < MAX_NR_ZONES; i++) {
345 if (z[i])
346 z[i] -= w;
347 w += z[i];
348 }
349
350 /* Compute holes */
351 w = 0;
352 for (i = 0; i < MAX_NR_ZONES; i++) {
353 unsigned long s = w;
354 w += z[i];
355 h[i] = e820_hole_size(s, w);
356 }
357}
358
Matt Tolentino2b976902005-06-23 00:08:06 -0700359#ifndef CONFIG_NUMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360void __init paging_init(void)
361{
Andi Kleena2f1b422005-11-05 17:25:53 +0100362 unsigned long zones[MAX_NR_ZONES], holes[MAX_NR_ZONES];
363 size_zones(zones, holes, 0, end_pfn);
364 free_area_init_node(0, NODE_DATA(0), zones,
365 __pa(PAGE_OFFSET) >> PAGE_SHIFT, holes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366}
367#endif
368
369/* Unmap a kernel mapping if it exists. This is useful to avoid prefetches
370 from the CPU leading to inconsistent cache lines. address and size
371 must be aligned to 2MB boundaries.
372 Does nothing when the mapping doesn't exist. */
373void __init clear_kernel_mapping(unsigned long address, unsigned long size)
374{
375 unsigned long end = address + size;
376
377 BUG_ON(address & ~LARGE_PAGE_MASK);
378 BUG_ON(size & ~LARGE_PAGE_MASK);
379
380 for (; address < end; address += LARGE_PAGE_SIZE) {
381 pgd_t *pgd = pgd_offset_k(address);
382 pud_t *pud;
383 pmd_t *pmd;
384 if (pgd_none(*pgd))
385 continue;
386 pud = pud_offset(pgd, address);
387 if (pud_none(*pud))
388 continue;
389 pmd = pmd_offset(pud, address);
390 if (!pmd || pmd_none(*pmd))
391 continue;
392 if (0 == (pmd_val(*pmd) & _PAGE_PSE)) {
393 /* Could handle this, but it should not happen currently. */
394 printk(KERN_ERR
395 "clear_kernel_mapping: mapping has been split. will leak memory\n");
396 pmd_ERROR(*pmd);
397 }
398 set_pmd(pmd, __pmd(0));
399 }
400 __flush_tlb_all();
401}
402
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
404 kcore_vsyscall;
405
406void __init mem_init(void)
407{
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200408 long codesize, reservedpages, datasize, initsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409
410#ifdef CONFIG_SWIOTLB
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 if (!iommu_aperture &&
412 (end_pfn >= 0xffffffff>>PAGE_SHIFT || force_iommu))
413 swiotlb = 1;
414 if (swiotlb)
415 swiotlb_init();
416#endif
417
418 /* How many end-of-memory variables you have, grandma! */
419 max_low_pfn = end_pfn;
420 max_pfn = end_pfn;
421 num_physpages = end_pfn;
422 high_memory = (void *) __va(end_pfn * PAGE_SIZE);
423
424 /* clear the zero-page */
425 memset(empty_zero_page, 0, PAGE_SIZE);
426
427 reservedpages = 0;
428
429 /* this will put all low memory onto the freelists */
Matt Tolentino2b976902005-06-23 00:08:06 -0700430#ifdef CONFIG_NUMA
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200431 totalram_pages = numa_free_all_bootmem();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432#else
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200433 totalram_pages = free_all_bootmem();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434#endif
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200435 reservedpages = end_pfn - totalram_pages - e820_hole_size(0, end_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436
437 after_bootmem = 1;
438
439 codesize = (unsigned long) &_etext - (unsigned long) &_text;
440 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
441 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
442
443 /* Register memory areas for /proc/kcore */
444 kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
445 kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
446 VMALLOC_END-VMALLOC_START);
447 kclist_add(&kcore_kernel, &_stext, _end - _stext);
448 kclist_add(&kcore_modules, (void *)MODULES_VADDR, MODULES_LEN);
449 kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START,
450 VSYSCALL_END - VSYSCALL_START);
451
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200452 printk("Memory: %luk/%luk available (%ldk kernel code, %ldk reserved, %ldk data, %ldk init)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
454 end_pfn << (PAGE_SHIFT-10),
455 codesize >> 10,
456 reservedpages << (PAGE_SHIFT-10),
457 datasize >> 10,
458 initsize >> 10);
459
460 /*
461 * Subtle. SMP is doing its boot stuff late (because it has to
462 * fork idle threads) - but it also needs low mappings for the
463 * protected-mode entry to work. We zap these entries only after
464 * the WP-bit has been tested.
465 */
466#ifndef CONFIG_SMP
467 zap_low_mappings();
468#endif
469}
470
471extern char __initdata_begin[], __initdata_end[];
472
473void free_initmem(void)
474{
475 unsigned long addr;
476
477 addr = (unsigned long)(&__init_begin);
478 for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
479 ClearPageReserved(virt_to_page(addr));
480 set_page_count(virt_to_page(addr), 1);
481 memset((void *)(addr & ~(PAGE_SIZE-1)), 0xcc, PAGE_SIZE);
482 free_page(addr);
483 totalram_pages++;
484 }
485 memset(__initdata_begin, 0xba, __initdata_end - __initdata_begin);
486 printk ("Freeing unused kernel memory: %luk freed\n", (&__init_end - &__init_begin) >> 10);
487}
488
489#ifdef CONFIG_BLK_DEV_INITRD
490void free_initrd_mem(unsigned long start, unsigned long end)
491{
492 if (start < (unsigned long)&_end)
493 return;
494 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
495 for (; start < end; start += PAGE_SIZE) {
496 ClearPageReserved(virt_to_page(start));
497 set_page_count(virt_to_page(start), 1);
498 free_page(start);
499 totalram_pages++;
500 }
501}
502#endif
503
504void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
505{
506 /* Should check here against the e820 map to avoid double free */
Matt Tolentino2b976902005-06-23 00:08:06 -0700507#ifdef CONFIG_NUMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 int nid = phys_to_nid(phys);
509 reserve_bootmem_node(NODE_DATA(nid), phys, len);
510#else
511 reserve_bootmem(phys, len);
512#endif
513}
514
515int kern_addr_valid(unsigned long addr)
516{
517 unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
518 pgd_t *pgd;
519 pud_t *pud;
520 pmd_t *pmd;
521 pte_t *pte;
522
523 if (above != 0 && above != -1UL)
524 return 0;
525
526 pgd = pgd_offset_k(addr);
527 if (pgd_none(*pgd))
528 return 0;
529
530 pud = pud_offset(pgd, addr);
531 if (pud_none(*pud))
532 return 0;
533
534 pmd = pmd_offset(pud, addr);
535 if (pmd_none(*pmd))
536 return 0;
537 if (pmd_large(*pmd))
538 return pfn_valid(pmd_pfn(*pmd));
539
540 pte = pte_offset_kernel(pmd, addr);
541 if (pte_none(*pte))
542 return 0;
543 return pfn_valid(pte_pfn(*pte));
544}
545
546#ifdef CONFIG_SYSCTL
547#include <linux/sysctl.h>
548
549extern int exception_trace, page_fault_trace;
550
551static ctl_table debug_table2[] = {
552 { 99, "exception-trace", &exception_trace, sizeof(int), 0644, NULL,
553 proc_dointvec },
554#ifdef CONFIG_CHECKING
555 { 100, "page-fault-trace", &page_fault_trace, sizeof(int), 0644, NULL,
556 proc_dointvec },
557#endif
558 { 0, }
559};
560
561static ctl_table debug_root_table2[] = {
562 { .ctl_name = CTL_DEBUG, .procname = "debug", .mode = 0555,
563 .child = debug_table2 },
564 { 0 },
565};
566
567static __init int x8664_sysctl_init(void)
568{
569 register_sysctl_table(debug_root_table2, 1);
570 return 0;
571}
572__initcall(x8664_sysctl_init);
573#endif
574
Andi Kleen1e014412005-04-16 15:24:55 -0700575/* A pseudo VMAs to allow ptrace access for the vsyscall page. This only
576 covers the 64bit vsyscall page now. 32bit has a real VMA now and does
577 not need special handling anymore. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578
579static struct vm_area_struct gate_vma = {
580 .vm_start = VSYSCALL_START,
581 .vm_end = VSYSCALL_END,
582 .vm_page_prot = PAGE_READONLY
583};
584
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
586{
587#ifdef CONFIG_IA32_EMULATION
Andi Kleen1e014412005-04-16 15:24:55 -0700588 if (test_tsk_thread_flag(tsk, TIF_IA32))
589 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590#endif
591 return &gate_vma;
592}
593
594int in_gate_area(struct task_struct *task, unsigned long addr)
595{
596 struct vm_area_struct *vma = get_gate_vma(task);
Andi Kleen1e014412005-04-16 15:24:55 -0700597 if (!vma)
598 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 return (addr >= vma->vm_start) && (addr < vma->vm_end);
600}
601
602/* Use this when you have no reliable task/vma, typically from interrupt
603 * context. It is less reliable than using the task's vma and may give
604 * false positives.
605 */
606int in_gate_area_no_task(unsigned long addr)
607{
Andi Kleen1e014412005-04-16 15:24:55 -0700608 return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609}