| /* |
| * |
| * Copyright (C) 1995 Linus Torvalds |
| * |
| * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 |
| */ |
| |
| #include <linux/module.h> |
| #include <linux/signal.h> |
| #include <linux/sched.h> |
| #include <linux/kernel.h> |
| #include <linux/errno.h> |
| #include <linux/string.h> |
| #include <linux/types.h> |
| #include <linux/ptrace.h> |
| #include <linux/mman.h> |
| #include <linux/mm.h> |
| #include <linux/hugetlb.h> |
| #include <linux/swap.h> |
| #include <linux/smp.h> |
| #include <linux/init.h> |
| #include <linux/highmem.h> |
| #include <linux/pagemap.h> |
| #include <linux/pfn.h> |
| #include <linux/poison.h> |
| #include <linux/bootmem.h> |
| #include <linux/slab.h> |
| #include <linux/proc_fs.h> |
| #include <linux/memory_hotplug.h> |
| #include <linux/initrd.h> |
| #include <linux/cpumask.h> |
| |
| #include <asm/asm.h> |
| #include <asm/bios_ebda.h> |
| #include <asm/processor.h> |
| #include <asm/system.h> |
| #include <asm/uaccess.h> |
| #include <asm/pgtable.h> |
| #include <asm/dma.h> |
| #include <asm/fixmap.h> |
| #include <asm/e820.h> |
| #include <asm/apic.h> |
| #include <asm/bugs.h> |
| #include <asm/tlb.h> |
| #include <asm/tlbflush.h> |
| #include <asm/pgalloc.h> |
| #include <asm/sections.h> |
| #include <asm/paravirt.h> |
| #include <asm/setup.h> |
| #include <asm/cacheflush.h> |
| #include <asm/smp.h> |
| |
| unsigned int __VMALLOC_RESERVE = 128 << 20; |
| |
| unsigned long max_low_pfn_mapped; |
| unsigned long max_pfn_mapped; |
| |
| DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); |
| unsigned long highstart_pfn, highend_pfn; |
| |
| static noinline int do_test_wp_bit(void); |
| |
| |
| static unsigned long __initdata table_start; |
| static unsigned long __meminitdata table_end; |
| static unsigned long __meminitdata table_top; |
| |
| static int __initdata after_init_bootmem; |
| |
| static __init void *alloc_low_page(unsigned long *phys) |
| { |
| unsigned long pfn = table_end++; |
| void *adr; |
| |
| if (pfn >= table_top) |
| panic("alloc_low_page: ran out of memory"); |
| |
| adr = __va(pfn * PAGE_SIZE); |
| memset(adr, 0, PAGE_SIZE); |
| *phys = pfn * PAGE_SIZE; |
| return adr; |
| } |
| |
| /* |
| * Creates a middle page table and puts a pointer to it in the |
| * given global directory entry. This only returns the gd entry |
| * in non-PAE compilation mode, since the middle layer is folded. |
| */ |
| static pmd_t * __init one_md_table_init(pgd_t *pgd) |
| { |
| pud_t *pud; |
| pmd_t *pmd_table; |
| |
| #ifdef CONFIG_X86_PAE |
| unsigned long phys; |
| if (!(pgd_val(*pgd) & _PAGE_PRESENT)) { |
| if (after_init_bootmem) |
| pmd_table = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE); |
| else |
| pmd_table = (pmd_t *)alloc_low_page(&phys); |
| paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT); |
| set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); |
| pud = pud_offset(pgd, 0); |
| BUG_ON(pmd_table != pmd_offset(pud, 0)); |
| } |
| #endif |
| pud = pud_offset(pgd, 0); |
| pmd_table = pmd_offset(pud, 0); |
| |
| return pmd_table; |
| } |
| |
| /* |
| * Create a page table and place a pointer to it in a middle page |
| * directory entry: |
| */ |
| static pte_t * __init one_page_table_init(pmd_t *pmd) |
| { |
| if (!(pmd_val(*pmd) & _PAGE_PRESENT)) { |
| pte_t *page_table = NULL; |
| |
| if (after_init_bootmem) { |
| #ifdef CONFIG_DEBUG_PAGEALLOC |
| page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE); |
| #endif |
| if (!page_table) |
| page_table = |
| (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); |
| } else { |
| unsigned long phys; |
| page_table = (pte_t *)alloc_low_page(&phys); |
| } |
| |
| paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT); |
| set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE)); |
| BUG_ON(page_table != pte_offset_kernel(pmd, 0)); |
| } |
| |
| return pte_offset_kernel(pmd, 0); |
| } |
| |
| /* |
| * This function initializes a certain range of kernel virtual memory |
| * with new bootmem page tables, everywhere page tables are missing in |
| * the given range. |
| * |
| * NOTE: The pagetables are allocated contiguous on the physical space |
| * so we can cache the place of the first one and move around without |
| * checking the pgd every time. |
| */ |
| static void __init |
| page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base) |
| { |
| int pgd_idx, pmd_idx; |
| unsigned long vaddr; |
| pgd_t *pgd; |
| pmd_t *pmd; |
| |
| vaddr = start; |
| pgd_idx = pgd_index(vaddr); |
| pmd_idx = pmd_index(vaddr); |
| pgd = pgd_base + pgd_idx; |
| |
| for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { |
| pmd = one_md_table_init(pgd); |
| pmd = pmd + pmd_index(vaddr); |
| for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end); |
| pmd++, pmd_idx++) { |
| one_page_table_init(pmd); |
| |
| vaddr += PMD_SIZE; |
| } |
| pmd_idx = 0; |
| } |
| } |
| |
| static inline int is_kernel_text(unsigned long addr) |
| { |
| if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end) |
| return 1; |
| return 0; |
| } |
| |
| /* |
| * This maps the physical memory to kernel virtual address space, a total |
| * of max_low_pfn pages, by creating page tables starting from address |
| * PAGE_OFFSET: |
| */ |
| static void __init kernel_physical_mapping_init(pgd_t *pgd_base, |
| unsigned long start_pfn, |
| unsigned long end_pfn, |
| int use_pse) |
| { |
| int pgd_idx, pmd_idx, pte_ofs; |
| unsigned long pfn; |
| pgd_t *pgd; |
| pmd_t *pmd; |
| pte_t *pte; |
| unsigned pages_2m, pages_4k; |
| int mapping_iter; |
| |
| /* |
| * First iteration will setup identity mapping using large/small pages |
| * based on use_pse, with other attributes same as set by |
| * the early code in head_32.S |
| * |
| * Second iteration will setup the appropriate attributes (NX, GLOBAL..) |
| * as desired for the kernel identity mapping. |
| * |
| * This two pass mechanism conforms to the TLB app note which says: |
| * |
| * "Software should not write to a paging-structure entry in a way |
| * that would change, for any linear address, both the page size |
| * and either the page frame or attributes." |
| */ |
| mapping_iter = 1; |
| |
| if (!cpu_has_pse) |
| use_pse = 0; |
| |
| repeat: |
| pages_2m = pages_4k = 0; |
| pfn = start_pfn; |
| pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET); |
| pgd = pgd_base + pgd_idx; |
| for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { |
| pmd = one_md_table_init(pgd); |
| |
| if (pfn >= end_pfn) |
| continue; |
| #ifdef CONFIG_X86_PAE |
| pmd_idx = pmd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET); |
| pmd += pmd_idx; |
| #else |
| pmd_idx = 0; |
| #endif |
| for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn; |
| pmd++, pmd_idx++) { |
| unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET; |
| |
| /* |
| * Map with big pages if possible, otherwise |
| * create normal page tables: |
| */ |
| if (use_pse) { |
| unsigned int addr2; |
| pgprot_t prot = PAGE_KERNEL_LARGE; |
| /* |
| * first pass will use the same initial |
| * identity mapping attribute + _PAGE_PSE. |
| */ |
| pgprot_t init_prot = |
| __pgprot(PTE_IDENT_ATTR | |
| _PAGE_PSE); |
| |
| addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE + |
| PAGE_OFFSET + PAGE_SIZE-1; |
| |
| if (is_kernel_text(addr) || |
| is_kernel_text(addr2)) |
| prot = PAGE_KERNEL_LARGE_EXEC; |
| |
| pages_2m++; |
| if (mapping_iter == 1) |
| set_pmd(pmd, pfn_pmd(pfn, init_prot)); |
| else |
| set_pmd(pmd, pfn_pmd(pfn, prot)); |
| |
| pfn += PTRS_PER_PTE; |
| continue; |
| } |
| pte = one_page_table_init(pmd); |
| |
| pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET); |
| pte += pte_ofs; |
| for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn; |
| pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) { |
| pgprot_t prot = PAGE_KERNEL; |
| /* |
| * first pass will use the same initial |
| * identity mapping attribute. |
| */ |
| pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR); |
| |
| if (is_kernel_text(addr)) |
| prot = PAGE_KERNEL_EXEC; |
| |
| pages_4k++; |
| if (mapping_iter == 1) |
| set_pte(pte, pfn_pte(pfn, init_prot)); |
| else |
| set_pte(pte, pfn_pte(pfn, prot)); |
| } |
| } |
| } |
| if (mapping_iter == 1) { |
| /* |
| * update direct mapping page count only in the first |
| * iteration. |
| */ |
| update_page_count(PG_LEVEL_2M, pages_2m); |
| update_page_count(PG_LEVEL_4K, pages_4k); |
| |
| /* |
| * local global flush tlb, which will flush the previous |
| * mappings present in both small and large page TLB's. |
| */ |
| __flush_tlb_all(); |
| |
| /* |
| * Second iteration will set the actual desired PTE attributes. |
| */ |
| mapping_iter = 2; |
| goto repeat; |
| } |
| } |
| |
| /* |
| * devmem_is_allowed() checks to see if /dev/mem access to a certain address |
| * is valid. The argument is a physical page number. |
| * |
| * |
| * On x86, access has to be given to the first megabyte of ram because that area |
| * contains bios code and data regions used by X and dosemu and similar apps. |
| * Access has to be given to non-kernel-ram areas as well, these contain the PCI |
| * mmio resources as well as potential bios/acpi data regions. |
| */ |
| int devmem_is_allowed(unsigned long pagenr) |
| { |
| if (pagenr <= 256) |
| return 1; |
| if (!page_is_ram(pagenr)) |
| return 1; |
| return 0; |
| } |
| |
| pte_t *kmap_pte; |
| pgprot_t kmap_prot; |
| |
| static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr) |
| { |
| return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), |
| vaddr), vaddr), vaddr); |
| } |
| |
| static void __init kmap_init(void) |
| { |
| unsigned long kmap_vstart; |
| |
| /* |
| * Cache the first kmap pte: |
| */ |
| kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); |
| kmap_pte = kmap_get_fixmap_pte(kmap_vstart); |
| |
| kmap_prot = PAGE_KERNEL; |
| } |
| |
| #ifdef CONFIG_HIGHMEM |
| static void __init permanent_kmaps_init(pgd_t *pgd_base) |
| { |
| unsigned long vaddr; |
| pgd_t *pgd; |
| pud_t *pud; |
| pmd_t *pmd; |
| pte_t *pte; |
| |
| vaddr = PKMAP_BASE; |
| page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base); |
| |
| pgd = swapper_pg_dir + pgd_index(vaddr); |
| pud = pud_offset(pgd, vaddr); |
| pmd = pmd_offset(pud, vaddr); |
| pte = pte_offset_kernel(pmd, vaddr); |
| pkmap_page_table = pte; |
| } |
| |
| static void __init add_one_highpage_init(struct page *page, int pfn) |
| { |
| ClearPageReserved(page); |
| init_page_count(page); |
| __free_page(page); |
| totalhigh_pages++; |
| } |
| |
| struct add_highpages_data { |
| unsigned long start_pfn; |
| unsigned long end_pfn; |
| }; |
| |
| static int __init add_highpages_work_fn(unsigned long start_pfn, |
| unsigned long end_pfn, void *datax) |
| { |
| int node_pfn; |
| struct page *page; |
| unsigned long final_start_pfn, final_end_pfn; |
| struct add_highpages_data *data; |
| |
| data = (struct add_highpages_data *)datax; |
| |
| final_start_pfn = max(start_pfn, data->start_pfn); |
| final_end_pfn = min(end_pfn, data->end_pfn); |
| if (final_start_pfn >= final_end_pfn) |
| return 0; |
| |
| for (node_pfn = final_start_pfn; node_pfn < final_end_pfn; |
| node_pfn++) { |
| if (!pfn_valid(node_pfn)) |
| continue; |
| page = pfn_to_page(node_pfn); |
| add_one_highpage_init(page, node_pfn); |
| } |
| |
| return 0; |
| |
| } |
| |
| void __init add_highpages_with_active_regions(int nid, unsigned long start_pfn, |
| unsigned long end_pfn) |
| { |
| struct add_highpages_data data; |
| |
| data.start_pfn = start_pfn; |
| data.end_pfn = end_pfn; |
| |
| work_with_active_regions(nid, add_highpages_work_fn, &data); |
| } |
| |
| #ifndef CONFIG_NUMA |
| static void __init set_highmem_pages_init(void) |
| { |
| add_highpages_with_active_regions(0, highstart_pfn, highend_pfn); |
| |
| totalram_pages += totalhigh_pages; |
| } |
| #endif /* !CONFIG_NUMA */ |
| |
| #else |
| # define permanent_kmaps_init(pgd_base) do { } while (0) |
| # define set_highmem_pages_init() do { } while (0) |
| #endif /* CONFIG_HIGHMEM */ |
| |
| void __init native_pagetable_setup_start(pgd_t *base) |
| { |
| unsigned long pfn, va; |
| pgd_t *pgd; |
| pud_t *pud; |
| pmd_t *pmd; |
| pte_t *pte; |
| |
| /* |
| * Remove any mappings which extend past the end of physical |
| * memory from the boot time page table: |
| */ |
| for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) { |
| va = PAGE_OFFSET + (pfn<<PAGE_SHIFT); |
| pgd = base + pgd_index(va); |
| if (!pgd_present(*pgd)) |
| break; |
| |
| pud = pud_offset(pgd, va); |
| pmd = pmd_offset(pud, va); |
| if (!pmd_present(*pmd)) |
| break; |
| |
| pte = pte_offset_kernel(pmd, va); |
| if (!pte_present(*pte)) |
| break; |
| |
| pte_clear(NULL, va, pte); |
| } |
| paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT); |
| } |
| |
| void __init native_pagetable_setup_done(pgd_t *base) |
| { |
| } |
| |
| /* |
| * Build a proper pagetable for the kernel mappings. Up until this |
| * point, we've been running on some set of pagetables constructed by |
| * the boot process. |
| * |
| * If we're booting on native hardware, this will be a pagetable |
| * constructed in arch/x86/kernel/head_32.S. The root of the |
| * pagetable will be swapper_pg_dir. |
| * |
| * If we're booting paravirtualized under a hypervisor, then there are |
| * more options: we may already be running PAE, and the pagetable may |
| * or may not be based in swapper_pg_dir. In any case, |
| * paravirt_pagetable_setup_start() will set up swapper_pg_dir |
| * appropriately for the rest of the initialization to work. |
| * |
| * In general, pagetable_init() assumes that the pagetable may already |
| * be partially populated, and so it avoids stomping on any existing |
| * mappings. |
| */ |
| static void __init early_ioremap_page_table_range_init(pgd_t *pgd_base) |
| { |
| unsigned long vaddr, end; |
| |
| /* |
| * Fixed mappings, only the page table structure has to be |
| * created - mappings will be set by set_fixmap(): |
| */ |
| early_ioremap_clear(); |
| vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; |
| end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK; |
| page_table_range_init(vaddr, end, pgd_base); |
| early_ioremap_reset(); |
| } |
| |
| static void __init pagetable_init(void) |
| { |
| pgd_t *pgd_base = swapper_pg_dir; |
| |
| permanent_kmaps_init(pgd_base); |
| } |
| |
| #ifdef CONFIG_ACPI_SLEEP |
| /* |
| * ACPI suspend needs this for resume, because things like the intel-agp |
| * driver might have split up a kernel 4MB mapping. |
| */ |
| char swsusp_pg_dir[PAGE_SIZE] |
| __attribute__ ((aligned(PAGE_SIZE))); |
| |
| static inline void save_pg_dir(void) |
| { |
| memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE); |
| } |
| #else /* !CONFIG_ACPI_SLEEP */ |
| static inline void save_pg_dir(void) |
| { |
| } |
| #endif /* !CONFIG_ACPI_SLEEP */ |
| |
| void zap_low_mappings(void) |
| { |
| int i; |
| |
| /* |
| * Zap initial low-memory mappings. |
| * |
| * Note that "pgd_clear()" doesn't do it for |
| * us, because pgd_clear() is a no-op on i386. |
| */ |
| for (i = 0; i < KERNEL_PGD_BOUNDARY; i++) { |
| #ifdef CONFIG_X86_PAE |
| set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page))); |
| #else |
| set_pgd(swapper_pg_dir+i, __pgd(0)); |
| #endif |
| } |
| flush_tlb_all(); |
| } |
| |
| int nx_enabled; |
| |
| pteval_t __supported_pte_mask __read_mostly = ~(_PAGE_NX | _PAGE_GLOBAL | _PAGE_IOMAP); |
| EXPORT_SYMBOL_GPL(__supported_pte_mask); |
| |
| #ifdef CONFIG_X86_PAE |
| |
| static int disable_nx __initdata; |
| |
| /* |
| * noexec = on|off |
| * |
| * Control non executable mappings. |
| * |
| * on Enable |
| * off Disable |
| */ |
| static int __init noexec_setup(char *str) |
| { |
| if (!str || !strcmp(str, "on")) { |
| if (cpu_has_nx) { |
| __supported_pte_mask |= _PAGE_NX; |
| disable_nx = 0; |
| } |
| } else { |
| if (!strcmp(str, "off")) { |
| disable_nx = 1; |
| __supported_pte_mask &= ~_PAGE_NX; |
| } else { |
| return -EINVAL; |
| } |
| } |
| |
| return 0; |
| } |
| early_param("noexec", noexec_setup); |
| |
| static void __init set_nx(void) |
| { |
| unsigned int v[4], l, h; |
| |
| if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) { |
| cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]); |
| |
| if ((v[3] & (1 << 20)) && !disable_nx) { |
| rdmsr(MSR_EFER, l, h); |
| l |= EFER_NX; |
| wrmsr(MSR_EFER, l, h); |
| nx_enabled = 1; |
| __supported_pte_mask |= _PAGE_NX; |
| } |
| } |
| } |
| #endif |
| |
| /* user-defined highmem size */ |
| static unsigned int highmem_pages = -1; |
| |
| /* |
| * highmem=size forces highmem to be exactly 'size' bytes. |
| * This works even on boxes that have no highmem otherwise. |
| * This also works to reduce highmem size on bigger boxes. |
| */ |
| static int __init parse_highmem(char *arg) |
| { |
| if (!arg) |
| return -EINVAL; |
| |
| highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT; |
| return 0; |
| } |
| early_param("highmem", parse_highmem); |
| |
| /* |
| * Determine low and high memory ranges: |
| */ |
| void __init find_low_pfn_range(void) |
| { |
| /* it could update max_pfn */ |
| |
| /* max_low_pfn is 0, we already have early_res support */ |
| |
| max_low_pfn = max_pfn; |
| if (max_low_pfn > MAXMEM_PFN) { |
| if (highmem_pages == -1) |
| highmem_pages = max_pfn - MAXMEM_PFN; |
| if (highmem_pages + MAXMEM_PFN < max_pfn) |
| max_pfn = MAXMEM_PFN + highmem_pages; |
| if (highmem_pages + MAXMEM_PFN > max_pfn) { |
| printk(KERN_WARNING "only %luMB highmem pages " |
| "available, ignoring highmem size of %uMB.\n", |
| pages_to_mb(max_pfn - MAXMEM_PFN), |
| pages_to_mb(highmem_pages)); |
| highmem_pages = 0; |
| } |
| max_low_pfn = MAXMEM_PFN; |
| #ifndef CONFIG_HIGHMEM |
| /* Maximum memory usable is what is directly addressable */ |
| printk(KERN_WARNING "Warning only %ldMB will be used.\n", |
| MAXMEM>>20); |
| if (max_pfn > MAX_NONPAE_PFN) |
| printk(KERN_WARNING |
| "Use a HIGHMEM64G enabled kernel.\n"); |
| else |
| printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n"); |
| max_pfn = MAXMEM_PFN; |
| #else /* !CONFIG_HIGHMEM */ |
| #ifndef CONFIG_HIGHMEM64G |
| if (max_pfn > MAX_NONPAE_PFN) { |
| max_pfn = MAX_NONPAE_PFN; |
| printk(KERN_WARNING "Warning only 4GB will be used." |
| "Use a HIGHMEM64G enabled kernel.\n"); |
| } |
| #endif /* !CONFIG_HIGHMEM64G */ |
| #endif /* !CONFIG_HIGHMEM */ |
| } else { |
| if (highmem_pages == -1) |
| highmem_pages = 0; |
| #ifdef CONFIG_HIGHMEM |
| if (highmem_pages >= max_pfn) { |
| printk(KERN_ERR "highmem size specified (%uMB) is " |
| "bigger than pages available (%luMB)!.\n", |
| pages_to_mb(highmem_pages), |
| pages_to_mb(max_pfn)); |
| highmem_pages = 0; |
| } |
| if (highmem_pages) { |
| if (max_low_pfn - highmem_pages < |
| 64*1024*1024/PAGE_SIZE){ |
| printk(KERN_ERR "highmem size %uMB results in " |
| "smaller than 64MB lowmem, ignoring it.\n" |
| , pages_to_mb(highmem_pages)); |
| highmem_pages = 0; |
| } |
| max_low_pfn -= highmem_pages; |
| } |
| #else |
| if (highmem_pages) |
| printk(KERN_ERR "ignoring highmem size on non-highmem" |
| " kernel!\n"); |
| #endif |
| } |
| } |
| |
| #ifndef CONFIG_NEED_MULTIPLE_NODES |
| void __init initmem_init(unsigned long start_pfn, |
| unsigned long end_pfn) |
| { |
| #ifdef CONFIG_HIGHMEM |
| highstart_pfn = highend_pfn = max_pfn; |
| if (max_pfn > max_low_pfn) |
| highstart_pfn = max_low_pfn; |
| memory_present(0, 0, highend_pfn); |
| e820_register_active_regions(0, 0, highend_pfn); |
| printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", |
| pages_to_mb(highend_pfn - highstart_pfn)); |
| num_physpages = highend_pfn; |
| high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1; |
| #else |
| memory_present(0, 0, max_low_pfn); |
| e820_register_active_regions(0, 0, max_low_pfn); |
| num_physpages = max_low_pfn; |
| high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1; |
| #endif |
| #ifdef CONFIG_FLATMEM |
| max_mapnr = num_physpages; |
| #endif |
| printk(KERN_NOTICE "%ldMB LOWMEM available.\n", |
| pages_to_mb(max_low_pfn)); |
| |
| setup_bootmem_allocator(); |
| } |
| #endif /* !CONFIG_NEED_MULTIPLE_NODES */ |
| |
| static void __init zone_sizes_init(void) |
| { |
| unsigned long max_zone_pfns[MAX_NR_ZONES]; |
| memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); |
| max_zone_pfns[ZONE_DMA] = |
| virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT; |
| max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
| #ifdef CONFIG_HIGHMEM |
| max_zone_pfns[ZONE_HIGHMEM] = highend_pfn; |
| #endif |
| |
| free_area_init_nodes(max_zone_pfns); |
| } |
| |
| void __init setup_bootmem_allocator(void) |
| { |
| int i; |
| unsigned long bootmap_size, bootmap; |
| /* |
| * Initialize the boot-time allocator (with low memory only): |
| */ |
| bootmap_size = bootmem_bootmap_pages(max_low_pfn)<<PAGE_SHIFT; |
| bootmap = find_e820_area(min_low_pfn<<PAGE_SHIFT, |
| max_pfn_mapped<<PAGE_SHIFT, bootmap_size, |
| PAGE_SIZE); |
| if (bootmap == -1L) |
| panic("Cannot find bootmem map of size %ld\n", bootmap_size); |
| reserve_early(bootmap, bootmap + bootmap_size, "BOOTMAP"); |
| |
| /* don't touch min_low_pfn */ |
| bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap >> PAGE_SHIFT, |
| min_low_pfn, max_low_pfn); |
| printk(KERN_INFO " mapped low ram: 0 - %08lx\n", |
| max_pfn_mapped<<PAGE_SHIFT); |
| printk(KERN_INFO " low ram: %08lx - %08lx\n", |
| min_low_pfn<<PAGE_SHIFT, max_low_pfn<<PAGE_SHIFT); |
| printk(KERN_INFO " bootmap %08lx - %08lx\n", |
| bootmap, bootmap + bootmap_size); |
| for_each_online_node(i) |
| free_bootmem_with_active_regions(i, max_low_pfn); |
| early_res_to_bootmem(0, max_low_pfn<<PAGE_SHIFT); |
| |
| after_init_bootmem = 1; |
| } |
| |
| static void __init find_early_table_space(unsigned long end, int use_pse) |
| { |
| unsigned long puds, pmds, ptes, tables, start; |
| |
| puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; |
| tables = PAGE_ALIGN(puds * sizeof(pud_t)); |
| |
| pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; |
| tables += PAGE_ALIGN(pmds * sizeof(pmd_t)); |
| |
| if (use_pse) { |
| unsigned long extra; |
| |
| extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT); |
| extra += PMD_SIZE; |
| ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; |
| } else |
| ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; |
| |
| tables += PAGE_ALIGN(ptes * sizeof(pte_t)); |
| |
| /* for fixmap */ |
| tables += PAGE_SIZE * 2; |
| |
| /* |
| * RED-PEN putting page tables only on node 0 could |
| * cause a hotspot and fill up ZONE_DMA. The page tables |
| * need roughly 0.5KB per GB. |
| */ |
| start = 0x7000; |
| table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT, |
| tables, PAGE_SIZE); |
| if (table_start == -1UL) |
| panic("Cannot find space for the kernel page tables"); |
| |
| table_start >>= PAGE_SHIFT; |
| table_end = table_start; |
| table_top = table_start + (tables>>PAGE_SHIFT); |
| |
| printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n", |
| end, table_start << PAGE_SHIFT, |
| (table_start << PAGE_SHIFT) + tables); |
| } |
| |
| unsigned long __init_refok init_memory_mapping(unsigned long start, |
| unsigned long end) |
| { |
| pgd_t *pgd_base = swapper_pg_dir; |
| unsigned long start_pfn, end_pfn; |
| unsigned long big_page_start; |
| #ifdef CONFIG_DEBUG_PAGEALLOC |
| /* |
| * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. |
| * This will simplify cpa(), which otherwise needs to support splitting |
| * large pages into small in interrupt context, etc. |
| */ |
| int use_pse = 0; |
| #else |
| int use_pse = cpu_has_pse; |
| #endif |
| |
| /* |
| * Find space for the kernel direct mapping tables. |
| */ |
| if (!after_init_bootmem) |
| find_early_table_space(end, use_pse); |
| |
| #ifdef CONFIG_X86_PAE |
| set_nx(); |
| if (nx_enabled) |
| printk(KERN_INFO "NX (Execute Disable) protection: active\n"); |
| #endif |
| |
| /* Enable PSE if available */ |
| if (cpu_has_pse) |
| set_in_cr4(X86_CR4_PSE); |
| |
| /* Enable PGE if available */ |
| if (cpu_has_pge) { |
| set_in_cr4(X86_CR4_PGE); |
| __supported_pte_mask |= _PAGE_GLOBAL; |
| } |
| |
| /* |
| * Don't use a large page for the first 2/4MB of memory |
| * because there are often fixed size MTRRs in there |
| * and overlapping MTRRs into large pages can cause |
| * slowdowns. |
| */ |
| big_page_start = PMD_SIZE; |
| |
| if (start < big_page_start) { |
| start_pfn = start >> PAGE_SHIFT; |
| end_pfn = min(big_page_start>>PAGE_SHIFT, end>>PAGE_SHIFT); |
| } else { |
| /* head is not big page alignment ? */ |
| start_pfn = start >> PAGE_SHIFT; |
| end_pfn = ((start + (PMD_SIZE - 1))>>PMD_SHIFT) |
| << (PMD_SHIFT - PAGE_SHIFT); |
| } |
| if (start_pfn < end_pfn) |
| kernel_physical_mapping_init(pgd_base, start_pfn, end_pfn, 0); |
| |
| /* big page range */ |
| start_pfn = ((start + (PMD_SIZE - 1))>>PMD_SHIFT) |
| << (PMD_SHIFT - PAGE_SHIFT); |
| if (start_pfn < (big_page_start >> PAGE_SHIFT)) |
| start_pfn = big_page_start >> PAGE_SHIFT; |
| end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); |
| if (start_pfn < end_pfn) |
| kernel_physical_mapping_init(pgd_base, start_pfn, end_pfn, |
| use_pse); |
| |
| /* tail is not big page alignment ? */ |
| start_pfn = end_pfn; |
| if (start_pfn > (big_page_start>>PAGE_SHIFT)) { |
| end_pfn = end >> PAGE_SHIFT; |
| if (start_pfn < end_pfn) |
| kernel_physical_mapping_init(pgd_base, start_pfn, |
| end_pfn, 0); |
| } |
| |
| early_ioremap_page_table_range_init(pgd_base); |
| |
| load_cr3(swapper_pg_dir); |
| |
| __flush_tlb_all(); |
| |
| if (!after_init_bootmem) |
| reserve_early(table_start << PAGE_SHIFT, |
| table_end << PAGE_SHIFT, "PGTABLE"); |
| |
| if (!after_init_bootmem) |
| early_memtest(start, end); |
| |
| return end >> PAGE_SHIFT; |
| } |
| |
| |
| /* |
| * paging_init() sets up the page tables - note that the first 8MB are |
| * already mapped by head.S. |
| * |
| * This routines also unmaps the page at virtual kernel address 0, so |
| * that we can trap those pesky NULL-reference errors in the kernel. |
| */ |
| void __init paging_init(void) |
| { |
| pagetable_init(); |
| |
| __flush_tlb_all(); |
| |
| kmap_init(); |
| |
| /* |
| * NOTE: at this point the bootmem allocator is fully available. |
| */ |
| sparse_init(); |
| zone_sizes_init(); |
| } |
| |
| /* |
| * Test if the WP bit works in supervisor mode. It isn't supported on 386's |
| * and also on some strange 486's. All 586+'s are OK. This used to involve |
| * black magic jumps to work around some nasty CPU bugs, but fortunately the |
| * switch to using exceptions got rid of all that. |
| */ |
| static void __init test_wp_bit(void) |
| { |
| printk(KERN_INFO |
| "Checking if this processor honours the WP bit even in supervisor mode..."); |
| |
| /* Any page-aligned address will do, the test is non-destructive */ |
| __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY); |
| boot_cpu_data.wp_works_ok = do_test_wp_bit(); |
| clear_fixmap(FIX_WP_TEST); |
| |
| if (!boot_cpu_data.wp_works_ok) { |
| printk(KERN_CONT "No.\n"); |
| #ifdef CONFIG_X86_WP_WORKS_OK |
| panic( |
| "This kernel doesn't support CPU's with broken WP. Recompile it for a 386!"); |
| #endif |
| } else { |
| printk(KERN_CONT "Ok.\n"); |
| } |
| } |
| |
| static struct kcore_list kcore_mem, kcore_vmalloc; |
| |
| void __init mem_init(void) |
| { |
| int codesize, reservedpages, datasize, initsize; |
| int tmp; |
| |
| #ifdef CONFIG_FLATMEM |
| BUG_ON(!mem_map); |
| #endif |
| /* this will put all low memory onto the freelists */ |
| totalram_pages += free_all_bootmem(); |
| |
| reservedpages = 0; |
| for (tmp = 0; tmp < max_low_pfn; tmp++) |
| /* |
| * Only count reserved RAM pages: |
| */ |
| if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp))) |
| reservedpages++; |
| |
| set_highmem_pages_init(); |
| |
| codesize = (unsigned long) &_etext - (unsigned long) &_text; |
| datasize = (unsigned long) &_edata - (unsigned long) &_etext; |
| initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; |
| |
| kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT); |
| kclist_add(&kcore_vmalloc, (void *)VMALLOC_START, |
| VMALLOC_END-VMALLOC_START); |
| |
| printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, " |
| "%dk reserved, %dk data, %dk init, %ldk highmem)\n", |
| (unsigned long) nr_free_pages() << (PAGE_SHIFT-10), |
| num_physpages << (PAGE_SHIFT-10), |
| codesize >> 10, |
| reservedpages << (PAGE_SHIFT-10), |
| datasize >> 10, |
| initsize >> 10, |
| (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)) |
| ); |
| |
| printk(KERN_INFO "virtual kernel memory layout:\n" |
| " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| #ifdef CONFIG_HIGHMEM |
| " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| #endif |
| " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" |
| " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" |
| " .init : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| " .data : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| " .text : 0x%08lx - 0x%08lx (%4ld kB)\n", |
| FIXADDR_START, FIXADDR_TOP, |
| (FIXADDR_TOP - FIXADDR_START) >> 10, |
| |
| #ifdef CONFIG_HIGHMEM |
| PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, |
| (LAST_PKMAP*PAGE_SIZE) >> 10, |
| #endif |
| |
| VMALLOC_START, VMALLOC_END, |
| (VMALLOC_END - VMALLOC_START) >> 20, |
| |
| (unsigned long)__va(0), (unsigned long)high_memory, |
| ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20, |
| |
| (unsigned long)&__init_begin, (unsigned long)&__init_end, |
| ((unsigned long)&__init_end - |
| (unsigned long)&__init_begin) >> 10, |
| |
| (unsigned long)&_etext, (unsigned long)&_edata, |
| ((unsigned long)&_edata - (unsigned long)&_etext) >> 10, |
| |
| (unsigned long)&_text, (unsigned long)&_etext, |
| ((unsigned long)&_etext - (unsigned long)&_text) >> 10); |
| |
| #ifdef CONFIG_HIGHMEM |
| BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE > FIXADDR_START); |
| BUG_ON(VMALLOC_END > PKMAP_BASE); |
| #endif |
| BUG_ON(VMALLOC_START > VMALLOC_END); |
| BUG_ON((unsigned long)high_memory > VMALLOC_START); |
| |
| if (boot_cpu_data.wp_works_ok < 0) |
| test_wp_bit(); |
| |
| save_pg_dir(); |
| zap_low_mappings(); |
| } |
| |
| #ifdef CONFIG_MEMORY_HOTPLUG |
| int arch_add_memory(int nid, u64 start, u64 size) |
| { |
| struct pglist_data *pgdata = NODE_DATA(nid); |
| struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM; |
| unsigned long start_pfn = start >> PAGE_SHIFT; |
| unsigned long nr_pages = size >> PAGE_SHIFT; |
| |
| return __add_pages(zone, start_pfn, nr_pages); |
| } |
| #endif |
| |
| /* |
| * This function cannot be __init, since exceptions don't work in that |
| * section. Put this after the callers, so that it cannot be inlined. |
| */ |
| static noinline int do_test_wp_bit(void) |
| { |
| char tmp_reg; |
| int flag; |
| |
| __asm__ __volatile__( |
| " movb %0, %1 \n" |
| "1: movb %1, %0 \n" |
| " xorl %2, %2 \n" |
| "2: \n" |
| _ASM_EXTABLE(1b,2b) |
| :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)), |
| "=q" (tmp_reg), |
| "=r" (flag) |
| :"2" (1) |
| :"memory"); |
| |
| return flag; |
| } |
| |
| #ifdef CONFIG_DEBUG_RODATA |
| const int rodata_test_data = 0xC3; |
| EXPORT_SYMBOL_GPL(rodata_test_data); |
| |
| void mark_rodata_ro(void) |
| { |
| unsigned long start = PFN_ALIGN(_text); |
| unsigned long size = PFN_ALIGN(_etext) - start; |
| |
| #ifndef CONFIG_DYNAMIC_FTRACE |
| /* Dynamic tracing modifies the kernel text section */ |
| set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
| printk(KERN_INFO "Write protecting the kernel text: %luk\n", |
| size >> 10); |
| |
| #ifdef CONFIG_CPA_DEBUG |
| printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n", |
| start, start+size); |
| set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT); |
| |
| printk(KERN_INFO "Testing CPA: write protecting again\n"); |
| set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); |
| #endif |
| #endif /* CONFIG_DYNAMIC_FTRACE */ |
| |
| start += size; |
| size = (unsigned long)__end_rodata - start; |
| set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
| printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", |
| size >> 10); |
| rodata_test(); |
| |
| #ifdef CONFIG_CPA_DEBUG |
| printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size); |
| set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT); |
| |
| printk(KERN_INFO "Testing CPA: write protecting again\n"); |
| set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
| #endif |
| } |
| #endif |
| |
| void free_init_pages(char *what, unsigned long begin, unsigned long end) |
| { |
| #ifdef CONFIG_DEBUG_PAGEALLOC |
| /* |
| * If debugging page accesses then do not free this memory but |
| * mark them not present - any buggy init-section access will |
| * create a kernel page fault: |
| */ |
| printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n", |
| begin, PAGE_ALIGN(end)); |
| set_memory_np(begin, (end - begin) >> PAGE_SHIFT); |
| #else |
| unsigned long addr; |
| |
| /* |
| * We just marked the kernel text read only above, now that |
| * we are going to free part of that, we need to make that |
| * writeable first. |
| */ |
| set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); |
| |
| for (addr = begin; addr < end; addr += PAGE_SIZE) { |
| ClearPageReserved(virt_to_page(addr)); |
| init_page_count(virt_to_page(addr)); |
| memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); |
| free_page(addr); |
| totalram_pages++; |
| } |
| printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); |
| #endif |
| } |
| |
| void free_initmem(void) |
| { |
| free_init_pages("unused kernel memory", |
| (unsigned long)(&__init_begin), |
| (unsigned long)(&__init_end)); |
| } |
| |
| #ifdef CONFIG_BLK_DEV_INITRD |
| void free_initrd_mem(unsigned long start, unsigned long end) |
| { |
| free_init_pages("initrd memory", start, end); |
| } |
| #endif |
| |
| int __init reserve_bootmem_generic(unsigned long phys, unsigned long len, |
| int flags) |
| { |
| return reserve_bootmem(phys, len, flags); |
| } |