Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Copyright (C) 1998-2003 Hewlett-Packard Co |
| 7 | * David Mosberger-Tang <davidm@hpl.hp.com> |
| 8 | * Stephane Eranian <eranian@hpl.hp.com> |
| 9 | * Copyright (C) 2000, Rohit Seth <rohit.seth@intel.com> |
| 10 | * Copyright (C) 1999 VA Linux Systems |
| 11 | * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> |
| 12 | * Copyright (C) 2003 Silicon Graphics, Inc. All rights reserved. |
| 13 | * |
| 14 | * Routines used by ia64 machines with contiguous (or virtually contiguous) |
| 15 | * memory. |
| 16 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <linux/bootmem.h> |
| 18 | #include <linux/efi.h> |
| 19 | #include <linux/mm.h> |
Prarit Bhargava | 99a19cf1 | 2007-08-22 14:34:38 -0400 | [diff] [blame] | 20 | #include <linux/nmi.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #include <linux/swap.h> |
| 22 | |
| 23 | #include <asm/meminit.h> |
| 24 | #include <asm/pgalloc.h> |
| 25 | #include <asm/pgtable.h> |
| 26 | #include <asm/sections.h> |
| 27 | #include <asm/mca.h> |
| 28 | |
| 29 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
Bob Picco | e44e41d | 2006-06-28 12:55:43 -0400 | [diff] [blame] | 30 | static unsigned long max_gap; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | #endif |
| 32 | |
| 33 | /** |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 34 | * show_mem - give short summary of memory stats |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | * |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 36 | * Shows a simple page count of reserved and used pages in the system. |
| 37 | * For discontig machines, it does this on a per-pgdat basis. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | */ |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 39 | void show_mem(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 40 | { |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 41 | int i, total_reserved = 0; |
| 42 | int total_shared = 0, total_cached = 0; |
| 43 | unsigned long total_present = 0; |
| 44 | pg_data_t *pgdat; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | |
Jes Sorensen | 709a6c1 | 2006-09-13 08:43:42 -0400 | [diff] [blame] | 46 | printk(KERN_INFO "Mem-info:\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | show_free_areas(); |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 48 | printk(KERN_INFO "Node memory in pages:\n"); |
| 49 | for_each_online_pgdat(pgdat) { |
| 50 | unsigned long present; |
| 51 | unsigned long flags; |
| 52 | int shared = 0, cached = 0, reserved = 0; |
| 53 | |
| 54 | pgdat_resize_lock(pgdat, &flags); |
| 55 | present = pgdat->node_present_pages; |
| 56 | for(i = 0; i < pgdat->node_spanned_pages; i++) { |
| 57 | struct page *page; |
Prarit Bhargava | 99a19cf1 | 2007-08-22 14:34:38 -0400 | [diff] [blame] | 58 | if (unlikely(i % MAX_ORDER_NR_PAGES == 0)) |
| 59 | touch_nmi_watchdog(); |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 60 | if (pfn_valid(pgdat->node_start_pfn + i)) |
| 61 | page = pfn_to_page(pgdat->node_start_pfn + i); |
| 62 | else { |
Bob Picco | e44e41d | 2006-06-28 12:55:43 -0400 | [diff] [blame] | 63 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 64 | if (max_gap < LARGE_GAP) |
| 65 | continue; |
Bob Picco | e44e41d | 2006-06-28 12:55:43 -0400 | [diff] [blame] | 66 | #endif |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 67 | i = vmemmap_find_next_valid_pfn(pgdat->node_id, |
| 68 | i) - 1; |
| 69 | continue; |
| 70 | } |
| 71 | if (PageReserved(page)) |
| 72 | reserved++; |
| 73 | else if (PageSwapCache(page)) |
| 74 | cached++; |
| 75 | else if (page_count(page)) |
| 76 | shared += page_count(page)-1; |
Bob Picco | e44e41d | 2006-06-28 12:55:43 -0400 | [diff] [blame] | 77 | } |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 78 | pgdat_resize_unlock(pgdat, &flags); |
| 79 | total_present += present; |
| 80 | total_reserved += reserved; |
| 81 | total_cached += cached; |
| 82 | total_shared += shared; |
| 83 | printk(KERN_INFO "Node %4d: RAM: %11ld, rsvd: %8d, " |
| 84 | "shrd: %10d, swpd: %10d\n", pgdat->node_id, |
| 85 | present, reserved, shared, cached); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 86 | } |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 87 | printk(KERN_INFO "%ld pages of RAM\n", total_present); |
| 88 | printk(KERN_INFO "%d reserved pages\n", total_reserved); |
| 89 | printk(KERN_INFO "%d pages shared\n", total_shared); |
| 90 | printk(KERN_INFO "%d pages swap cached\n", total_cached); |
| 91 | printk(KERN_INFO "Total of %ld pages in page table cache\n", |
Christoph Lameter | 2bd62a4 | 2007-05-10 22:42:53 -0700 | [diff] [blame] | 92 | quicklist_total_size()); |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 93 | printk(KERN_INFO "%d free buffer pages\n", nr_free_buffer_pages()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 94 | } |
| 95 | |
George Beshers | f1c0afa | 2007-02-05 16:20:04 -0800 | [diff] [blame] | 96 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | /* physical address where the bootmem map is located */ |
| 98 | unsigned long bootmap_start; |
| 99 | |
| 100 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | * find_bootmap_location - callback to find a memory area for the bootmap |
| 102 | * @start: start of region |
| 103 | * @end: end of region |
| 104 | * @arg: unused callback data |
| 105 | * |
| 106 | * Find a place to put the bootmap and return its starting address in |
| 107 | * bootmap_start. This address must be page-aligned. |
| 108 | */ |
Chen, Kenneth W | dae2806 | 2006-03-22 16:54:15 -0800 | [diff] [blame] | 109 | static int __init |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | find_bootmap_location (unsigned long start, unsigned long end, void *arg) |
| 111 | { |
| 112 | unsigned long needed = *(unsigned long *)arg; |
| 113 | unsigned long range_start, range_end, free_start; |
| 114 | int i; |
| 115 | |
| 116 | #if IGNORE_PFN0 |
| 117 | if (start == PAGE_OFFSET) { |
| 118 | start += PAGE_SIZE; |
| 119 | if (start >= end) |
| 120 | return 0; |
| 121 | } |
| 122 | #endif |
| 123 | |
| 124 | free_start = PAGE_OFFSET; |
| 125 | |
| 126 | for (i = 0; i < num_rsvd_regions; i++) { |
| 127 | range_start = max(start, free_start); |
| 128 | range_end = min(end, rsvd_region[i].start & PAGE_MASK); |
| 129 | |
| 130 | free_start = PAGE_ALIGN(rsvd_region[i].end); |
| 131 | |
| 132 | if (range_end <= range_start) |
| 133 | continue; /* skip over empty range */ |
| 134 | |
| 135 | if (range_end - range_start >= needed) { |
| 136 | bootmap_start = __pa(range_start); |
| 137 | return -1; /* done */ |
| 138 | } |
| 139 | |
| 140 | /* nothing more available in this segment */ |
| 141 | if (range_end == end) |
| 142 | return 0; |
| 143 | } |
| 144 | return 0; |
| 145 | } |
| 146 | |
Tony Luck | 4b9ddc7 | 2007-11-06 15:14:45 -0800 | [diff] [blame] | 147 | #ifdef CONFIG_SMP |
| 148 | static void *cpu_data; |
| 149 | /** |
| 150 | * per_cpu_init - setup per-cpu variables |
| 151 | * |
| 152 | * Allocate and setup per-cpu data areas. |
| 153 | */ |
| 154 | void * __cpuinit |
| 155 | per_cpu_init (void) |
| 156 | { |
| 157 | int cpu; |
| 158 | static int first_time=1; |
| 159 | |
| 160 | /* |
| 161 | * get_free_pages() cannot be used before cpu_init() done. BSP |
| 162 | * allocates "NR_CPUS" pages for all CPUs to avoid that AP calls |
| 163 | * get_zeroed_page(). |
| 164 | */ |
| 165 | if (first_time) { |
Tony Luck | 10617bb | 2008-08-12 10:34:20 -0700 | [diff] [blame] | 166 | void *cpu0_data = __phys_per_cpu_start - PERCPU_PAGE_SIZE; |
| 167 | |
Tony Luck | 4b9ddc7 | 2007-11-06 15:14:45 -0800 | [diff] [blame] | 168 | first_time=0; |
Tony Luck | 10617bb | 2008-08-12 10:34:20 -0700 | [diff] [blame] | 169 | |
| 170 | __per_cpu_offset[0] = (char *) cpu0_data - __per_cpu_start; |
| 171 | per_cpu(local_per_cpu_offset, 0) = __per_cpu_offset[0]; |
| 172 | |
| 173 | for (cpu = 1; cpu < NR_CPUS; cpu++) { |
Tony Luck | 4b9ddc7 | 2007-11-06 15:14:45 -0800 | [diff] [blame] | 174 | memcpy(cpu_data, __phys_per_cpu_start, __per_cpu_end - __per_cpu_start); |
| 175 | __per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start; |
| 176 | cpu_data += PERCPU_PAGE_SIZE; |
| 177 | per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; |
| 178 | } |
| 179 | } |
| 180 | return __per_cpu_start + __per_cpu_offset[smp_processor_id()]; |
| 181 | } |
| 182 | |
| 183 | static inline void |
| 184 | alloc_per_cpu_data(void) |
| 185 | { |
Tony Luck | 10617bb | 2008-08-12 10:34:20 -0700 | [diff] [blame] | 186 | cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS-1, |
Tony Luck | 4b9ddc7 | 2007-11-06 15:14:45 -0800 | [diff] [blame] | 187 | PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); |
| 188 | } |
| 189 | #else |
| 190 | #define alloc_per_cpu_data() do { } while (0) |
| 191 | #endif /* CONFIG_SMP */ |
| 192 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | /** |
| 194 | * find_memory - setup memory map |
| 195 | * |
| 196 | * Walk the EFI memory map and find usable memory for the system, taking |
| 197 | * into account reserved areas. |
| 198 | */ |
Chen, Kenneth W | dae2806 | 2006-03-22 16:54:15 -0800 | [diff] [blame] | 199 | void __init |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | find_memory (void) |
| 201 | { |
| 202 | unsigned long bootmap_size; |
| 203 | |
| 204 | reserve_memory(); |
| 205 | |
| 206 | /* first find highest page frame number */ |
Zou Nan hai | a3f5c33 | 2007-03-20 13:41:57 -0700 | [diff] [blame] | 207 | min_low_pfn = ~0UL; |
| 208 | max_low_pfn = 0; |
| 209 | efi_memmap_walk(find_max_min_low_pfn, NULL); |
| 210 | max_pfn = max_low_pfn; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | /* how many bytes to cover all the pages */ |
| 212 | bootmap_size = bootmem_bootmap_pages(max_pfn) << PAGE_SHIFT; |
| 213 | |
| 214 | /* look for a location to hold the bootmap */ |
| 215 | bootmap_start = ~0UL; |
| 216 | efi_memmap_walk(find_bootmap_location, &bootmap_size); |
| 217 | if (bootmap_start == ~0UL) |
| 218 | panic("Cannot find %ld bytes for bootmap\n", bootmap_size); |
| 219 | |
Zou Nan hai | a3f5c33 | 2007-03-20 13:41:57 -0700 | [diff] [blame] | 220 | bootmap_size = init_bootmem_node(NODE_DATA(0), |
| 221 | (bootmap_start >> PAGE_SHIFT), 0, max_pfn); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | |
| 223 | /* Free all available memory, then mark bootmem-map as being in use. */ |
| 224 | efi_memmap_walk(filter_rsvd_memory, free_bootmem); |
Bernhard Walle | 72a7fe3 | 2008-02-07 00:15:17 -0800 | [diff] [blame] | 225 | reserve_bootmem(bootmap_start, bootmap_size, BOOTMEM_DEFAULT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 226 | |
| 227 | find_initrd(); |
Horms | 45a98fc | 2006-12-12 17:49:03 +0900 | [diff] [blame] | 228 | |
Tony Luck | 4b9ddc7 | 2007-11-06 15:14:45 -0800 | [diff] [blame] | 229 | alloc_per_cpu_data(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 230 | } |
| 231 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 232 | static int |
| 233 | count_pages (u64 start, u64 end, void *arg) |
| 234 | { |
| 235 | unsigned long *count = arg; |
| 236 | |
| 237 | *count += (end - start) >> PAGE_SHIFT; |
| 238 | return 0; |
| 239 | } |
| 240 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 241 | /* |
| 242 | * Set up the page tables. |
| 243 | */ |
| 244 | |
Chen, Kenneth W | dae2806 | 2006-03-22 16:54:15 -0800 | [diff] [blame] | 245 | void __init |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 246 | paging_init (void) |
| 247 | { |
| 248 | unsigned long max_dma; |
Mel Gorman | 05e0caa | 2006-09-27 01:49:54 -0700 | [diff] [blame] | 249 | unsigned long max_zone_pfns[MAX_NR_ZONES]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | |
| 251 | num_physpages = 0; |
| 252 | efi_memmap_walk(count_pages, &num_physpages); |
| 253 | |
Mel Gorman | 6391af1 | 2006-10-11 01:20:39 -0700 | [diff] [blame] | 254 | memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); |
Christoph Lameter | 09ae1f5 | 2007-02-10 01:43:11 -0800 | [diff] [blame] | 255 | #ifdef CONFIG_ZONE_DMA |
| 256 | max_dma = virt_to_phys((void *) MAX_DMA_ADDRESS) >> PAGE_SHIFT; |
Mel Gorman | 05e0caa | 2006-09-27 01:49:54 -0700 | [diff] [blame] | 257 | max_zone_pfns[ZONE_DMA] = max_dma; |
Christoph Lameter | 09ae1f5 | 2007-02-10 01:43:11 -0800 | [diff] [blame] | 258 | #endif |
Mel Gorman | 05e0caa | 2006-09-27 01:49:54 -0700 | [diff] [blame] | 259 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 260 | |
| 261 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
Zoltan Menyhart | 98075d2 | 2008-04-11 15:21:35 -0700 | [diff] [blame] | 262 | efi_memmap_walk(filter_memory, register_active_ranges); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 263 | efi_memmap_walk(find_largest_hole, (u64 *)&max_gap); |
| 264 | if (max_gap < LARGE_GAP) { |
| 265 | vmem_map = (struct page *) 0; |
Mel Gorman | 05e0caa | 2006-09-27 01:49:54 -0700 | [diff] [blame] | 266 | free_area_init_nodes(max_zone_pfns); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 267 | } else { |
| 268 | unsigned long map_size; |
| 269 | |
| 270 | /* allocate virtual_mem_map */ |
| 271 | |
Bob Picco | 921eea1 | 2006-06-28 12:54:55 -0400 | [diff] [blame] | 272 | map_size = PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * |
| 273 | sizeof(struct page)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 274 | vmalloc_end -= map_size; |
| 275 | vmem_map = (struct page *) vmalloc_end; |
| 276 | efi_memmap_walk(create_mem_map_page_table, NULL); |
| 277 | |
Mel Gorman | 05e0caa | 2006-09-27 01:49:54 -0700 | [diff] [blame] | 278 | /* |
| 279 | * alloc_node_mem_map makes an adjustment for mem_map |
| 280 | * which isn't compatible with vmem_map. |
| 281 | */ |
| 282 | NODE_DATA(0)->node_mem_map = vmem_map + |
| 283 | find_min_pfn_with_active_regions(); |
| 284 | free_area_init_nodes(max_zone_pfns); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 285 | |
| 286 | printk("Virtual mem_map starts at 0x%p\n", mem_map); |
| 287 | } |
| 288 | #else /* !CONFIG_VIRTUAL_MEM_MAP */ |
Mel Gorman | 05e0caa | 2006-09-27 01:49:54 -0700 | [diff] [blame] | 289 | add_active_range(0, 0, max_low_pfn); |
| 290 | free_area_init_nodes(max_zone_pfns); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 291 | #endif /* !CONFIG_VIRTUAL_MEM_MAP */ |
| 292 | zero_page_memmap_ptr = virt_to_page(ia64_imva(empty_zero_page)); |
| 293 | } |