Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * S390 version |
Heiko Carstens | a53c8fa | 2012-07-20 11:15:04 +0200 | [diff] [blame] | 3 | * Copyright IBM Corp. 1999 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * Author(s): Hartmut Penner (hp@de.ibm.com) |
| 5 | * |
| 6 | * Derived from "arch/i386/mm/init.c" |
| 7 | * Copyright (C) 1995 Linus Torvalds |
| 8 | */ |
| 9 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #include <linux/signal.h> |
| 11 | #include <linux/sched.h> |
| 12 | #include <linux/kernel.h> |
| 13 | #include <linux/errno.h> |
| 14 | #include <linux/string.h> |
| 15 | #include <linux/types.h> |
| 16 | #include <linux/ptrace.h> |
| 17 | #include <linux/mman.h> |
| 18 | #include <linux/mm.h> |
| 19 | #include <linux/swap.h> |
| 20 | #include <linux/smp.h> |
| 21 | #include <linux/init.h> |
| 22 | #include <linux/pagemap.h> |
| 23 | #include <linux/bootmem.h> |
Heiko Carstens | e5d709b | 2013-05-02 09:15:58 +0200 | [diff] [blame] | 24 | #include <linux/memory.h> |
Heiko Carstens | d882b17 | 2006-07-01 04:36:31 -0700 | [diff] [blame] | 25 | #include <linux/pfn.h> |
Heiko Carstens | 028d9b3 | 2006-12-08 15:56:13 +0100 | [diff] [blame] | 26 | #include <linux/poison.h> |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 27 | #include <linux/initrd.h> |
Heiko Carstens | 3a4c5d5 | 2011-07-30 09:25:15 +0200 | [diff] [blame] | 28 | #include <linux/export.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 29 | #include <linux/gfp.h> |
Gerald Schaefer | 199071f | 2015-05-08 17:40:43 +0200 | [diff] [blame] | 30 | #include <linux/memblock.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | #include <asm/processor.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | #include <asm/uaccess.h> |
| 33 | #include <asm/pgtable.h> |
| 34 | #include <asm/pgalloc.h> |
| 35 | #include <asm/dma.h> |
| 36 | #include <asm/lowcore.h> |
| 37 | #include <asm/tlb.h> |
| 38 | #include <asm/tlbflush.h> |
Heiko Carstens | d882b17 | 2006-07-01 04:36:31 -0700 | [diff] [blame] | 39 | #include <asm/sections.h> |
David Howells | a0616cd | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 40 | #include <asm/ctl_reg.h> |
Heiko Carstens | e5d709b | 2013-05-02 09:15:58 +0200 | [diff] [blame] | 41 | #include <asm/sclp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 43 | pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__((__aligned__(PAGE_SIZE))); |
Heiko Carstens | 1485c5c | 2009-03-26 15:24:04 +0100 | [diff] [blame] | 44 | |
Martin Schwidefsky | 238ec4e | 2010-10-25 16:10:07 +0200 | [diff] [blame] | 45 | unsigned long empty_zero_page, zero_page_mask; |
Heiko Carstens | 1485c5c | 2009-03-26 15:24:04 +0100 | [diff] [blame] | 46 | EXPORT_SYMBOL(empty_zero_page); |
Ard Biesheuvel | 0b70068 | 2014-09-12 22:17:23 +0200 | [diff] [blame] | 47 | EXPORT_SYMBOL(zero_page_mask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | |
Jiang Liu | 0999f11 | 2013-04-29 15:06:48 -0700 | [diff] [blame] | 49 | static void __init setup_zero_pages(void) |
Martin Schwidefsky | 238ec4e | 2010-10-25 16:10:07 +0200 | [diff] [blame] | 50 | { |
Martin Schwidefsky | 238ec4e | 2010-10-25 16:10:07 +0200 | [diff] [blame] | 51 | unsigned int order; |
Martin Schwidefsky | 238ec4e | 2010-10-25 16:10:07 +0200 | [diff] [blame] | 52 | struct page *page; |
| 53 | int i; |
| 54 | |
Martin Schwidefsky | c7e8b2c | 2015-11-10 12:30:28 +0100 | [diff] [blame] | 55 | /* Latest machines require a mapping granularity of 512KB */ |
| 56 | order = 7; |
| 57 | |
Martin Schwidefsky | 7919e91 | 2013-02-28 11:08:54 +0100 | [diff] [blame] | 58 | /* Limit number of empty zero pages for small memory sizes */ |
Martin Schwidefsky | 1f6b83e | 2015-01-14 17:51:17 +0100 | [diff] [blame] | 59 | while (order > 2 && (totalram_pages >> 10) < (1UL << order)) |
| 60 | order--; |
Martin Schwidefsky | 238ec4e | 2010-10-25 16:10:07 +0200 | [diff] [blame] | 61 | |
| 62 | empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); |
| 63 | if (!empty_zero_page) |
| 64 | panic("Out of memory in setup_zero_pages"); |
| 65 | |
| 66 | page = virt_to_page((void *) empty_zero_page); |
| 67 | split_page(page, order); |
| 68 | for (i = 1 << order; i > 0; i--) { |
Jiang Liu | 0999f11 | 2013-04-29 15:06:48 -0700 | [diff] [blame] | 69 | mark_page_reserved(page); |
Martin Schwidefsky | 238ec4e | 2010-10-25 16:10:07 +0200 | [diff] [blame] | 70 | page++; |
| 71 | } |
| 72 | |
Jiang Liu | 0999f11 | 2013-04-29 15:06:48 -0700 | [diff] [blame] | 73 | zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; |
Martin Schwidefsky | 238ec4e | 2010-10-25 16:10:07 +0200 | [diff] [blame] | 74 | } |
| 75 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 76 | /* |
| 77 | * paging_init() sets up the page tables |
| 78 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 79 | void __init paging_init(void) |
| 80 | { |
Heiko Carstens | 39b742f | 2006-12-08 15:56:10 +0100 | [diff] [blame] | 81 | unsigned long max_zone_pfns[MAX_NR_ZONES]; |
Martin Schwidefsky | 14045eb | 2011-12-27 11:27:07 +0100 | [diff] [blame] | 82 | unsigned long pgd_type, asce_bits; |
Heiko Carstens | d882b17 | 2006-07-01 04:36:31 -0700 | [diff] [blame] | 83 | |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 84 | init_mm.pgd = swapper_pg_dir; |
Martin Schwidefsky | 14045eb | 2011-12-27 11:27:07 +0100 | [diff] [blame] | 85 | if (VMALLOC_END > (1UL << 42)) { |
| 86 | asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; |
| 87 | pgd_type = _REGION2_ENTRY_EMPTY; |
| 88 | } else { |
| 89 | asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; |
| 90 | pgd_type = _REGION3_ENTRY_EMPTY; |
| 91 | } |
Martin Schwidefsky | 14045eb | 2011-12-27 11:27:07 +0100 | [diff] [blame] | 92 | S390_lowcore.kernel_asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 93 | clear_table((unsigned long *) init_mm.pgd, pgd_type, |
| 94 | sizeof(unsigned long)*2048); |
Heiko Carstens | f4eb07c | 2006-12-08 15:56:07 +0100 | [diff] [blame] | 95 | vmem_map_init(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | /* enable virtual mapping in kernel mode */ |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 98 | __ctl_load(S390_lowcore.kernel_asce, 1, 1); |
| 99 | __ctl_load(S390_lowcore.kernel_asce, 7, 7); |
| 100 | __ctl_load(S390_lowcore.kernel_asce, 13, 13); |
Christian Borntraeger | 204ee2c | 2016-01-11 09:17:18 +0100 | [diff] [blame] | 101 | __arch_local_irq_stosm(0x04); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | |
Heiko Carstens | 17f3458 | 2008-04-30 13:38:47 +0200 | [diff] [blame] | 103 | sparse_memory_present_with_active_regions(MAX_NUMNODES); |
| 104 | sparse_init(); |
Heiko Carstens | 39b742f | 2006-12-08 15:56:10 +0100 | [diff] [blame] | 105 | memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); |
| 106 | max_zone_pfns[ZONE_DMA] = PFN_DOWN(MAX_DMA_ADDRESS); |
| 107 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
| 108 | free_area_init_nodes(max_zone_pfns); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | |
| 111 | void __init mem_init(void) |
| 112 | { |
Martin Schwidefsky | 1b948d6 | 2014-04-03 13:55:01 +0200 | [diff] [blame] | 113 | if (MACHINE_HAS_TLB_LC) |
| 114 | cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask); |
| 115 | cpumask_set_cpu(0, mm_cpumask(&init_mm)); |
| 116 | atomic_set(&init_mm.context.attach_count, 1); |
| 117 | |
Philipp Hachtmann | 3a368f7 | 2014-03-06 18:25:13 +0100 | [diff] [blame] | 118 | set_max_mapnr(max_low_pfn); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 119 | high_memory = (void *) __va(max_low_pfn * PAGE_SIZE); |
| 120 | |
Martin Schwidefsky | 45e576b | 2008-05-07 09:22:59 +0200 | [diff] [blame] | 121 | /* Setup guest page hinting */ |
| 122 | cmma_init(); |
| 123 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 124 | /* this will put all low memory onto the freelists */ |
Jiang Liu | 0c98853 | 2013-07-03 15:03:24 -0700 | [diff] [blame] | 125 | free_all_bootmem(); |
Jiang Liu | 0999f11 | 2013-04-29 15:06:48 -0700 | [diff] [blame] | 126 | setup_zero_pages(); /* Setup zeroed pages. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 127 | |
Jiang Liu | a18d0e2 | 2013-07-03 15:04:10 -0700 | [diff] [blame] | 128 | mem_init_print_info(NULL); |
Heiko Carstens | d882b17 | 2006-07-01 04:36:31 -0700 | [diff] [blame] | 129 | printk("Write protected kernel read-only data: %#lx - %#lx\n", |
Heiko Carstens | 162e006 | 2007-02-05 21:18:41 +0100 | [diff] [blame] | 130 | (unsigned long)&_stext, |
| 131 | PFN_ALIGN((unsigned long)&_eshared) - 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | } |
| 133 | |
Heiko Carstens | d96221a | 2010-02-26 22:37:42 +0100 | [diff] [blame] | 134 | void free_initmem(void) |
| 135 | { |
Jiang Liu | dbe67df | 2013-07-03 15:02:51 -0700 | [diff] [blame] | 136 | free_initmem_default(POISON_FREE_INITMEM); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 137 | } |
| 138 | |
| 139 | #ifdef CONFIG_BLK_DEV_INITRD |
Heiko Carstens | 5e249d6 | 2012-09-24 08:17:58 +0200 | [diff] [blame] | 140 | void __init free_initrd_mem(unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | { |
Jiang Liu | 1119969 | 2013-07-03 15:02:48 -0700 | [diff] [blame] | 142 | free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, |
| 143 | "initrd"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 144 | } |
| 145 | #endif |
Heiko Carstens | 421c175 | 2008-07-14 09:59:18 +0200 | [diff] [blame] | 146 | |
| 147 | #ifdef CONFIG_MEMORY_HOTPLUG |
Dan Williams | 033fbae | 2015-08-09 15:29:06 -0400 | [diff] [blame] | 148 | int arch_add_memory(int nid, u64 start, u64 size, bool for_device) |
Heiko Carstens | 421c175 | 2008-07-14 09:59:18 +0200 | [diff] [blame] | 149 | { |
Gerald Schaefer | 199071f | 2015-05-08 17:40:43 +0200 | [diff] [blame] | 150 | unsigned long normal_end_pfn = PFN_DOWN(memblock_end_of_DRAM()); |
| 151 | unsigned long dma_end_pfn = PFN_DOWN(MAX_DMA_ADDRESS); |
Gerald Schaefer | 892365a | 2012-02-24 18:01:29 +0100 | [diff] [blame] | 152 | unsigned long start_pfn = PFN_DOWN(start); |
| 153 | unsigned long size_pages = PFN_DOWN(size); |
Gerald Schaefer | 199071f | 2015-05-08 17:40:43 +0200 | [diff] [blame] | 154 | unsigned long nr_pages; |
| 155 | int rc, zone_enum; |
Heiko Carstens | 421c175 | 2008-07-14 09:59:18 +0200 | [diff] [blame] | 156 | |
Heiko Carstens | 421c175 | 2008-07-14 09:59:18 +0200 | [diff] [blame] | 157 | rc = vmem_add_mapping(start, size); |
| 158 | if (rc) |
| 159 | return rc; |
Gerald Schaefer | 199071f | 2015-05-08 17:40:43 +0200 | [diff] [blame] | 160 | |
| 161 | while (size_pages > 0) { |
| 162 | if (start_pfn < dma_end_pfn) { |
| 163 | nr_pages = (start_pfn + size_pages > dma_end_pfn) ? |
| 164 | dma_end_pfn - start_pfn : size_pages; |
| 165 | zone_enum = ZONE_DMA; |
| 166 | } else if (start_pfn < normal_end_pfn) { |
| 167 | nr_pages = (start_pfn + size_pages > normal_end_pfn) ? |
| 168 | normal_end_pfn - start_pfn : size_pages; |
| 169 | zone_enum = ZONE_NORMAL; |
Gerald Schaefer | 892365a | 2012-02-24 18:01:29 +0100 | [diff] [blame] | 170 | } else { |
Gerald Schaefer | 199071f | 2015-05-08 17:40:43 +0200 | [diff] [blame] | 171 | nr_pages = size_pages; |
| 172 | zone_enum = ZONE_MOVABLE; |
Gerald Schaefer | 892365a | 2012-02-24 18:01:29 +0100 | [diff] [blame] | 173 | } |
Gerald Schaefer | 199071f | 2015-05-08 17:40:43 +0200 | [diff] [blame] | 174 | rc = __add_pages(nid, NODE_DATA(nid)->node_zones + zone_enum, |
| 175 | start_pfn, size_pages); |
Gerald Schaefer | 892365a | 2012-02-24 18:01:29 +0100 | [diff] [blame] | 176 | if (rc) |
| 177 | break; |
| 178 | start_pfn += nr_pages; |
| 179 | size_pages -= nr_pages; |
Gerald Schaefer | 892365a | 2012-02-24 18:01:29 +0100 | [diff] [blame] | 180 | } |
Heiko Carstens | 421c175 | 2008-07-14 09:59:18 +0200 | [diff] [blame] | 181 | if (rc) |
| 182 | vmem_remove_mapping(start, size); |
| 183 | return rc; |
| 184 | } |
Wen Congyang | 24d335c | 2013-02-22 16:32:58 -0800 | [diff] [blame] | 185 | |
Heiko Carstens | e5d709b | 2013-05-02 09:15:58 +0200 | [diff] [blame] | 186 | unsigned long memory_block_size_bytes(void) |
| 187 | { |
| 188 | /* |
| 189 | * Make sure the memory block size is always greater |
| 190 | * or equal than the memory increment size. |
| 191 | */ |
David Hildenbrand | 37c5f6c | 2015-05-06 13:18:59 +0200 | [diff] [blame] | 192 | return max_t(unsigned long, MIN_MEMORY_BLOCK_SIZE, sclp.rzm); |
Heiko Carstens | e5d709b | 2013-05-02 09:15:58 +0200 | [diff] [blame] | 193 | } |
| 194 | |
Wen Congyang | 24d335c | 2013-02-22 16:32:58 -0800 | [diff] [blame] | 195 | #ifdef CONFIG_MEMORY_HOTREMOVE |
| 196 | int arch_remove_memory(u64 start, u64 size) |
| 197 | { |
| 198 | /* |
| 199 | * There is no hardware or firmware interface which could trigger a |
| 200 | * hot memory remove on s390. So there is nothing that needs to be |
| 201 | * implemented. |
| 202 | */ |
| 203 | return -EBUSY; |
| 204 | } |
| 205 | #endif |
Heiko Carstens | 421c175 | 2008-07-14 09:59:18 +0200 | [diff] [blame] | 206 | #endif /* CONFIG_MEMORY_HOTPLUG */ |