Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/mm/memory_hotplug.c |
| 3 | * |
| 4 | * Copyright (C) |
| 5 | */ |
| 6 | |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 7 | #include <linux/stddef.h> |
| 8 | #include <linux/mm.h> |
Ingo Molnar | 174cd4b | 2017-02-02 19:15:33 +0100 | [diff] [blame] | 9 | #include <linux/sched/signal.h> |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 10 | #include <linux/swap.h> |
| 11 | #include <linux/interrupt.h> |
| 12 | #include <linux/pagemap.h> |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 13 | #include <linux/compiler.h> |
Paul Gortmaker | b95f1b31 | 2011-10-16 02:01:52 -0400 | [diff] [blame] | 14 | #include <linux/export.h> |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 15 | #include <linux/pagevec.h> |
Chandra Seetharaman | 2d1d43f | 2006-09-29 02:01:25 -0700 | [diff] [blame] | 16 | #include <linux/writeback.h> |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 17 | #include <linux/slab.h> |
| 18 | #include <linux/sysctl.h> |
| 19 | #include <linux/cpu.h> |
| 20 | #include <linux/memory.h> |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 21 | #include <linux/memremap.h> |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 22 | #include <linux/memory_hotplug.h> |
| 23 | #include <linux/highmem.h> |
| 24 | #include <linux/vmalloc.h> |
KAMEZAWA Hiroyuki | 0a54703 | 2006-06-27 02:53:35 -0700 | [diff] [blame] | 25 | #include <linux/ioport.h> |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 26 | #include <linux/delay.h> |
| 27 | #include <linux/migrate.h> |
| 28 | #include <linux/page-isolation.h> |
Badari Pulavarty | 7108878 | 2008-10-18 20:25:58 -0700 | [diff] [blame] | 29 | #include <linux/pfn.h> |
Andi Kleen | 6ad696d | 2009-11-17 14:06:22 -0800 | [diff] [blame] | 30 | #include <linux/suspend.h> |
KOSAKI Motohiro | 6d9c285 | 2009-12-14 17:58:11 -0800 | [diff] [blame] | 31 | #include <linux/mm_inline.h> |
akpm@linux-foundation.org | d96ae53 | 2010-03-05 13:41:58 -0800 | [diff] [blame] | 32 | #include <linux/firmware-map.h> |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 33 | #include <linux/stop_machine.h> |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 34 | #include <linux/hugetlb.h> |
Tang Chen | c532092 | 2013-11-12 15:08:10 -0800 | [diff] [blame] | 35 | #include <linux/memblock.h> |
Tang Chen | f784a3f | 2014-11-13 15:19:39 -0800 | [diff] [blame] | 36 | #include <linux/bootmem.h> |
Vlastimil Babka | 698b1b3 | 2016-03-17 14:18:08 -0700 | [diff] [blame] | 37 | #include <linux/compaction.h> |
Michal Hocko | 2c87072 | 2018-12-28 00:38:01 -0800 | [diff] [blame] | 38 | #include <linux/rmap.h> |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 39 | |
| 40 | #include <asm/tlbflush.h> |
| 41 | |
Adrian Bunk | 1e5ad9a | 2008-04-28 20:40:08 +0300 | [diff] [blame] | 42 | #include "internal.h" |
| 43 | |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 44 | /* |
| 45 | * online_page_callback contains pointer to current page onlining function. |
| 46 | * Initially it is generic_online_page(). If it is required it could be |
| 47 | * changed by calling set_online_page_callback() for callback registration |
| 48 | * and restore_online_page_callback() for generic callback restore. |
| 49 | */ |
| 50 | |
| 51 | static void generic_online_page(struct page *page); |
| 52 | |
| 53 | static online_page_callback_t online_page_callback = generic_online_page; |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 54 | static DEFINE_MUTEX(online_page_callback_lock); |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 55 | |
Thomas Gleixner | 3f906ba | 2017-07-10 15:50:09 -0700 | [diff] [blame] | 56 | DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock); |
KOSAKI Motohiro | 20d6c96 | 2010-12-02 14:31:19 -0800 | [diff] [blame] | 57 | |
Thomas Gleixner | 3f906ba | 2017-07-10 15:50:09 -0700 | [diff] [blame] | 58 | void get_online_mems(void) |
| 59 | { |
| 60 | percpu_down_read(&mem_hotplug_lock); |
| 61 | } |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 62 | |
Thomas Gleixner | 3f906ba | 2017-07-10 15:50:09 -0700 | [diff] [blame] | 63 | void put_online_mems(void) |
| 64 | { |
| 65 | percpu_up_read(&mem_hotplug_lock); |
| 66 | } |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 67 | |
Michal Hocko | 4932381 | 2017-07-06 15:41:05 -0700 | [diff] [blame] | 68 | bool movable_node_enabled = false; |
| 69 | |
Vitaly Kuznetsov | 8604d9e | 2016-05-19 17:13:03 -0700 | [diff] [blame] | 70 | #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE |
Vitaly Kuznetsov | 31bc385 | 2016-03-15 14:56:48 -0700 | [diff] [blame] | 71 | bool memhp_auto_online; |
Vitaly Kuznetsov | 8604d9e | 2016-05-19 17:13:03 -0700 | [diff] [blame] | 72 | #else |
| 73 | bool memhp_auto_online = true; |
| 74 | #endif |
Vitaly Kuznetsov | 31bc385 | 2016-03-15 14:56:48 -0700 | [diff] [blame] | 75 | EXPORT_SYMBOL_GPL(memhp_auto_online); |
| 76 | |
Vitaly Kuznetsov | 86dd995 | 2016-05-19 17:13:06 -0700 | [diff] [blame] | 77 | static int __init setup_memhp_default_state(char *str) |
| 78 | { |
| 79 | if (!strcmp(str, "online")) |
| 80 | memhp_auto_online = true; |
| 81 | else if (!strcmp(str, "offline")) |
| 82 | memhp_auto_online = false; |
| 83 | |
| 84 | return 1; |
| 85 | } |
| 86 | __setup("memhp_default_state=", setup_memhp_default_state); |
| 87 | |
David Rientjes | 30467e0 | 2015-04-14 15:45:11 -0700 | [diff] [blame] | 88 | void mem_hotplug_begin(void) |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 89 | { |
Thomas Gleixner | 3f906ba | 2017-07-10 15:50:09 -0700 | [diff] [blame] | 90 | cpus_read_lock(); |
| 91 | percpu_down_write(&mem_hotplug_lock); |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 92 | } |
| 93 | |
David Rientjes | 30467e0 | 2015-04-14 15:45:11 -0700 | [diff] [blame] | 94 | void mem_hotplug_done(void) |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 95 | { |
Thomas Gleixner | 3f906ba | 2017-07-10 15:50:09 -0700 | [diff] [blame] | 96 | percpu_up_write(&mem_hotplug_lock); |
| 97 | cpus_read_unlock(); |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 98 | } |
KOSAKI Motohiro | 20d6c96 | 2010-12-02 14:31:19 -0800 | [diff] [blame] | 99 | |
Keith Mannthey | 45e0b78 | 2006-09-30 23:27:09 -0700 | [diff] [blame] | 100 | /* add this memory to iomem resource */ |
| 101 | static struct resource *register_memory_resource(u64 start, u64 size) |
| 102 | { |
Jérôme Glisse | 5042db4 | 2017-09-08 16:11:43 -0700 | [diff] [blame] | 103 | struct resource *res, *conflict; |
Keith Mannthey | 45e0b78 | 2006-09-30 23:27:09 -0700 | [diff] [blame] | 104 | res = kzalloc(sizeof(struct resource), GFP_KERNEL); |
Vitaly Kuznetsov | 6f754ba | 2016-01-14 15:21:55 -0800 | [diff] [blame] | 105 | if (!res) |
| 106 | return ERR_PTR(-ENOMEM); |
Keith Mannthey | 45e0b78 | 2006-09-30 23:27:09 -0700 | [diff] [blame] | 107 | |
| 108 | res->name = "System RAM"; |
| 109 | res->start = start; |
| 110 | res->end = start + size - 1; |
Toshi Kani | 782b866 | 2016-01-26 21:57:24 +0100 | [diff] [blame] | 111 | res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; |
Jérôme Glisse | 5042db4 | 2017-09-08 16:11:43 -0700 | [diff] [blame] | 112 | conflict = request_resource_conflict(&iomem_resource, res); |
| 113 | if (conflict) { |
| 114 | if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) { |
| 115 | pr_debug("Device unaddressable memory block " |
| 116 | "memory hotplug at %#010llx !\n", |
| 117 | (unsigned long long)start); |
| 118 | } |
Toshi Kani | 4996eed | 2013-07-03 15:02:39 -0700 | [diff] [blame] | 119 | pr_debug("System RAM resource %pR cannot be added\n", res); |
Keith Mannthey | 45e0b78 | 2006-09-30 23:27:09 -0700 | [diff] [blame] | 120 | kfree(res); |
Vitaly Kuznetsov | 6f754ba | 2016-01-14 15:21:55 -0800 | [diff] [blame] | 121 | return ERR_PTR(-EEXIST); |
Keith Mannthey | 45e0b78 | 2006-09-30 23:27:09 -0700 | [diff] [blame] | 122 | } |
| 123 | return res; |
| 124 | } |
| 125 | |
| 126 | static void release_memory_resource(struct resource *res) |
| 127 | { |
| 128 | if (!res) |
| 129 | return; |
| 130 | release_resource(res); |
| 131 | kfree(res); |
| 132 | return; |
| 133 | } |
| 134 | |
Keith Mannthey | 5394702 | 2006-09-30 23:27:08 -0700 | [diff] [blame] | 135 | #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE |
Yasuaki Ishimatsu | 46723bf | 2013-02-22 16:33:00 -0800 | [diff] [blame] | 136 | void get_page_bootmem(unsigned long info, struct page *page, |
| 137 | unsigned long type) |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 138 | { |
Yasuaki Ishimatsu | ddffe98 | 2017-02-22 15:45:13 -0800 | [diff] [blame] | 139 | page->freelist = (void *)type; |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 140 | SetPagePrivate(page); |
| 141 | set_page_private(page, info); |
Joonsoo Kim | fe896d1 | 2016-03-17 14:19:26 -0700 | [diff] [blame] | 142 | page_ref_inc(page); |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 143 | } |
| 144 | |
Jiang Liu | 170a5a7 | 2013-07-03 15:03:17 -0700 | [diff] [blame] | 145 | void put_page_bootmem(struct page *page) |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 146 | { |
Andrea Arcangeli | 5f24ce5 | 2011-01-13 15:47:00 -0800 | [diff] [blame] | 147 | unsigned long type; |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 148 | |
Yasuaki Ishimatsu | ddffe98 | 2017-02-22 15:45:13 -0800 | [diff] [blame] | 149 | type = (unsigned long) page->freelist; |
Andrea Arcangeli | 5f24ce5 | 2011-01-13 15:47:00 -0800 | [diff] [blame] | 150 | BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || |
| 151 | type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 152 | |
Joonsoo Kim | fe896d1 | 2016-03-17 14:19:26 -0700 | [diff] [blame] | 153 | if (page_ref_dec_return(page) == 1) { |
Yasuaki Ishimatsu | ddffe98 | 2017-02-22 15:45:13 -0800 | [diff] [blame] | 154 | page->freelist = NULL; |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 155 | ClearPagePrivate(page); |
| 156 | set_page_private(page, 0); |
Andrea Arcangeli | 5f24ce5 | 2011-01-13 15:47:00 -0800 | [diff] [blame] | 157 | INIT_LIST_HEAD(&page->lru); |
Jiang Liu | 170a5a7 | 2013-07-03 15:03:17 -0700 | [diff] [blame] | 158 | free_reserved_page(page); |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 159 | } |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 160 | } |
| 161 | |
Yasuaki Ishimatsu | 46723bf | 2013-02-22 16:33:00 -0800 | [diff] [blame] | 162 | #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE |
| 163 | #ifndef CONFIG_SPARSEMEM_VMEMMAP |
Adrian Bunk | d92bc31 | 2008-07-23 21:28:12 -0700 | [diff] [blame] | 164 | static void register_page_bootmem_info_section(unsigned long start_pfn) |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 165 | { |
| 166 | unsigned long *usemap, mapsize, section_nr, i; |
| 167 | struct mem_section *ms; |
| 168 | struct page *page, *memmap; |
| 169 | |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 170 | section_nr = pfn_to_section_nr(start_pfn); |
| 171 | ms = __nr_to_section(section_nr); |
| 172 | |
| 173 | /* Get section's memmap address */ |
| 174 | memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); |
| 175 | |
| 176 | /* |
| 177 | * Get page for the memmap's phys address |
| 178 | * XXX: need more consideration for sparse_vmemmap... |
| 179 | */ |
| 180 | page = virt_to_page(memmap); |
| 181 | mapsize = sizeof(struct page) * PAGES_PER_SECTION; |
| 182 | mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; |
| 183 | |
| 184 | /* remember memmap's page */ |
| 185 | for (i = 0; i < mapsize; i++, page++) |
| 186 | get_page_bootmem(section_nr, page, SECTION_INFO); |
| 187 | |
Oscar Salvador | 9ac9322 | 2018-01-31 16:17:25 -0800 | [diff] [blame] | 188 | usemap = ms->pageblock_flags; |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 189 | page = virt_to_page(usemap); |
| 190 | |
| 191 | mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; |
| 192 | |
| 193 | for (i = 0; i < mapsize; i++, page++) |
Yasunori Goto | af370fb | 2008-07-23 21:28:17 -0700 | [diff] [blame] | 194 | get_page_bootmem(section_nr, page, MIX_SECTION_INFO); |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 195 | |
| 196 | } |
Yasuaki Ishimatsu | 46723bf | 2013-02-22 16:33:00 -0800 | [diff] [blame] | 197 | #else /* CONFIG_SPARSEMEM_VMEMMAP */ |
| 198 | static void register_page_bootmem_info_section(unsigned long start_pfn) |
| 199 | { |
| 200 | unsigned long *usemap, mapsize, section_nr, i; |
| 201 | struct mem_section *ms; |
| 202 | struct page *page, *memmap; |
| 203 | |
Yasuaki Ishimatsu | 46723bf | 2013-02-22 16:33:00 -0800 | [diff] [blame] | 204 | section_nr = pfn_to_section_nr(start_pfn); |
| 205 | ms = __nr_to_section(section_nr); |
| 206 | |
| 207 | memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); |
| 208 | |
| 209 | register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); |
| 210 | |
Oscar Salvador | 9ac9322 | 2018-01-31 16:17:25 -0800 | [diff] [blame] | 211 | usemap = ms->pageblock_flags; |
Yasuaki Ishimatsu | 46723bf | 2013-02-22 16:33:00 -0800 | [diff] [blame] | 212 | page = virt_to_page(usemap); |
| 213 | |
| 214 | mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; |
| 215 | |
| 216 | for (i = 0; i < mapsize; i++, page++) |
| 217 | get_page_bootmem(section_nr, page, MIX_SECTION_INFO); |
| 218 | } |
| 219 | #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 220 | |
Linus Torvalds | 7ded384 | 2016-05-27 15:23:32 -0700 | [diff] [blame] | 221 | void __init register_page_bootmem_info_node(struct pglist_data *pgdat) |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 222 | { |
| 223 | unsigned long i, pfn, end_pfn, nr_pages; |
| 224 | int node = pgdat->node_id; |
| 225 | struct page *page; |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 226 | |
| 227 | nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; |
| 228 | page = virt_to_page(pgdat); |
| 229 | |
| 230 | for (i = 0; i < nr_pages; i++, page++) |
| 231 | get_page_bootmem(node, page, NODE_INFO); |
| 232 | |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 233 | pfn = pgdat->node_start_pfn; |
Cody P Schafer | c1f1949 | 2013-02-22 16:35:32 -0800 | [diff] [blame] | 234 | end_pfn = pgdat_end_pfn(pgdat); |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 235 | |
Tang Chen | 7e9f5eb | 2013-07-08 16:00:23 -0700 | [diff] [blame] | 236 | /* register section info */ |
qiuxishi | f14851a | 2012-09-17 14:09:24 -0700 | [diff] [blame] | 237 | for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { |
| 238 | /* |
| 239 | * Some platforms can assign the same pfn to multiple nodes - on |
| 240 | * node0 as well as nodeN. To avoid registering a pfn against |
| 241 | * multiple nodes we check that this pfn does not already |
Tang Chen | 7e9f5eb | 2013-07-08 16:00:23 -0700 | [diff] [blame] | 242 | * reside in some other nodes. |
qiuxishi | f14851a | 2012-09-17 14:09:24 -0700 | [diff] [blame] | 243 | */ |
Yang Shi | f65e91df | 2016-05-27 14:27:32 -0700 | [diff] [blame] | 244 | if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) |
qiuxishi | f14851a | 2012-09-17 14:09:24 -0700 | [diff] [blame] | 245 | register_page_bootmem_info_section(pfn); |
| 246 | } |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 247 | } |
Yasuaki Ishimatsu | 46723bf | 2013-02-22 16:33:00 -0800 | [diff] [blame] | 248 | #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ |
Yasunori Goto | 0475327 | 2008-04-28 02:13:31 -0700 | [diff] [blame] | 249 | |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 250 | static int __meminit __add_section(int nid, unsigned long phys_start_pfn, |
Christoph Hellwig | 7b73d97 | 2017-12-29 08:53:54 +0100 | [diff] [blame] | 251 | struct vmem_altmap *altmap, bool want_memblock) |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 252 | { |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 253 | int ret; |
| 254 | |
KAMEZAWA Hiroyuki | ebd1530 | 2006-08-05 12:15:06 -0700 | [diff] [blame] | 255 | if (pfn_valid(phys_start_pfn)) |
| 256 | return -EEXIST; |
| 257 | |
Wei Yang | aa2e8b68 | 2020-01-28 10:50:00 +0100 | [diff] [blame] | 258 | ret = sparse_add_one_section(nid, phys_start_pfn, altmap); |
David Hildenbrand | aa49b6ab | 2020-01-28 10:50:13 +0100 | [diff] [blame] | 259 | return ret < 0 ? ret : 0; |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 260 | } |
| 261 | |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 262 | /* |
| 263 | * Reasonably generic function for adding memory. It is |
| 264 | * expected that archs that support memory hotplug will |
| 265 | * call this function after deciding the zone to which to |
| 266 | * add the new pages. |
| 267 | */ |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 268 | int __ref __add_pages(int nid, unsigned long phys_start_pfn, |
Christoph Hellwig | 24e6d5a | 2017-12-29 08:53:53 +0100 | [diff] [blame] | 269 | unsigned long nr_pages, struct vmem_altmap *altmap, |
| 270 | bool want_memblock) |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 271 | { |
| 272 | unsigned long i; |
| 273 | int err = 0; |
| 274 | int start_sec, end_sec; |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 275 | |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 276 | /* during initialize mem_map, align hot-added range to section */ |
| 277 | start_sec = pfn_to_section_nr(phys_start_pfn); |
| 278 | end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1); |
| 279 | |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 280 | if (altmap) { |
| 281 | /* |
| 282 | * Validate altmap is within bounds of the total request |
| 283 | */ |
| 284 | if (altmap->base_pfn != phys_start_pfn |
| 285 | || vmem_altmap_offset(altmap) > nr_pages) { |
| 286 | pr_warn_once("memory add fail, invalid altmap\n"); |
Joonsoo Kim | 7cf91a9 | 2016-03-15 14:57:51 -0700 | [diff] [blame] | 287 | err = -EINVAL; |
| 288 | goto out; |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 289 | } |
| 290 | altmap->alloc = 0; |
| 291 | } |
| 292 | |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 293 | for (i = start_sec; i <= end_sec; i++) { |
Christoph Hellwig | 7b73d97 | 2017-12-29 08:53:54 +0100 | [diff] [blame] | 294 | err = __add_section(nid, section_nr_to_pfn(i), altmap, |
| 295 | want_memblock); |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 296 | |
| 297 | /* |
| 298 | * EEXIST is finally dealt with by ioresource collision |
| 299 | * check. see add_memory() => register_memory_resource() |
| 300 | * Warning will be printed if there is collision. |
| 301 | */ |
| 302 | if (err && (err != -EEXIST)) |
| 303 | break; |
| 304 | err = 0; |
Michal Hocko | f64ac5e | 2017-10-03 16:16:16 -0700 | [diff] [blame] | 305 | cond_resched(); |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 306 | } |
Zhu Guihua | c435a39 | 2015-06-24 16:58:42 -0700 | [diff] [blame] | 307 | vmemmap_populate_print_last(); |
Joonsoo Kim | 7cf91a9 | 2016-03-15 14:57:51 -0700 | [diff] [blame] | 308 | out: |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 309 | return err; |
| 310 | } |
David Rientjes | 4edd7ce | 2013-04-29 15:08:22 -0700 | [diff] [blame] | 311 | |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 312 | /* find the smallest valid pfn in the range [start_pfn, end_pfn) */ |
YASUAKI ISHIMATSU | d09b013 | 2017-10-03 16:16:32 -0700 | [diff] [blame] | 313 | static unsigned long find_smallest_section_pfn(int nid, struct zone *zone, |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 314 | unsigned long start_pfn, |
| 315 | unsigned long end_pfn) |
| 316 | { |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 317 | for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { |
David Hildenbrand | 5779cbc | 2019-11-21 17:53:56 -0800 | [diff] [blame] | 318 | if (unlikely(!pfn_to_online_page(start_pfn))) |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 319 | continue; |
| 320 | |
| 321 | if (unlikely(pfn_to_nid(start_pfn) != nid)) |
| 322 | continue; |
| 323 | |
| 324 | if (zone && zone != page_zone(pfn_to_page(start_pfn))) |
| 325 | continue; |
| 326 | |
| 327 | return start_pfn; |
| 328 | } |
| 329 | |
| 330 | return 0; |
| 331 | } |
| 332 | |
| 333 | /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ |
YASUAKI ISHIMATSU | d09b013 | 2017-10-03 16:16:32 -0700 | [diff] [blame] | 334 | static unsigned long find_biggest_section_pfn(int nid, struct zone *zone, |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 335 | unsigned long start_pfn, |
| 336 | unsigned long end_pfn) |
| 337 | { |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 338 | unsigned long pfn; |
| 339 | |
| 340 | /* pfn is the end pfn of a memory section. */ |
| 341 | pfn = end_pfn - 1; |
| 342 | for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { |
David Hildenbrand | 5779cbc | 2019-11-21 17:53:56 -0800 | [diff] [blame] | 343 | if (unlikely(!pfn_to_online_page(pfn))) |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 344 | continue; |
| 345 | |
| 346 | if (unlikely(pfn_to_nid(pfn) != nid)) |
| 347 | continue; |
| 348 | |
| 349 | if (zone && zone != page_zone(pfn_to_page(pfn))) |
| 350 | continue; |
| 351 | |
| 352 | return pfn; |
| 353 | } |
| 354 | |
| 355 | return 0; |
| 356 | } |
| 357 | |
| 358 | static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, |
| 359 | unsigned long end_pfn) |
| 360 | { |
Xishi Qiu | c33bc31 | 2013-09-11 14:21:44 -0700 | [diff] [blame] | 361 | unsigned long zone_start_pfn = zone->zone_start_pfn; |
| 362 | unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */ |
| 363 | unsigned long zone_end_pfn = z; |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 364 | unsigned long pfn; |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 365 | int nid = zone_to_nid(zone); |
| 366 | |
| 367 | zone_span_writelock(zone); |
| 368 | if (zone_start_pfn == start_pfn) { |
| 369 | /* |
| 370 | * If the section is smallest section in the zone, it need |
| 371 | * shrink zone->zone_start_pfn and zone->zone_spanned_pages. |
| 372 | * In this case, we find second smallest valid mem_section |
| 373 | * for shrinking zone. |
| 374 | */ |
| 375 | pfn = find_smallest_section_pfn(nid, zone, end_pfn, |
| 376 | zone_end_pfn); |
| 377 | if (pfn) { |
| 378 | zone->zone_start_pfn = pfn; |
| 379 | zone->spanned_pages = zone_end_pfn - pfn; |
| 380 | } |
| 381 | } else if (zone_end_pfn == end_pfn) { |
| 382 | /* |
| 383 | * If the section is biggest section in the zone, it need |
| 384 | * shrink zone->spanned_pages. |
| 385 | * In this case, we find second biggest valid mem_section for |
| 386 | * shrinking zone. |
| 387 | */ |
| 388 | pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn, |
| 389 | start_pfn); |
| 390 | if (pfn) |
| 391 | zone->spanned_pages = pfn - zone_start_pfn + 1; |
| 392 | } |
| 393 | |
| 394 | /* |
| 395 | * The section is not biggest or smallest mem_section in the zone, it |
| 396 | * only creates a hole in the zone. So in this case, we need not |
| 397 | * change the zone. But perhaps, the zone has only hole data. Thus |
| 398 | * it check the zone has only hole or not. |
| 399 | */ |
| 400 | pfn = zone_start_pfn; |
| 401 | for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) { |
David Hildenbrand | 5779cbc | 2019-11-21 17:53:56 -0800 | [diff] [blame] | 402 | if (unlikely(!pfn_to_online_page(pfn))) |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 403 | continue; |
| 404 | |
| 405 | if (page_zone(pfn_to_page(pfn)) != zone) |
| 406 | continue; |
| 407 | |
| 408 | /* If the section is current section, it continues the loop */ |
| 409 | if (start_pfn == pfn) |
| 410 | continue; |
| 411 | |
| 412 | /* If we find valid section, we have nothing to do */ |
| 413 | zone_span_writeunlock(zone); |
| 414 | return; |
| 415 | } |
| 416 | |
| 417 | /* The zone has no valid section */ |
| 418 | zone->zone_start_pfn = 0; |
| 419 | zone->spanned_pages = 0; |
| 420 | zone_span_writeunlock(zone); |
| 421 | } |
| 422 | |
David Hildenbrand | 6631def | 2019-10-18 20:19:33 -0700 | [diff] [blame] | 423 | static void update_pgdat_span(struct pglist_data *pgdat) |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 424 | { |
David Hildenbrand | 6631def | 2019-10-18 20:19:33 -0700 | [diff] [blame] | 425 | unsigned long node_start_pfn = 0, node_end_pfn = 0; |
| 426 | struct zone *zone; |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 427 | |
David Hildenbrand | 6631def | 2019-10-18 20:19:33 -0700 | [diff] [blame] | 428 | for (zone = pgdat->node_zones; |
| 429 | zone < pgdat->node_zones + MAX_NR_ZONES; zone++) { |
| 430 | unsigned long zone_end_pfn = zone->zone_start_pfn + |
| 431 | zone->spanned_pages; |
| 432 | |
| 433 | /* No need to lock the zones, they can't change. */ |
David Hildenbrand | f8b09a0 | 2019-11-05 21:17:10 -0800 | [diff] [blame] | 434 | if (!zone->spanned_pages) |
| 435 | continue; |
| 436 | if (!node_end_pfn) { |
| 437 | node_start_pfn = zone->zone_start_pfn; |
| 438 | node_end_pfn = zone_end_pfn; |
| 439 | continue; |
| 440 | } |
| 441 | |
David Hildenbrand | 6631def | 2019-10-18 20:19:33 -0700 | [diff] [blame] | 442 | if (zone_end_pfn > node_end_pfn) |
| 443 | node_end_pfn = zone_end_pfn; |
| 444 | if (zone->zone_start_pfn < node_start_pfn) |
| 445 | node_start_pfn = zone->zone_start_pfn; |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 446 | } |
| 447 | |
David Hildenbrand | 6631def | 2019-10-18 20:19:33 -0700 | [diff] [blame] | 448 | pgdat->node_start_pfn = node_start_pfn; |
| 449 | pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 450 | } |
| 451 | |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 452 | void __ref remove_pfn_range_from_zone(struct zone *zone, |
| 453 | unsigned long start_pfn, |
| 454 | unsigned long nr_pages) |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 455 | { |
| 456 | struct pglist_data *pgdat = zone->zone_pgdat; |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 457 | unsigned long flags; |
| 458 | |
David Hildenbrand | 5779cbc | 2019-11-21 17:53:56 -0800 | [diff] [blame] | 459 | #ifdef CONFIG_ZONE_DEVICE |
| 460 | /* |
| 461 | * Zone shrinking code cannot properly deal with ZONE_DEVICE. So |
| 462 | * we will not try to shrink the zones - which is okay as |
| 463 | * set_zone_contiguous() cannot deal with ZONE_DEVICE either way. |
| 464 | */ |
| 465 | if (zone_idx(zone) == ZONE_DEVICE) |
| 466 | return; |
| 467 | #endif |
| 468 | |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 469 | clear_zone_contiguous(zone); |
| 470 | |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 471 | pgdat_resize_lock(zone->zone_pgdat, &flags); |
| 472 | shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); |
David Hildenbrand | 6631def | 2019-10-18 20:19:33 -0700 | [diff] [blame] | 473 | update_pgdat_span(pgdat); |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 474 | pgdat_resize_unlock(zone->zone_pgdat, &flags); |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 475 | |
| 476 | set_zone_contiguous(zone); |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 477 | } |
| 478 | |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 479 | static void __remove_section(struct mem_section *ms, unsigned long map_offset, |
David Hildenbrand | efaa8fb | 2020-01-28 10:50:07 +0100 | [diff] [blame] | 480 | struct vmem_altmap *altmap) |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 481 | { |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 482 | unsigned long start_pfn; |
| 483 | int scn_nr; |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 484 | |
David Hildenbrand | efaa8fb | 2020-01-28 10:50:07 +0100 | [diff] [blame] | 485 | if (WARN_ON_ONCE(!valid_section(ms))) |
| 486 | return; |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 487 | |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 488 | scn_nr = __section_nr(ms); |
YASUAKI ISHIMATSU | 1dd2bfc | 2017-10-03 16:16:29 -0700 | [diff] [blame] | 489 | start_pfn = section_nr_to_pfn((unsigned long)scn_nr); |
Yasuaki Ishimatsu | 815121d | 2013-02-22 16:33:12 -0800 | [diff] [blame] | 490 | |
David Hildenbrand | dc6be85 | 2020-01-28 10:50:16 +0100 | [diff] [blame] | 491 | sparse_remove_one_section(ms, map_offset, altmap); |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 492 | } |
| 493 | |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 494 | /** |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 495 | * __remove_pages() - remove sections of pages |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 496 | * @phys_start_pfn: starting pageframe (must be aligned to start of a section) |
| 497 | * @nr_pages: number of pages to remove (must be multiple of section size) |
Mike Rapoport | e8b098f | 2018-04-05 16:24:57 -0700 | [diff] [blame] | 498 | * @altmap: alternative device page map or %NULL if default memmap is used |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 499 | * |
| 500 | * Generic helper function to remove section mappings and sysfs entries |
| 501 | * for the section of the memory we are removing. Caller needs to make |
| 502 | * sure that pages are marked reserved and zones are adjust properly by |
| 503 | * calling offline_pages(). |
| 504 | */ |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 505 | void __remove_pages(unsigned long phys_start_pfn, unsigned long nr_pages, |
| 506 | struct vmem_altmap *altmap) |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 507 | { |
Toshi Kani | fe74ebb | 2013-04-29 15:08:20 -0700 | [diff] [blame] | 508 | unsigned long i; |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 509 | unsigned long map_offset = 0; |
David Hildenbrand | efaa8fb | 2020-01-28 10:50:07 +0100 | [diff] [blame] | 510 | int sections_to_remove; |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 511 | |
Dan Williams | b9cda65 | 2020-01-28 10:50:17 +0100 | [diff] [blame] | 512 | if (altmap) |
| 513 | map_offset = vmem_altmap_offset(altmap); |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 514 | |
| 515 | /* |
| 516 | * We can only remove entire sections |
| 517 | */ |
| 518 | BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK); |
| 519 | BUG_ON(nr_pages % PAGES_PER_SECTION); |
| 520 | |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 521 | sections_to_remove = nr_pages / PAGES_PER_SECTION; |
| 522 | for (i = 0; i < sections_to_remove; i++) { |
| 523 | unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION; |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 524 | |
Michal Hocko | ccd35ba5 | 2018-11-02 15:48:46 -0700 | [diff] [blame] | 525 | cond_resched(); |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 526 | __remove_section(__pfn_to_section(pfn), map_offset, altmap); |
Dan Williams | 4b94ffd | 2016-01-15 16:56:22 -0800 | [diff] [blame] | 527 | map_offset = 0; |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 528 | } |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 529 | } |
Badari Pulavarty | ea01ea9 | 2008-04-28 02:12:01 -0700 | [diff] [blame] | 530 | |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 531 | int set_online_page_callback(online_page_callback_t callback) |
| 532 | { |
| 533 | int rc = -EINVAL; |
| 534 | |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 535 | get_online_mems(); |
| 536 | mutex_lock(&online_page_callback_lock); |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 537 | |
| 538 | if (online_page_callback == generic_online_page) { |
| 539 | online_page_callback = callback; |
| 540 | rc = 0; |
| 541 | } |
| 542 | |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 543 | mutex_unlock(&online_page_callback_lock); |
| 544 | put_online_mems(); |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 545 | |
| 546 | return rc; |
| 547 | } |
| 548 | EXPORT_SYMBOL_GPL(set_online_page_callback); |
| 549 | |
| 550 | int restore_online_page_callback(online_page_callback_t callback) |
| 551 | { |
| 552 | int rc = -EINVAL; |
| 553 | |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 554 | get_online_mems(); |
| 555 | mutex_lock(&online_page_callback_lock); |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 556 | |
| 557 | if (online_page_callback == callback) { |
| 558 | online_page_callback = generic_online_page; |
| 559 | rc = 0; |
| 560 | } |
| 561 | |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 562 | mutex_unlock(&online_page_callback_lock); |
| 563 | put_online_mems(); |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 564 | |
| 565 | return rc; |
| 566 | } |
| 567 | EXPORT_SYMBOL_GPL(restore_online_page_callback); |
| 568 | |
| 569 | void __online_page_set_limits(struct page *page) |
Jeremy Fitzhardinge | 180c06e | 2008-04-28 02:12:03 -0700 | [diff] [blame] | 570 | { |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 571 | } |
| 572 | EXPORT_SYMBOL_GPL(__online_page_set_limits); |
| 573 | |
| 574 | void __online_page_increment_counters(struct page *page) |
| 575 | { |
Jiang Liu | 3dcc057 | 2013-07-03 15:03:21 -0700 | [diff] [blame] | 576 | adjust_managed_page_count(page, 1); |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 577 | } |
| 578 | EXPORT_SYMBOL_GPL(__online_page_increment_counters); |
Jeremy Fitzhardinge | 180c06e | 2008-04-28 02:12:03 -0700 | [diff] [blame] | 579 | |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 580 | void __online_page_free(struct page *page) |
| 581 | { |
Jiang Liu | 3dcc057 | 2013-07-03 15:03:21 -0700 | [diff] [blame] | 582 | __free_reserved_page(page); |
Jeremy Fitzhardinge | 180c06e | 2008-04-28 02:12:03 -0700 | [diff] [blame] | 583 | } |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 584 | EXPORT_SYMBOL_GPL(__online_page_free); |
| 585 | |
| 586 | static void generic_online_page(struct page *page) |
| 587 | { |
| 588 | __online_page_set_limits(page); |
| 589 | __online_page_increment_counters(page); |
| 590 | __online_page_free(page); |
| 591 | } |
Jeremy Fitzhardinge | 180c06e | 2008-04-28 02:12:03 -0700 | [diff] [blame] | 592 | |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 593 | static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, |
| 594 | void *arg) |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 595 | { |
| 596 | unsigned long i; |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 597 | unsigned long onlined_pages = *(unsigned long *)arg; |
| 598 | struct page *page; |
Michal Hocko | 2d070ea | 2017-07-06 15:37:56 -0700 | [diff] [blame] | 599 | |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 600 | if (PageReserved(pfn_to_page(start_pfn))) |
| 601 | for (i = 0; i < nr_pages; i++) { |
| 602 | page = pfn_to_page(start_pfn + i); |
Daniel Kiper | 9d0ad8c | 2011-07-25 17:12:05 -0700 | [diff] [blame] | 603 | (*online_page_callback)(page); |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 604 | onlined_pages++; |
| 605 | } |
Michal Hocko | 2d070ea | 2017-07-06 15:37:56 -0700 | [diff] [blame] | 606 | |
| 607 | online_mem_sections(start_pfn, start_pfn + nr_pages); |
| 608 | |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 609 | *(unsigned long *)arg = onlined_pages; |
| 610 | return 0; |
| 611 | } |
| 612 | |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 613 | /* check which state of node_states will be changed when online memory */ |
| 614 | static void node_states_check_changes_online(unsigned long nr_pages, |
| 615 | struct zone *zone, struct memory_notify *arg) |
| 616 | { |
| 617 | int nid = zone_to_nid(zone); |
| 618 | enum zone_type zone_last = ZONE_NORMAL; |
| 619 | |
| 620 | /* |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 621 | * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] |
| 622 | * contains nodes which have zones of 0...ZONE_NORMAL, |
| 623 | * set zone_last to ZONE_NORMAL. |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 624 | * |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 625 | * If we don't have HIGHMEM nor movable node, |
| 626 | * node_states[N_NORMAL_MEMORY] contains nodes which have zones of |
| 627 | * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 628 | */ |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 629 | if (N_MEMORY == N_NORMAL_MEMORY) |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 630 | zone_last = ZONE_MOVABLE; |
| 631 | |
| 632 | /* |
| 633 | * if the memory to be online is in a zone of 0...zone_last, and |
| 634 | * the zones of 0...zone_last don't have memory before online, we will |
| 635 | * need to set the node to node_states[N_NORMAL_MEMORY] after |
| 636 | * the memory is online. |
| 637 | */ |
| 638 | if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY)) |
| 639 | arg->status_change_nid_normal = nid; |
| 640 | else |
| 641 | arg->status_change_nid_normal = -1; |
| 642 | |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 643 | #ifdef CONFIG_HIGHMEM |
| 644 | /* |
| 645 | * If we have movable node, node_states[N_HIGH_MEMORY] |
| 646 | * contains nodes which have zones of 0...ZONE_HIGHMEM, |
| 647 | * set zone_last to ZONE_HIGHMEM. |
| 648 | * |
| 649 | * If we don't have movable node, node_states[N_NORMAL_MEMORY] |
| 650 | * contains nodes which have zones of 0...ZONE_MOVABLE, |
| 651 | * set zone_last to ZONE_MOVABLE. |
| 652 | */ |
| 653 | zone_last = ZONE_HIGHMEM; |
| 654 | if (N_MEMORY == N_HIGH_MEMORY) |
| 655 | zone_last = ZONE_MOVABLE; |
| 656 | |
| 657 | if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY)) |
| 658 | arg->status_change_nid_high = nid; |
| 659 | else |
| 660 | arg->status_change_nid_high = -1; |
| 661 | #else |
| 662 | arg->status_change_nid_high = arg->status_change_nid_normal; |
| 663 | #endif |
| 664 | |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 665 | /* |
| 666 | * if the node don't have memory befor online, we will need to |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 667 | * set the node to node_states[N_MEMORY] after the memory |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 668 | * is online. |
| 669 | */ |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 670 | if (!node_state(nid, N_MEMORY)) |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 671 | arg->status_change_nid = nid; |
| 672 | else |
| 673 | arg->status_change_nid = -1; |
| 674 | } |
| 675 | |
| 676 | static void node_states_set_node(int node, struct memory_notify *arg) |
| 677 | { |
| 678 | if (arg->status_change_nid_normal >= 0) |
| 679 | node_set_state(node, N_NORMAL_MEMORY); |
| 680 | |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 681 | if (arg->status_change_nid_high >= 0) |
| 682 | node_set_state(node, N_HIGH_MEMORY); |
| 683 | |
| 684 | node_set_state(node, N_MEMORY); |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 685 | } |
| 686 | |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 687 | static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn, |
| 688 | unsigned long nr_pages) |
| 689 | { |
| 690 | unsigned long old_end_pfn = zone_end_pfn(zone); |
| 691 | |
| 692 | if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) |
| 693 | zone->zone_start_pfn = start_pfn; |
| 694 | |
| 695 | zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn; |
| 696 | } |
| 697 | |
| 698 | static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn, |
| 699 | unsigned long nr_pages) |
| 700 | { |
| 701 | unsigned long old_end_pfn = pgdat_end_pfn(pgdat); |
| 702 | |
| 703 | if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) |
| 704 | pgdat->node_start_pfn = start_pfn; |
| 705 | |
| 706 | pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn; |
| 707 | } |
| 708 | |
Christoph Hellwig | a99583e | 2017-12-29 08:53:57 +0100 | [diff] [blame] | 709 | void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, |
| 710 | unsigned long nr_pages, struct vmem_altmap *altmap) |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 711 | { |
| 712 | struct pglist_data *pgdat = zone->zone_pgdat; |
| 713 | int nid = pgdat->node_id; |
| 714 | unsigned long flags; |
| 715 | |
| 716 | if (zone_is_empty(zone)) |
| 717 | init_currently_empty_zone(zone, start_pfn, nr_pages); |
| 718 | |
| 719 | clear_zone_contiguous(zone); |
| 720 | |
| 721 | /* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */ |
| 722 | pgdat_resize_lock(pgdat, &flags); |
| 723 | zone_span_writelock(zone); |
| 724 | resize_zone_range(zone, start_pfn, nr_pages); |
| 725 | zone_span_writeunlock(zone); |
| 726 | resize_pgdat_range(pgdat, start_pfn, nr_pages); |
| 727 | pgdat_resize_unlock(pgdat, &flags); |
| 728 | |
| 729 | /* |
| 730 | * TODO now we have a visible range of pages which are not associated |
| 731 | * with their zone properly. Not nice but set_pfnblock_flags_mask |
| 732 | * expects the zone spans the pfn range. All the pages in the range |
| 733 | * are reserved so nobody should be touching them so we should be safe |
| 734 | */ |
Christoph Hellwig | a99583e | 2017-12-29 08:53:57 +0100 | [diff] [blame] | 735 | memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, |
Laurent Dufour | 25eaea1 | 2020-09-25 21:19:28 -0700 | [diff] [blame] | 736 | MEMINIT_HOTPLUG, altmap); |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 737 | |
| 738 | set_zone_contiguous(zone); |
| 739 | } |
| 740 | |
| 741 | /* |
Michal Hocko | c246a21 | 2017-07-06 15:38:18 -0700 | [diff] [blame] | 742 | * Returns a default kernel memory zone for the given pfn range. |
| 743 | * If no kernel zone covers this pfn range it will automatically go |
| 744 | * to the ZONE_NORMAL. |
| 745 | */ |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 746 | static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn, |
Michal Hocko | c246a21 | 2017-07-06 15:38:18 -0700 | [diff] [blame] | 747 | unsigned long nr_pages) |
| 748 | { |
| 749 | struct pglist_data *pgdat = NODE_DATA(nid); |
| 750 | int zid; |
| 751 | |
| 752 | for (zid = 0; zid <= ZONE_NORMAL; zid++) { |
| 753 | struct zone *zone = &pgdat->node_zones[zid]; |
| 754 | |
| 755 | if (zone_intersects(zone, start_pfn, nr_pages)) |
| 756 | return zone; |
| 757 | } |
| 758 | |
| 759 | return &pgdat->node_zones[ZONE_NORMAL]; |
| 760 | } |
| 761 | |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 762 | static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn, |
| 763 | unsigned long nr_pages) |
Michal Hocko | e5e6893 | 2017-09-06 16:19:37 -0700 | [diff] [blame] | 764 | { |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 765 | struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn, |
| 766 | nr_pages); |
| 767 | struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; |
| 768 | bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages); |
| 769 | bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages); |
Michal Hocko | e5e6893 | 2017-09-06 16:19:37 -0700 | [diff] [blame] | 770 | |
| 771 | /* |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 772 | * We inherit the existing zone in a simple case where zones do not |
| 773 | * overlap in the given range |
Michal Hocko | e5e6893 | 2017-09-06 16:19:37 -0700 | [diff] [blame] | 774 | */ |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 775 | if (in_kernel ^ in_movable) |
| 776 | return (in_kernel) ? kernel_zone : movable_zone; |
Michal Hocko | e5e6893 | 2017-09-06 16:19:37 -0700 | [diff] [blame] | 777 | |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 778 | /* |
| 779 | * If the range doesn't belong to any zone or two zones overlap in the |
| 780 | * given range then we use movable zone only if movable_node is |
| 781 | * enabled because we always online to a kernel zone by default. |
| 782 | */ |
| 783 | return movable_node_enabled ? movable_zone : kernel_zone; |
Michal Hocko | 9f123ab | 2017-07-10 15:48:37 -0700 | [diff] [blame] | 784 | } |
| 785 | |
Michal Hocko | e5e6893 | 2017-09-06 16:19:37 -0700 | [diff] [blame] | 786 | struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn, |
| 787 | unsigned long nr_pages) |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 788 | { |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 789 | if (online_type == MMOP_ONLINE_KERNEL) |
| 790 | return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages); |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 791 | |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 792 | if (online_type == MMOP_ONLINE_MOVABLE) |
| 793 | return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; |
Reza Arbab | df429ac | 2016-07-26 15:22:23 -0700 | [diff] [blame] | 794 | |
Michal Hocko | c6f03e2 | 2017-09-06 16:19:40 -0700 | [diff] [blame] | 795 | return default_zone_for_pfn(nid, start_pfn, nr_pages); |
Michal Hocko | e5e6893 | 2017-09-06 16:19:37 -0700 | [diff] [blame] | 796 | } |
| 797 | |
| 798 | /* |
| 799 | * Associates the given pfn range with the given node and the zone appropriate |
| 800 | * for the given online type. |
| 801 | */ |
| 802 | static struct zone * __meminit move_pfn_range(int online_type, int nid, |
| 803 | unsigned long start_pfn, unsigned long nr_pages) |
| 804 | { |
| 805 | struct zone *zone; |
| 806 | |
| 807 | zone = zone_for_pfn_range(online_type, nid, start_pfn, nr_pages); |
Christoph Hellwig | a99583e | 2017-12-29 08:53:57 +0100 | [diff] [blame] | 808 | move_pfn_range_to_zone(zone, start_pfn, nr_pages, NULL); |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 809 | return zone; |
Reza Arbab | df429ac | 2016-07-26 15:22:23 -0700 | [diff] [blame] | 810 | } |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 811 | |
Lai Jiangshan | 511c2ab | 2012-12-11 16:03:16 -0800 | [diff] [blame] | 812 | int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type) |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 813 | { |
Cody P Schafer | aa47228 | 2013-07-03 15:02:10 -0700 | [diff] [blame] | 814 | unsigned long flags; |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 815 | unsigned long onlined_pages = 0; |
| 816 | struct zone *zone; |
Yasunori Goto | 6811378 | 2006-06-23 02:03:11 -0700 | [diff] [blame] | 817 | int need_zonelists_rebuild = 0; |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 818 | int nid; |
| 819 | int ret; |
| 820 | struct memory_notify arg; |
Pavel Tatashin | d0dc12e | 2018-04-05 16:23:00 -0700 | [diff] [blame] | 821 | struct memory_block *mem; |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 822 | |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 823 | mem_hotplug_begin(); |
| 824 | |
Pavel Tatashin | d0dc12e | 2018-04-05 16:23:00 -0700 | [diff] [blame] | 825 | /* |
| 826 | * We can't use pfn_to_nid() because nid might be stored in struct page |
| 827 | * which is not yet initialized. Instead, we find nid from memory block. |
| 828 | */ |
| 829 | mem = find_memory_block(__pfn_to_section(pfn)); |
| 830 | nid = mem->nid; |
David Hildenbrand | 6a60fb6 | 2019-04-25 22:23:37 -0700 | [diff] [blame] | 831 | put_device(&mem->dev); |
Pavel Tatashin | d0dc12e | 2018-04-05 16:23:00 -0700 | [diff] [blame] | 832 | |
Michal Hocko | f1dd2cd | 2017-07-06 15:38:11 -0700 | [diff] [blame] | 833 | /* associate pfn range with the zone */ |
| 834 | zone = move_pfn_range(online_type, nid, pfn, nr_pages); |
| 835 | |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 836 | arg.start_pfn = pfn; |
| 837 | arg.nr_pages = nr_pages; |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 838 | node_states_check_changes_online(nr_pages, zone, &arg); |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 839 | |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 840 | ret = memory_notify(MEM_GOING_ONLINE, &arg); |
| 841 | ret = notifier_to_errno(ret); |
Chen Yucong | e33e33b | 2016-03-17 14:19:35 -0700 | [diff] [blame] | 842 | if (ret) |
| 843 | goto failed_addition; |
| 844 | |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 845 | /* |
Yasunori Goto | 6811378 | 2006-06-23 02:03:11 -0700 | [diff] [blame] | 846 | * If this zone is not populated, then it is not in zonelist. |
| 847 | * This means the page allocator ignores this zone. |
| 848 | * So, zonelist must be updated after online. |
| 849 | */ |
Wen Congyang | 6dcd73d | 2012-12-11 16:01:01 -0800 | [diff] [blame] | 850 | if (!populated_zone(zone)) { |
Yasunori Goto | 6811378 | 2006-06-23 02:03:11 -0700 | [diff] [blame] | 851 | need_zonelists_rebuild = 1; |
Michal Hocko | 72675e1 | 2017-09-06 16:20:24 -0700 | [diff] [blame] | 852 | setup_zone_pageset(zone); |
Wen Congyang | 6dcd73d | 2012-12-11 16:01:01 -0800 | [diff] [blame] | 853 | } |
Yasunori Goto | 6811378 | 2006-06-23 02:03:11 -0700 | [diff] [blame] | 854 | |
KAMEZAWA Hiroyuki | 908eedc | 2009-09-22 16:45:46 -0700 | [diff] [blame] | 855 | ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, |
KAMEZAWA Hiroyuki | 75884fb | 2007-10-16 01:26:10 -0700 | [diff] [blame] | 856 | online_pages_range); |
Geoff Levand | fd8a422 | 2008-05-14 16:05:50 -0700 | [diff] [blame] | 857 | if (ret) { |
Wen Congyang | 6dcd73d | 2012-12-11 16:01:01 -0800 | [diff] [blame] | 858 | if (need_zonelists_rebuild) |
| 859 | zone_pcp_reset(zone); |
Chen Yucong | e33e33b | 2016-03-17 14:19:35 -0700 | [diff] [blame] | 860 | goto failed_addition; |
Geoff Levand | fd8a422 | 2008-05-14 16:05:50 -0700 | [diff] [blame] | 861 | } |
| 862 | |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 863 | zone->present_pages += onlined_pages; |
Cody P Schafer | aa47228 | 2013-07-03 15:02:10 -0700 | [diff] [blame] | 864 | |
| 865 | pgdat_resize_lock(zone->zone_pgdat, &flags); |
Yasunori Goto | f2937be | 2006-03-09 17:33:51 -0800 | [diff] [blame] | 866 | zone->zone_pgdat->node_present_pages += onlined_pages; |
Cody P Schafer | aa47228 | 2013-07-03 15:02:10 -0700 | [diff] [blame] | 867 | pgdat_resize_unlock(zone->zone_pgdat, &flags); |
| 868 | |
Jiang Liu | 08dff7b | 2012-07-31 16:43:30 -0700 | [diff] [blame] | 869 | if (onlined_pages) { |
Vlastimil Babka | e888ca3 | 2016-03-17 14:18:12 -0700 | [diff] [blame] | 870 | node_states_set_node(nid, &arg); |
Jiang Liu | 08dff7b | 2012-07-31 16:43:30 -0700 | [diff] [blame] | 871 | if (need_zonelists_rebuild) |
Michal Hocko | 72675e1 | 2017-09-06 16:20:24 -0700 | [diff] [blame] | 872 | build_all_zonelists(NULL); |
Jiang Liu | 08dff7b | 2012-07-31 16:43:30 -0700 | [diff] [blame] | 873 | else |
| 874 | zone_pcp_update(zone); |
| 875 | } |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 876 | |
KOSAKI Motohiro | 1b79acc | 2011-05-24 17:11:32 -0700 | [diff] [blame] | 877 | init_per_zone_wmark_min(); |
| 878 | |
Vlastimil Babka | 698b1b3 | 2016-03-17 14:18:08 -0700 | [diff] [blame] | 879 | if (onlined_pages) { |
Vlastimil Babka | e888ca3 | 2016-03-17 14:18:12 -0700 | [diff] [blame] | 880 | kswapd_run(nid); |
Vlastimil Babka | 698b1b3 | 2016-03-17 14:18:08 -0700 | [diff] [blame] | 881 | kcompactd_run(nid); |
| 882 | } |
Dave Hansen | 61b1399 | 2005-10-29 18:16:56 -0700 | [diff] [blame] | 883 | |
Haicheng Li | 1f52250 | 2010-05-24 14:32:51 -0700 | [diff] [blame] | 884 | vm_total_pages = nr_free_pagecache_pages(); |
Kent Liu | 2f7f24e | 2008-07-23 21:28:18 -0700 | [diff] [blame] | 885 | |
Chandra Seetharaman | 2d1d43f | 2006-09-29 02:01:25 -0700 | [diff] [blame] | 886 | writeback_set_ratelimit(); |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 887 | |
| 888 | if (onlined_pages) |
| 889 | memory_notify(MEM_ONLINE, &arg); |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 890 | mem_hotplug_done(); |
David Rientjes | 30467e0 | 2015-04-14 15:45:11 -0700 | [diff] [blame] | 891 | return 0; |
Chen Yucong | e33e33b | 2016-03-17 14:19:35 -0700 | [diff] [blame] | 892 | |
| 893 | failed_addition: |
| 894 | pr_debug("online_pages [mem %#010llx-%#010llx] failed\n", |
| 895 | (unsigned long long) pfn << PAGE_SHIFT, |
| 896 | (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1); |
| 897 | memory_notify(MEM_CANCEL_ONLINE, &arg); |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 898 | remove_pfn_range_from_zone(zone, pfn, nr_pages); |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 899 | mem_hotplug_done(); |
Chen Yucong | e33e33b | 2016-03-17 14:19:35 -0700 | [diff] [blame] | 900 | return ret; |
Dave Hansen | 3947be1 | 2005-10-29 18:16:54 -0700 | [diff] [blame] | 901 | } |
Keith Mannthey | 5394702 | 2006-09-30 23:27:08 -0700 | [diff] [blame] | 902 | #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 903 | |
Tang Chen | 0bd8542 | 2014-11-13 15:19:41 -0800 | [diff] [blame] | 904 | static void reset_node_present_pages(pg_data_t *pgdat) |
| 905 | { |
| 906 | struct zone *z; |
| 907 | |
| 908 | for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) |
| 909 | z->present_pages = 0; |
| 910 | |
| 911 | pgdat->node_present_pages = 0; |
| 912 | } |
| 913 | |
Hidetoshi Seto | e131933 | 2009-11-17 14:06:18 -0800 | [diff] [blame] | 914 | /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ |
| 915 | static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start) |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 916 | { |
| 917 | struct pglist_data *pgdat; |
Fabian Frederick | c8e861a | 2014-06-04 16:07:51 -0700 | [diff] [blame] | 918 | unsigned long start_pfn = PFN_DOWN(start); |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 919 | |
Tang Chen | a1e565a | 2013-02-22 16:33:18 -0800 | [diff] [blame] | 920 | pgdat = NODE_DATA(nid); |
| 921 | if (!pgdat) { |
| 922 | pgdat = arch_alloc_nodedata(nid); |
| 923 | if (!pgdat) |
| 924 | return NULL; |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 925 | |
Tang Chen | a1e565a | 2013-02-22 16:33:18 -0800 | [diff] [blame] | 926 | arch_refresh_nodedata(nid, pgdat); |
Gu Zheng | b0dc3a3 | 2015-03-25 15:55:20 -0700 | [diff] [blame] | 927 | } else { |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 928 | /* |
| 929 | * Reset the nr_zones, order and classzone_idx before reuse. |
| 930 | * Note that kswapd will init kswapd_classzone_idx properly |
| 931 | * when it starts in the near future. |
| 932 | */ |
Gu Zheng | b0dc3a3 | 2015-03-25 15:55:20 -0700 | [diff] [blame] | 933 | pgdat->nr_zones = 0; |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 934 | pgdat->kswapd_order = 0; |
| 935 | pgdat->kswapd_classzone_idx = 0; |
Tang Chen | a1e565a | 2013-02-22 16:33:18 -0800 | [diff] [blame] | 936 | } |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 937 | |
| 938 | /* we can use NODE_DATA(nid) from here */ |
| 939 | |
Oscar Salvador | 03e85f9 | 2018-08-21 21:53:43 -0700 | [diff] [blame] | 940 | pgdat->node_id = nid; |
| 941 | pgdat->node_start_pfn = start_pfn; |
| 942 | |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 943 | /* init node's zones as empty zones, we don't have any present pages.*/ |
Oscar Salvador | 03e85f9 | 2018-08-21 21:53:43 -0700 | [diff] [blame] | 944 | free_area_init_core_hotplug(nid); |
Reza Arbab | 5830169 | 2016-08-11 15:33:12 -0700 | [diff] [blame] | 945 | pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 946 | |
KAMEZAWA Hiroyuki | 959ecc4 | 2011-06-15 15:08:38 -0700 | [diff] [blame] | 947 | /* |
| 948 | * The node we allocated has no zone fallback lists. For avoiding |
| 949 | * to access not-initialized zonelist, build here. |
| 950 | */ |
Michal Hocko | 72675e1 | 2017-09-06 16:20:24 -0700 | [diff] [blame] | 951 | build_all_zonelists(pgdat); |
KAMEZAWA Hiroyuki | 959ecc4 | 2011-06-15 15:08:38 -0700 | [diff] [blame] | 952 | |
Tang Chen | f784a3f | 2014-11-13 15:19:39 -0800 | [diff] [blame] | 953 | /* |
Tang Chen | 0bd8542 | 2014-11-13 15:19:41 -0800 | [diff] [blame] | 954 | * When memory is hot-added, all the memory is in offline state. So |
| 955 | * clear all zones' present_pages because they will be updated in |
| 956 | * online_pages() and offline_pages(). |
| 957 | */ |
Oscar Salvador | 03e85f9 | 2018-08-21 21:53:43 -0700 | [diff] [blame] | 958 | reset_node_managed_pages(pgdat); |
Tang Chen | 0bd8542 | 2014-11-13 15:19:41 -0800 | [diff] [blame] | 959 | reset_node_present_pages(pgdat); |
| 960 | |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 961 | return pgdat; |
| 962 | } |
| 963 | |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 964 | static void rollback_node_hotadd(int nid) |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 965 | { |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 966 | pg_data_t *pgdat = NODE_DATA(nid); |
| 967 | |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 968 | arch_refresh_nodedata(nid, NULL); |
Reza Arbab | 5830169 | 2016-08-11 15:33:12 -0700 | [diff] [blame] | 969 | free_percpu(pgdat->per_cpu_nodestats); |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 970 | arch_free_nodedata(pgdat); |
| 971 | return; |
| 972 | } |
| 973 | |
KAMEZAWA Hiroyuki | 0a54703 | 2006-06-27 02:53:35 -0700 | [diff] [blame] | 974 | |
Toshi Kani | 01b0f19 | 2013-11-12 15:07:25 -0800 | [diff] [blame] | 975 | /** |
| 976 | * try_online_node - online a node if offlined |
Mike Rapoport | e8b098f | 2018-04-05 16:24:57 -0700 | [diff] [blame] | 977 | * @nid: the node ID |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 978 | * @start: start addr of the node |
| 979 | * @set_node_online: Whether we want to online the node |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 980 | * called by cpu_up() to online a node without onlined memory. |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 981 | * |
| 982 | * Returns: |
| 983 | * 1 -> a new node has been allocated |
| 984 | * 0 -> the node is already online |
| 985 | * -ENOMEM -> the node could not be allocated |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 986 | */ |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 987 | static int __try_online_node(int nid, u64 start, bool set_node_online) |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 988 | { |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 989 | pg_data_t *pgdat; |
| 990 | int ret = 1; |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 991 | |
Toshi Kani | 01b0f19 | 2013-11-12 15:07:25 -0800 | [diff] [blame] | 992 | if (node_online(nid)) |
| 993 | return 0; |
| 994 | |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 995 | pgdat = hotadd_new_pgdat(nid, start); |
David Rientjes | 7553e8f | 2011-06-22 18:13:01 -0700 | [diff] [blame] | 996 | if (!pgdat) { |
Toshi Kani | 01b0f19 | 2013-11-12 15:07:25 -0800 | [diff] [blame] | 997 | pr_err("Cannot online node %d due to NULL pgdat\n", nid); |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 998 | ret = -ENOMEM; |
| 999 | goto out; |
| 1000 | } |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 1001 | |
| 1002 | if (set_node_online) { |
| 1003 | node_set_online(nid); |
| 1004 | ret = register_one_node(nid); |
| 1005 | BUG_ON(ret); |
| 1006 | } |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 1007 | out: |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 1008 | return ret; |
| 1009 | } |
| 1010 | |
| 1011 | /* |
| 1012 | * Users of this function always want to online/register the node |
| 1013 | */ |
| 1014 | int try_online_node(int nid) |
| 1015 | { |
| 1016 | int ret; |
| 1017 | |
| 1018 | mem_hotplug_begin(); |
| 1019 | ret = __try_online_node(nid, 0, true); |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 1020 | mem_hotplug_done(); |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 1021 | return ret; |
| 1022 | } |
| 1023 | |
Toshi Kani | 27356f5 | 2013-09-11 14:21:49 -0700 | [diff] [blame] | 1024 | static int check_hotplug_memory_range(u64 start, u64 size) |
| 1025 | { |
Pavel Tatashin | ba32558 | 2018-04-05 16:22:39 -0700 | [diff] [blame] | 1026 | unsigned long block_sz = memory_block_size_bytes(); |
| 1027 | u64 block_nr_pages = block_sz >> PAGE_SHIFT; |
Toshi Kani | 27356f5 | 2013-09-11 14:21:49 -0700 | [diff] [blame] | 1028 | u64 nr_pages = size >> PAGE_SHIFT; |
Pavel Tatashin | ba32558 | 2018-04-05 16:22:39 -0700 | [diff] [blame] | 1029 | u64 start_pfn = PFN_DOWN(start); |
Toshi Kani | 27356f5 | 2013-09-11 14:21:49 -0700 | [diff] [blame] | 1030 | |
Pavel Tatashin | ba32558 | 2018-04-05 16:22:39 -0700 | [diff] [blame] | 1031 | /* memory range must be block size aligned */ |
| 1032 | if (!nr_pages || !IS_ALIGNED(start_pfn, block_nr_pages) || |
| 1033 | !IS_ALIGNED(nr_pages, block_nr_pages)) { |
| 1034 | pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx", |
| 1035 | block_sz, start, size); |
Toshi Kani | 27356f5 | 2013-09-11 14:21:49 -0700 | [diff] [blame] | 1036 | return -EINVAL; |
| 1037 | } |
| 1038 | |
| 1039 | return 0; |
| 1040 | } |
| 1041 | |
Vitaly Kuznetsov | 31bc385 | 2016-03-15 14:56:48 -0700 | [diff] [blame] | 1042 | static int online_memory_block(struct memory_block *mem, void *arg) |
| 1043 | { |
Nathan Fontenot | dc18d70 | 2017-02-24 15:00:02 -0800 | [diff] [blame] | 1044 | return device_online(&mem->dev); |
Vitaly Kuznetsov | 31bc385 | 2016-03-15 14:56:48 -0700 | [diff] [blame] | 1045 | } |
| 1046 | |
David Hildenbrand | 02735d5 | 2018-10-30 15:10:24 -0700 | [diff] [blame] | 1047 | /* |
| 1048 | * NOTE: The caller must call lock_device_hotplug() to serialize hotplug |
| 1049 | * and online/offline operations (triggered e.g. by sysfs). |
| 1050 | * |
| 1051 | * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG |
| 1052 | */ |
Vitaly Kuznetsov | 31bc385 | 2016-03-15 14:56:48 -0700 | [diff] [blame] | 1053 | int __ref add_memory_resource(int nid, struct resource *res, bool online) |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 1054 | { |
David Vrabel | 62cedb9 | 2015-06-25 16:35:49 +0100 | [diff] [blame] | 1055 | u64 start, size; |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 1056 | bool new_node = false; |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 1057 | int ret; |
| 1058 | |
David Vrabel | 62cedb9 | 2015-06-25 16:35:49 +0100 | [diff] [blame] | 1059 | start = res->start; |
| 1060 | size = resource_size(res); |
| 1061 | |
Toshi Kani | 27356f5 | 2013-09-11 14:21:49 -0700 | [diff] [blame] | 1062 | ret = check_hotplug_memory_range(start, size); |
| 1063 | if (ret) |
| 1064 | return ret; |
| 1065 | |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 1066 | mem_hotplug_begin(); |
Nathan Zimmer | ac13c46 | 2014-01-23 15:53:26 -0800 | [diff] [blame] | 1067 | |
Tang Chen | 7f36e3e | 2015-09-04 15:42:32 -0700 | [diff] [blame] | 1068 | /* |
| 1069 | * Add new range to memblock so that when hotadd_new_pgdat() is called |
| 1070 | * to allocate new pgdat, get_pfn_range_for_nid() will be able to find |
| 1071 | * this new range and calculate total pages correctly. The range will |
| 1072 | * be removed at hot-remove time. |
| 1073 | */ |
| 1074 | memblock_add_node(start, size, nid); |
| 1075 | |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 1076 | ret = __try_online_node(nid, start, false); |
| 1077 | if (ret < 0) |
| 1078 | goto error; |
| 1079 | new_node = ret; |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 1080 | |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 1081 | /* call arch's memory hotadd */ |
Christoph Hellwig | 24e6d5a | 2017-12-29 08:53:53 +0100 | [diff] [blame] | 1082 | ret = arch_add_memory(nid, start, size, NULL, true); |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 1083 | if (ret < 0) |
| 1084 | goto error; |
| 1085 | |
David Hildenbrand | aa49b6ab | 2020-01-28 10:50:13 +0100 | [diff] [blame] | 1086 | /* create memory block devices after memory was added */ |
| 1087 | ret = create_memory_block_devices(start, size); |
| 1088 | if (ret) { |
| 1089 | arch_remove_memory(nid, start, size, NULL); |
| 1090 | goto error; |
| 1091 | } |
| 1092 | |
Tang Chen | a1e565a | 2013-02-22 16:33:18 -0800 | [diff] [blame] | 1093 | if (new_node) { |
Oscar Salvador | d5b6f6a3 | 2018-08-17 15:46:18 -0700 | [diff] [blame] | 1094 | /* If sysfs file of new node can't be created, cpu on the node |
Yasunori Goto | 0fc4415 | 2006-06-27 02:53:38 -0700 | [diff] [blame] | 1095 | * can't be hot-added. There is no rollback way now. |
| 1096 | * So, check by BUG_ON() to catch it reluctantly.. |
Oscar Salvador | d5b6f6a3 | 2018-08-17 15:46:18 -0700 | [diff] [blame] | 1097 | * We online node here. We can't roll back from here. |
Yasunori Goto | 0fc4415 | 2006-06-27 02:53:38 -0700 | [diff] [blame] | 1098 | */ |
Oscar Salvador | d5b6f6a3 | 2018-08-17 15:46:18 -0700 | [diff] [blame] | 1099 | node_set_online(nid); |
| 1100 | ret = __register_one_node(nid); |
Yasunori Goto | 0fc4415 | 2006-06-27 02:53:38 -0700 | [diff] [blame] | 1101 | BUG_ON(ret); |
| 1102 | } |
| 1103 | |
Oscar Salvador | d5b6f6a3 | 2018-08-17 15:46:18 -0700 | [diff] [blame] | 1104 | /* link memory sections under this node.*/ |
Laurent Dufour | b6f69f7 | 2020-09-25 21:19:31 -0700 | [diff] [blame] | 1105 | ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1), |
| 1106 | MEMINIT_HOTPLUG); |
Oscar Salvador | d5b6f6a3 | 2018-08-17 15:46:18 -0700 | [diff] [blame] | 1107 | BUG_ON(ret); |
| 1108 | |
akpm@linux-foundation.org | d96ae53 | 2010-03-05 13:41:58 -0800 | [diff] [blame] | 1109 | /* create new memmap entry */ |
| 1110 | firmware_map_add_hotplug(start, start + size, "System RAM"); |
| 1111 | |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1112 | /* device_online() will take the lock when calling online_pages() */ |
| 1113 | mem_hotplug_done(); |
| 1114 | |
Vitaly Kuznetsov | 31bc385 | 2016-03-15 14:56:48 -0700 | [diff] [blame] | 1115 | /* online pages if requested */ |
| 1116 | if (online) |
| 1117 | walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), |
| 1118 | NULL, online_memory_block); |
| 1119 | |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1120 | return ret; |
Yasunori Goto | 9af3c2d | 2006-06-27 02:53:34 -0700 | [diff] [blame] | 1121 | error: |
| 1122 | /* rollback pgdat allocation and others */ |
Oscar Salvador | b9ff036 | 2018-08-17 15:46:15 -0700 | [diff] [blame] | 1123 | if (new_node) |
| 1124 | rollback_node_hotadd(nid); |
Tang Chen | 7f36e3e | 2015-09-04 15:42:32 -0700 | [diff] [blame] | 1125 | memblock_remove(start, size); |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 1126 | mem_hotplug_done(); |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 1127 | return ret; |
| 1128 | } |
David Vrabel | 62cedb9 | 2015-06-25 16:35:49 +0100 | [diff] [blame] | 1129 | |
David Hildenbrand | 02735d5 | 2018-10-30 15:10:24 -0700 | [diff] [blame] | 1130 | /* requires device_hotplug_lock, see add_memory_resource() */ |
| 1131 | int __ref __add_memory(int nid, u64 start, u64 size) |
David Vrabel | 62cedb9 | 2015-06-25 16:35:49 +0100 | [diff] [blame] | 1132 | { |
| 1133 | struct resource *res; |
| 1134 | int ret; |
| 1135 | |
| 1136 | res = register_memory_resource(start, size); |
Vitaly Kuznetsov | 6f754ba | 2016-01-14 15:21:55 -0800 | [diff] [blame] | 1137 | if (IS_ERR(res)) |
| 1138 | return PTR_ERR(res); |
David Vrabel | 62cedb9 | 2015-06-25 16:35:49 +0100 | [diff] [blame] | 1139 | |
Vitaly Kuznetsov | 31bc385 | 2016-03-15 14:56:48 -0700 | [diff] [blame] | 1140 | ret = add_memory_resource(nid, res, memhp_auto_online); |
David Vrabel | 62cedb9 | 2015-06-25 16:35:49 +0100 | [diff] [blame] | 1141 | if (ret < 0) |
| 1142 | release_memory_resource(res); |
| 1143 | return ret; |
| 1144 | } |
David Hildenbrand | 02735d5 | 2018-10-30 15:10:24 -0700 | [diff] [blame] | 1145 | |
| 1146 | int add_memory(int nid, u64 start, u64 size) |
| 1147 | { |
| 1148 | int rc; |
| 1149 | |
| 1150 | lock_device_hotplug(); |
| 1151 | rc = __add_memory(nid, start, size); |
| 1152 | unlock_device_hotplug(); |
| 1153 | |
| 1154 | return rc; |
| 1155 | } |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 1156 | EXPORT_SYMBOL_GPL(add_memory); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1157 | |
| 1158 | #ifdef CONFIG_MEMORY_HOTREMOVE |
| 1159 | /* |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1160 | * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy |
| 1161 | * set and the size of the free page is given by page_order(). Using this, |
| 1162 | * the function determines if the pageblock contains only free pages. |
| 1163 | * Due to buddy contraints, a free page at least the size of a pageblock will |
| 1164 | * be located at the start of the pageblock |
| 1165 | */ |
| 1166 | static inline int pageblock_free(struct page *page) |
| 1167 | { |
| 1168 | return PageBuddy(page) && page_order(page) >= pageblock_order; |
| 1169 | } |
| 1170 | |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1171 | /* Return the pfn of the start of the next active pageblock after a given pfn */ |
| 1172 | static unsigned long next_active_pageblock(unsigned long pfn) |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1173 | { |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1174 | struct page *page = pfn_to_page(pfn); |
| 1175 | |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1176 | /* Ensure the starting page is pageblock-aligned */ |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1177 | BUG_ON(pfn & (pageblock_nr_pages - 1)); |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1178 | |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1179 | /* If the entire pageblock is free, move to the end of free page */ |
KAMEZAWA Hiroyuki | 0dcc48c | 2010-09-09 16:38:01 -0700 | [diff] [blame] | 1180 | if (pageblock_free(page)) { |
| 1181 | int order; |
| 1182 | /* be careful. we don't have locks, page_order can be changed.*/ |
| 1183 | order = page_order(page); |
| 1184 | if ((order < MAX_ORDER) && (order >= pageblock_order)) |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1185 | return pfn + (1 << order); |
KAMEZAWA Hiroyuki | 0dcc48c | 2010-09-09 16:38:01 -0700 | [diff] [blame] | 1186 | } |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1187 | |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1188 | return pfn + pageblock_nr_pages; |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1189 | } |
| 1190 | |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1191 | static bool is_pageblock_removable_nolock(unsigned long pfn) |
Mathieu Malaterre | fb52bba | 2018-06-07 17:07:43 -0700 | [diff] [blame] | 1192 | { |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1193 | struct page *page = pfn_to_page(pfn); |
Mathieu Malaterre | fb52bba | 2018-06-07 17:07:43 -0700 | [diff] [blame] | 1194 | struct zone *zone; |
Mathieu Malaterre | fb52bba | 2018-06-07 17:07:43 -0700 | [diff] [blame] | 1195 | |
| 1196 | /* |
| 1197 | * We have to be careful here because we are iterating over memory |
| 1198 | * sections which are not zone aware so we might end up outside of |
| 1199 | * the zone but still within the section. |
| 1200 | * We have to take care about the node as well. If the node is offline |
| 1201 | * its NODE_DATA will be NULL - see page_zone. |
| 1202 | */ |
| 1203 | if (!node_online(page_to_nid(page))) |
| 1204 | return false; |
| 1205 | |
| 1206 | zone = page_zone(page); |
| 1207 | pfn = page_to_pfn(page); |
| 1208 | if (!zone_spans_pfn(zone, pfn)) |
| 1209 | return false; |
| 1210 | |
| 1211 | return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, true); |
| 1212 | } |
| 1213 | |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1214 | /* Checks if this range of memory is likely to be hot-removable. */ |
Yaowei Bai | c98940f | 2016-05-19 17:11:26 -0700 | [diff] [blame] | 1215 | bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1216 | { |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1217 | unsigned long end_pfn, pfn; |
| 1218 | |
| 1219 | end_pfn = min(start_pfn + nr_pages, |
| 1220 | zone_end_pfn(page_zone(pfn_to_page(start_pfn)))); |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1221 | |
| 1222 | /* Check the starting page of each pageblock within the range */ |
Michal Hocko | 2cc84e2 | 2019-02-20 22:20:46 -0800 | [diff] [blame] | 1223 | for (pfn = start_pfn; pfn < end_pfn; pfn = next_active_pageblock(pfn)) { |
| 1224 | if (!is_pageblock_removable_nolock(pfn)) |
Yaowei Bai | c98940f | 2016-05-19 17:11:26 -0700 | [diff] [blame] | 1225 | return false; |
KAMEZAWA Hiroyuki | 49ac825 | 2010-10-26 14:21:30 -0700 | [diff] [blame] | 1226 | cond_resched(); |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1227 | } |
| 1228 | |
| 1229 | /* All pageblocks in the memory block are likely to be hot-removable */ |
Yaowei Bai | c98940f | 2016-05-19 17:11:26 -0700 | [diff] [blame] | 1230 | return true; |
Badari Pulavarty | 5c755e9 | 2008-07-23 21:28:19 -0700 | [diff] [blame] | 1231 | } |
| 1232 | |
| 1233 | /* |
Toshi Kani | deb88a2 | 2017-02-03 13:13:20 -0800 | [diff] [blame] | 1234 | * Confirm all pages in a range [start, end) belong to the same zone. |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1235 | * When true, return its valid [start, end). |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1236 | */ |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1237 | int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn, |
| 1238 | unsigned long *valid_start, unsigned long *valid_end) |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1239 | { |
Andrew Banman | 5f0f288 | 2015-12-29 14:54:25 -0800 | [diff] [blame] | 1240 | unsigned long pfn, sec_end_pfn; |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1241 | unsigned long start, end; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1242 | struct zone *zone = NULL; |
| 1243 | struct page *page; |
| 1244 | int i; |
Toshi Kani | deb88a2 | 2017-02-03 13:13:20 -0800 | [diff] [blame] | 1245 | for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1246 | pfn < end_pfn; |
Toshi Kani | deb88a2 | 2017-02-03 13:13:20 -0800 | [diff] [blame] | 1247 | pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) { |
Andrew Banman | 5f0f288 | 2015-12-29 14:54:25 -0800 | [diff] [blame] | 1248 | /* Make sure the memory section is present first */ |
| 1249 | if (!present_section_nr(pfn_to_section_nr(pfn))) |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1250 | continue; |
Andrew Banman | 5f0f288 | 2015-12-29 14:54:25 -0800 | [diff] [blame] | 1251 | for (; pfn < sec_end_pfn && pfn < end_pfn; |
| 1252 | pfn += MAX_ORDER_NR_PAGES) { |
| 1253 | i = 0; |
| 1254 | /* This is just a CONFIG_HOLES_IN_ZONE check.*/ |
| 1255 | while ((i < MAX_ORDER_NR_PAGES) && |
| 1256 | !pfn_valid_within(pfn + i)) |
| 1257 | i++; |
zhong jiang | d6d8c8a | 2017-02-24 14:59:30 -0800 | [diff] [blame] | 1258 | if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn) |
Andrew Banman | 5f0f288 | 2015-12-29 14:54:25 -0800 | [diff] [blame] | 1259 | continue; |
Mikhail Zaslonko | 71df1c8 | 2019-02-01 14:20:38 -0800 | [diff] [blame] | 1260 | /* Check if we got outside of the zone */ |
| 1261 | if (zone && !zone_spans_pfn(zone, pfn + i)) |
| 1262 | return 0; |
Andrew Banman | 5f0f288 | 2015-12-29 14:54:25 -0800 | [diff] [blame] | 1263 | page = pfn_to_page(pfn + i); |
| 1264 | if (zone && page_zone(page) != zone) |
| 1265 | return 0; |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1266 | if (!zone) |
| 1267 | start = pfn + i; |
Andrew Banman | 5f0f288 | 2015-12-29 14:54:25 -0800 | [diff] [blame] | 1268 | zone = page_zone(page); |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1269 | end = pfn + MAX_ORDER_NR_PAGES; |
Andrew Banman | 5f0f288 | 2015-12-29 14:54:25 -0800 | [diff] [blame] | 1270 | } |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1271 | } |
Toshi Kani | deb88a2 | 2017-02-03 13:13:20 -0800 | [diff] [blame] | 1272 | |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1273 | if (zone) { |
| 1274 | *valid_start = start; |
zhong jiang | d6d8c8a | 2017-02-24 14:59:30 -0800 | [diff] [blame] | 1275 | *valid_end = min(end, end_pfn); |
Toshi Kani | deb88a2 | 2017-02-03 13:13:20 -0800 | [diff] [blame] | 1276 | return 1; |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1277 | } else { |
Toshi Kani | deb88a2 | 2017-02-03 13:13:20 -0800 | [diff] [blame] | 1278 | return 0; |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1279 | } |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1280 | } |
| 1281 | |
| 1282 | /* |
Yisheng Xie | 0efadf4 | 2017-02-24 14:57:39 -0800 | [diff] [blame] | 1283 | * Scan pfn range [start,end) to find movable/migratable pages (LRU pages, |
| 1284 | * non-lru movable pages and hugepages). We scan pfn because it's much |
| 1285 | * easier than scanning over linked list. This function returns the pfn |
| 1286 | * of the first found movable page if it's found, otherwise 0. |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1287 | */ |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 1288 | static unsigned long scan_movable_pages(unsigned long start, unsigned long end) |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1289 | { |
| 1290 | unsigned long pfn; |
Oscar Salvador | d9f4d88 | 2019-02-01 14:20:47 -0800 | [diff] [blame] | 1291 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1292 | for (pfn = start; pfn < end; pfn++) { |
Oscar Salvador | d9f4d88 | 2019-02-01 14:20:47 -0800 | [diff] [blame] | 1293 | struct page *page, *head; |
| 1294 | unsigned long skip; |
| 1295 | |
| 1296 | if (!pfn_valid(pfn)) |
| 1297 | continue; |
| 1298 | page = pfn_to_page(pfn); |
| 1299 | if (PageLRU(page)) |
| 1300 | return pfn; |
| 1301 | if (__PageMovable(page)) |
| 1302 | return pfn; |
| 1303 | |
| 1304 | if (!PageHuge(page)) |
| 1305 | continue; |
| 1306 | head = compound_head(page); |
| 1307 | if (hugepage_migration_supported(page_hstate(head)) && |
| 1308 | page_huge_active(head)) |
| 1309 | return pfn; |
| 1310 | skip = (1 << compound_order(head)) - (page - head); |
| 1311 | pfn += skip - 1; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1312 | } |
| 1313 | return 0; |
| 1314 | } |
| 1315 | |
Michal Hocko | 666feb2 | 2018-04-10 16:30:03 -0700 | [diff] [blame] | 1316 | static struct page *new_node_page(struct page *page, unsigned long private) |
Xishi Qiu | 394e31d | 2016-07-28 15:48:53 -0700 | [diff] [blame] | 1317 | { |
Xishi Qiu | 394e31d | 2016-07-28 15:48:53 -0700 | [diff] [blame] | 1318 | int nid = page_to_nid(page); |
Li Zhong | 231e97e | 2016-09-28 15:22:38 -0700 | [diff] [blame] | 1319 | nodemask_t nmask = node_states[N_MEMORY]; |
Michal Hocko | 7f252f2 | 2017-07-10 15:48:41 -0700 | [diff] [blame] | 1320 | |
| 1321 | /* |
| 1322 | * try to allocate from a different node but reuse this node if there |
| 1323 | * are no other online nodes to be used (e.g. we are offlining a part |
| 1324 | * of the only existing node) |
| 1325 | */ |
| 1326 | node_clear(nid, nmask); |
| 1327 | if (nodes_empty(nmask)) |
| 1328 | node_set(nid, nmask); |
Xishi Qiu | 394e31d | 2016-07-28 15:48:53 -0700 | [diff] [blame] | 1329 | |
Michal Hocko | 8b91323 | 2017-07-10 15:48:47 -0700 | [diff] [blame] | 1330 | return new_page_nodemask(page, nid, &nmask); |
Xishi Qiu | 394e31d | 2016-07-28 15:48:53 -0700 | [diff] [blame] | 1331 | } |
| 1332 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1333 | #define NR_OFFLINE_AT_ONCE_PAGES (256) |
| 1334 | static int |
| 1335 | do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) |
| 1336 | { |
| 1337 | unsigned long pfn; |
| 1338 | struct page *page; |
| 1339 | int move_pages = NR_OFFLINE_AT_ONCE_PAGES; |
| 1340 | int not_managed = 0; |
| 1341 | int ret = 0; |
| 1342 | LIST_HEAD(source); |
| 1343 | |
| 1344 | for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) { |
| 1345 | if (!pfn_valid(pfn)) |
| 1346 | continue; |
| 1347 | page = pfn_to_page(pfn); |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 1348 | |
| 1349 | if (PageHuge(page)) { |
| 1350 | struct page *head = compound_head(page); |
| 1351 | pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1; |
| 1352 | if (compound_order(head) > PFN_SECTION_SHIFT) { |
| 1353 | ret = -EBUSY; |
| 1354 | break; |
| 1355 | } |
| 1356 | if (isolate_huge_page(page, &source)) |
| 1357 | move_pages -= 1 << compound_order(head); |
| 1358 | continue; |
Michal Hocko | 94723aa | 2018-04-10 16:30:07 -0700 | [diff] [blame] | 1359 | } else if (PageTransHuge(page)) |
Naoya Horiguchi | 8135d89 | 2017-09-08 16:11:15 -0700 | [diff] [blame] | 1360 | pfn = page_to_pfn(compound_head(page)) |
| 1361 | + hpage_nr_pages(page) - 1; |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 1362 | |
Michal Hocko | 2c87072 | 2018-12-28 00:38:01 -0800 | [diff] [blame] | 1363 | /* |
| 1364 | * HWPoison pages have elevated reference counts so the migration would |
| 1365 | * fail on them. It also doesn't make any sense to migrate them in the |
| 1366 | * first place. Still try to unmap such a page in case it is still mapped |
| 1367 | * (e.g. current hwpoison implementation doesn't unmap KSM pages but keep |
| 1368 | * the unmap as the catch all safety net). |
| 1369 | */ |
| 1370 | if (PageHWPoison(page)) { |
| 1371 | if (WARN_ON(PageLRU(page))) |
| 1372 | isolate_lru_page(page); |
| 1373 | if (page_mapped(page)) |
| 1374 | try_to_unmap(page, TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS); |
| 1375 | continue; |
| 1376 | } |
| 1377 | |
Konstantin Khlebnikov | 700c2a4 | 2011-05-24 17:12:19 -0700 | [diff] [blame] | 1378 | if (!get_page_unless_zero(page)) |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1379 | continue; |
| 1380 | /* |
Yisheng Xie | 0efadf4 | 2017-02-24 14:57:39 -0800 | [diff] [blame] | 1381 | * We can skip free pages. And we can deal with pages on |
| 1382 | * LRU and non-lru movable pages. |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1383 | */ |
Yisheng Xie | 0efadf4 | 2017-02-24 14:57:39 -0800 | [diff] [blame] | 1384 | if (PageLRU(page)) |
| 1385 | ret = isolate_lru_page(page); |
| 1386 | else |
| 1387 | ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1388 | if (!ret) { /* Success */ |
Konstantin Khlebnikov | 700c2a4 | 2011-05-24 17:12:19 -0700 | [diff] [blame] | 1389 | put_page(page); |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 1390 | list_add_tail(&page->lru, &source); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1391 | move_pages--; |
Yisheng Xie | 0efadf4 | 2017-02-24 14:57:39 -0800 | [diff] [blame] | 1392 | if (!__PageMovable(page)) |
| 1393 | inc_node_page_state(page, NR_ISOLATED_ANON + |
| 1394 | page_is_file_cache(page)); |
KOSAKI Motohiro | 6d9c285 | 2009-12-14 17:58:11 -0800 | [diff] [blame] | 1395 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1396 | } else { |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1397 | #ifdef CONFIG_DEBUG_VM |
Yisheng Xie | 0efadf4 | 2017-02-24 14:57:39 -0800 | [diff] [blame] | 1398 | pr_alert("failed to isolate pfn %lx\n", pfn); |
| 1399 | dump_page(page, "isolation failed"); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1400 | #endif |
Konstantin Khlebnikov | 700c2a4 | 2011-05-24 17:12:19 -0700 | [diff] [blame] | 1401 | put_page(page); |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 1402 | /* Because we don't have big zone->lock. we should |
Bob Liu | 809c444 | 2010-10-26 14:22:10 -0700 | [diff] [blame] | 1403 | check this again here. */ |
| 1404 | if (page_count(page)) { |
| 1405 | not_managed++; |
Bob Liu | f3ab263 | 2010-10-26 14:22:10 -0700 | [diff] [blame] | 1406 | ret = -EBUSY; |
Bob Liu | 809c444 | 2010-10-26 14:22:10 -0700 | [diff] [blame] | 1407 | break; |
| 1408 | } |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1409 | } |
| 1410 | } |
Bob Liu | f3ab263 | 2010-10-26 14:22:10 -0700 | [diff] [blame] | 1411 | if (!list_empty(&source)) { |
| 1412 | if (not_managed) { |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 1413 | putback_movable_pages(&source); |
Bob Liu | f3ab263 | 2010-10-26 14:22:10 -0700 | [diff] [blame] | 1414 | goto out; |
| 1415 | } |
Minchan Kim | 74c08f9 | 2012-10-08 16:32:54 -0700 | [diff] [blame] | 1416 | |
Xishi Qiu | 394e31d | 2016-07-28 15:48:53 -0700 | [diff] [blame] | 1417 | /* Allocate a new page from the nearest neighbor node */ |
| 1418 | ret = migrate_pages(&source, new_node_page, NULL, 0, |
Hugh Dickins | 9c620e2 | 2013-02-22 16:35:14 -0800 | [diff] [blame] | 1419 | MIGRATE_SYNC, MR_MEMORY_HOTPLUG); |
Bob Liu | f3ab263 | 2010-10-26 14:22:10 -0700 | [diff] [blame] | 1420 | if (ret) |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 1421 | putback_movable_pages(&source); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1422 | } |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1423 | out: |
| 1424 | return ret; |
| 1425 | } |
| 1426 | |
| 1427 | /* |
| 1428 | * remove from free_area[] and mark all as Reserved. |
| 1429 | */ |
| 1430 | static int |
| 1431 | offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, |
| 1432 | void *data) |
| 1433 | { |
| 1434 | __offline_isolated_pages(start, start + nr_pages); |
| 1435 | return 0; |
| 1436 | } |
| 1437 | |
| 1438 | static void |
| 1439 | offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) |
| 1440 | { |
KAMEZAWA Hiroyuki | 908eedc | 2009-09-22 16:45:46 -0700 | [diff] [blame] | 1441 | walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL, |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1442 | offline_isolated_pages_cb); |
| 1443 | } |
| 1444 | |
| 1445 | /* |
| 1446 | * Check all pages in range, recoreded as memory resource, are isolated. |
| 1447 | */ |
| 1448 | static int |
| 1449 | check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, |
| 1450 | void *data) |
| 1451 | { |
| 1452 | int ret; |
| 1453 | long offlined = *(long *)data; |
Wen Congyang | b023f46 | 2012-12-11 16:00:45 -0800 | [diff] [blame] | 1454 | ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1455 | offlined = nr_pages; |
| 1456 | if (!ret) |
| 1457 | *(long *)data += offlined; |
| 1458 | return ret; |
| 1459 | } |
| 1460 | |
| 1461 | static long |
| 1462 | check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) |
| 1463 | { |
| 1464 | long offlined = 0; |
| 1465 | int ret; |
| 1466 | |
KAMEZAWA Hiroyuki | 908eedc | 2009-09-22 16:45:46 -0700 | [diff] [blame] | 1467 | ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined, |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1468 | check_pages_isolated_cb); |
| 1469 | if (ret < 0) |
| 1470 | offlined = (long)ret; |
| 1471 | return offlined; |
| 1472 | } |
| 1473 | |
Tang Chen | c532092 | 2013-11-12 15:08:10 -0800 | [diff] [blame] | 1474 | static int __init cmdline_parse_movable_node(char *p) |
| 1475 | { |
Michal Hocko | 4932381 | 2017-07-06 15:41:05 -0700 | [diff] [blame] | 1476 | #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP |
Tang Chen | 55ac590 | 2014-01-21 15:49:35 -0800 | [diff] [blame] | 1477 | movable_node_enabled = true; |
Michal Hocko | 4932381 | 2017-07-06 15:41:05 -0700 | [diff] [blame] | 1478 | #else |
| 1479 | pr_warn("movable_node parameter depends on CONFIG_HAVE_MEMBLOCK_NODE_MAP to work properly\n"); |
| 1480 | #endif |
Tang Chen | c532092 | 2013-11-12 15:08:10 -0800 | [diff] [blame] | 1481 | return 0; |
| 1482 | } |
| 1483 | early_param("movable_node", cmdline_parse_movable_node); |
| 1484 | |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1485 | /* check which state of node_states will be changed when offline memory */ |
| 1486 | static void node_states_check_changes_offline(unsigned long nr_pages, |
| 1487 | struct zone *zone, struct memory_notify *arg) |
| 1488 | { |
| 1489 | struct pglist_data *pgdat = zone->zone_pgdat; |
| 1490 | unsigned long present_pages = 0; |
| 1491 | enum zone_type zt, zone_last = ZONE_NORMAL; |
| 1492 | |
| 1493 | /* |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 1494 | * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] |
| 1495 | * contains nodes which have zones of 0...ZONE_NORMAL, |
| 1496 | * set zone_last to ZONE_NORMAL. |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1497 | * |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 1498 | * If we don't have HIGHMEM nor movable node, |
| 1499 | * node_states[N_NORMAL_MEMORY] contains nodes which have zones of |
| 1500 | * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1501 | */ |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 1502 | if (N_MEMORY == N_NORMAL_MEMORY) |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1503 | zone_last = ZONE_MOVABLE; |
| 1504 | |
| 1505 | /* |
| 1506 | * check whether node_states[N_NORMAL_MEMORY] will be changed. |
| 1507 | * If the memory to be offline is in a zone of 0...zone_last, |
| 1508 | * and it is the last present memory, 0...zone_last will |
| 1509 | * become empty after offline , thus we can determind we will |
| 1510 | * need to clear the node from node_states[N_NORMAL_MEMORY]. |
| 1511 | */ |
| 1512 | for (zt = 0; zt <= zone_last; zt++) |
| 1513 | present_pages += pgdat->node_zones[zt].present_pages; |
| 1514 | if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) |
| 1515 | arg->status_change_nid_normal = zone_to_nid(zone); |
| 1516 | else |
| 1517 | arg->status_change_nid_normal = -1; |
| 1518 | |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 1519 | #ifdef CONFIG_HIGHMEM |
| 1520 | /* |
| 1521 | * If we have movable node, node_states[N_HIGH_MEMORY] |
| 1522 | * contains nodes which have zones of 0...ZONE_HIGHMEM, |
| 1523 | * set zone_last to ZONE_HIGHMEM. |
| 1524 | * |
| 1525 | * If we don't have movable node, node_states[N_NORMAL_MEMORY] |
| 1526 | * contains nodes which have zones of 0...ZONE_MOVABLE, |
| 1527 | * set zone_last to ZONE_MOVABLE. |
| 1528 | */ |
| 1529 | zone_last = ZONE_HIGHMEM; |
| 1530 | if (N_MEMORY == N_HIGH_MEMORY) |
| 1531 | zone_last = ZONE_MOVABLE; |
| 1532 | |
| 1533 | for (; zt <= zone_last; zt++) |
| 1534 | present_pages += pgdat->node_zones[zt].present_pages; |
| 1535 | if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) |
| 1536 | arg->status_change_nid_high = zone_to_nid(zone); |
| 1537 | else |
| 1538 | arg->status_change_nid_high = -1; |
| 1539 | #else |
| 1540 | arg->status_change_nid_high = arg->status_change_nid_normal; |
| 1541 | #endif |
| 1542 | |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1543 | /* |
| 1544 | * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE |
| 1545 | */ |
| 1546 | zone_last = ZONE_MOVABLE; |
| 1547 | |
| 1548 | /* |
| 1549 | * check whether node_states[N_HIGH_MEMORY] will be changed |
| 1550 | * If we try to offline the last present @nr_pages from the node, |
| 1551 | * we can determind we will need to clear the node from |
| 1552 | * node_states[N_HIGH_MEMORY]. |
| 1553 | */ |
| 1554 | for (; zt <= zone_last; zt++) |
| 1555 | present_pages += pgdat->node_zones[zt].present_pages; |
| 1556 | if (nr_pages >= present_pages) |
| 1557 | arg->status_change_nid = zone_to_nid(zone); |
| 1558 | else |
| 1559 | arg->status_change_nid = -1; |
| 1560 | } |
| 1561 | |
| 1562 | static void node_states_clear_node(int node, struct memory_notify *arg) |
| 1563 | { |
| 1564 | if (arg->status_change_nid_normal >= 0) |
| 1565 | node_clear_state(node, N_NORMAL_MEMORY); |
| 1566 | |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 1567 | if ((N_MEMORY != N_NORMAL_MEMORY) && |
| 1568 | (arg->status_change_nid_high >= 0)) |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1569 | node_clear_state(node, N_HIGH_MEMORY); |
Lai Jiangshan | 6715ddf | 2012-12-12 13:51:49 -0800 | [diff] [blame] | 1570 | |
| 1571 | if ((N_MEMORY != N_HIGH_MEMORY) && |
| 1572 | (arg->status_change_nid >= 0)) |
| 1573 | node_clear_state(node, N_MEMORY); |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1574 | } |
| 1575 | |
Wen Congyang | a16cee1 | 2012-10-08 16:33:58 -0700 | [diff] [blame] | 1576 | static int __ref __offline_pages(unsigned long start_pfn, |
Michal Hocko | ecde0f3 | 2017-11-15 17:33:38 -0800 | [diff] [blame] | 1577 | unsigned long end_pfn) |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1578 | { |
Michal Hocko | ecde0f3 | 2017-11-15 17:33:38 -0800 | [diff] [blame] | 1579 | unsigned long pfn, nr_pages; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1580 | long offlined_pages; |
Michal Hocko | 72b39cf | 2017-11-15 17:33:34 -0800 | [diff] [blame] | 1581 | int ret, node; |
Cody P Schafer | d702909 | 2013-07-03 15:02:11 -0700 | [diff] [blame] | 1582 | unsigned long flags; |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1583 | unsigned long valid_start, valid_end; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1584 | struct zone *zone; |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1585 | struct memory_notify arg; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1586 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1587 | /* at least, alignment against pageblock is necessary */ |
| 1588 | if (!IS_ALIGNED(start_pfn, pageblock_nr_pages)) |
| 1589 | return -EINVAL; |
| 1590 | if (!IS_ALIGNED(end_pfn, pageblock_nr_pages)) |
| 1591 | return -EINVAL; |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1592 | |
| 1593 | mem_hotplug_begin(); |
| 1594 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1595 | /* This makes hotplug much easier...and readable. |
| 1596 | we assume this for now. .*/ |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1597 | if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start, |
| 1598 | &valid_end)) { |
| 1599 | mem_hotplug_done(); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1600 | return -EINVAL; |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1601 | } |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1602 | |
Toshi Kani | a96dfdd | 2017-02-03 13:13:23 -0800 | [diff] [blame] | 1603 | zone = page_zone(pfn_to_page(valid_start)); |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1604 | node = zone_to_nid(zone); |
| 1605 | nr_pages = end_pfn - start_pfn; |
| 1606 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1607 | /* set above range as isolated */ |
Wen Congyang | b023f46 | 2012-12-11 16:00:45 -0800 | [diff] [blame] | 1608 | ret = start_isolate_page_range(start_pfn, end_pfn, |
| 1609 | MIGRATE_MOVABLE, true); |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1610 | if (ret) { |
| 1611 | mem_hotplug_done(); |
David Rientjes | 30467e0 | 2015-04-14 15:45:11 -0700 | [diff] [blame] | 1612 | return ret; |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1613 | } |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1614 | |
| 1615 | arg.start_pfn = start_pfn; |
| 1616 | arg.nr_pages = nr_pages; |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1617 | node_states_check_changes_offline(nr_pages, zone, &arg); |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1618 | |
| 1619 | ret = memory_notify(MEM_GOING_OFFLINE, &arg); |
| 1620 | ret = notifier_to_errno(ret); |
| 1621 | if (ret) |
| 1622 | goto failed_removal; |
| 1623 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1624 | pfn = start_pfn; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1625 | repeat: |
| 1626 | /* start memory hot removal */ |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1627 | ret = -EINTR; |
| 1628 | if (signal_pending(current)) |
| 1629 | goto failed_removal; |
Michal Hocko | 72b39cf | 2017-11-15 17:33:34 -0800 | [diff] [blame] | 1630 | |
| 1631 | cond_resched(); |
Michal Hocko | 9852a72 | 2018-01-31 16:16:19 -0800 | [diff] [blame] | 1632 | lru_add_drain_all(); |
Michal Hocko | 72b39cf | 2017-11-15 17:33:34 -0800 | [diff] [blame] | 1633 | drain_all_pages(zone); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1634 | |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 1635 | pfn = scan_movable_pages(start_pfn, end_pfn); |
| 1636 | if (pfn) { /* We have movable pages */ |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1637 | ret = do_migrate_range(pfn, end_pfn); |
Michal Hocko | 72b39cf | 2017-11-15 17:33:34 -0800 | [diff] [blame] | 1638 | goto repeat; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1639 | } |
Michal Hocko | 72b39cf | 2017-11-15 17:33:34 -0800 | [diff] [blame] | 1640 | |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 1641 | /* |
| 1642 | * dissolve free hugepages in the memory block before doing offlining |
| 1643 | * actually in order to make hugetlbfs's object counting consistent. |
| 1644 | */ |
Gerald Schaefer | 082d5b6 | 2016-10-07 17:01:10 -0700 | [diff] [blame] | 1645 | ret = dissolve_free_huge_pages(start_pfn, end_pfn); |
| 1646 | if (ret) |
| 1647 | goto failed_removal; |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1648 | /* check again */ |
| 1649 | offlined_pages = check_pages_isolated(start_pfn, end_pfn); |
Michal Hocko | 72b39cf | 2017-11-15 17:33:34 -0800 | [diff] [blame] | 1650 | if (offlined_pages < 0) |
| 1651 | goto repeat; |
Chen Yucong | e33e33b | 2016-03-17 14:19:35 -0700 | [diff] [blame] | 1652 | pr_info("Offlined Pages %ld\n", offlined_pages); |
Adam Buchbinder | b3834be | 2012-09-19 21:48:02 -0400 | [diff] [blame] | 1653 | /* Ok, all of our target is isolated. |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1654 | We cannot do rollback at this point. */ |
| 1655 | offline_isolated_pages(start_pfn, end_pfn); |
KAMEZAWA Hiroyuki | dbc0e4c | 2007-11-14 16:59:12 -0800 | [diff] [blame] | 1656 | /* reset pagetype flags and makes migrate type to be MOVABLE */ |
Michal Nazarewicz | 0815f3d | 2012-04-03 15:06:15 +0200 | [diff] [blame] | 1657 | undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1658 | /* removal success */ |
Jiang Liu | 3dcc057 | 2013-07-03 15:03:21 -0700 | [diff] [blame] | 1659 | adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1660 | zone->present_pages -= offlined_pages; |
Cody P Schafer | d702909 | 2013-07-03 15:02:11 -0700 | [diff] [blame] | 1661 | |
| 1662 | pgdat_resize_lock(zone->zone_pgdat, &flags); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1663 | zone->zone_pgdat->node_present_pages -= offlined_pages; |
Cody P Schafer | d702909 | 2013-07-03 15:02:11 -0700 | [diff] [blame] | 1664 | pgdat_resize_unlock(zone->zone_pgdat, &flags); |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1665 | |
KOSAKI Motohiro | 1b79acc | 2011-05-24 17:11:32 -0700 | [diff] [blame] | 1666 | init_per_zone_wmark_min(); |
| 1667 | |
Xishi Qiu | 1e8537b | 2012-10-08 16:31:51 -0700 | [diff] [blame] | 1668 | if (!populated_zone(zone)) { |
Jiang Liu | 340175b | 2012-07-31 16:43:32 -0700 | [diff] [blame] | 1669 | zone_pcp_reset(zone); |
Michal Hocko | 72675e1 | 2017-09-06 16:20:24 -0700 | [diff] [blame] | 1670 | build_all_zonelists(NULL); |
Xishi Qiu | 1e8537b | 2012-10-08 16:31:51 -0700 | [diff] [blame] | 1671 | } else |
| 1672 | zone_pcp_update(zone); |
Jiang Liu | 340175b | 2012-07-31 16:43:32 -0700 | [diff] [blame] | 1673 | |
Lai Jiangshan | d971367 | 2012-12-11 16:01:03 -0800 | [diff] [blame] | 1674 | node_states_clear_node(node, &arg); |
Vlastimil Babka | 698b1b3 | 2016-03-17 14:18:08 -0700 | [diff] [blame] | 1675 | if (arg.status_change_nid >= 0) { |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 1676 | kswapd_stop(node); |
Vlastimil Babka | 698b1b3 | 2016-03-17 14:18:08 -0700 | [diff] [blame] | 1677 | kcompactd_stop(node); |
| 1678 | } |
Minchan Kim | bce7394 | 2009-06-16 15:32:50 -0700 | [diff] [blame] | 1679 | |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1680 | vm_total_pages = nr_free_pagecache_pages(); |
| 1681 | writeback_set_ratelimit(); |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1682 | |
| 1683 | memory_notify(MEM_OFFLINE, &arg); |
David Hildenbrand | 8683489 | 2020-01-28 10:50:21 +0100 | [diff] [blame] | 1684 | remove_pfn_range_from_zone(zone, start_pfn, nr_pages); |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1685 | mem_hotplug_done(); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1686 | return 0; |
| 1687 | |
| 1688 | failed_removal: |
Chen Yucong | e33e33b | 2016-03-17 14:19:35 -0700 | [diff] [blame] | 1689 | pr_debug("memory offlining [mem %#010llx-%#010llx] failed\n", |
| 1690 | (unsigned long long) start_pfn << PAGE_SHIFT, |
| 1691 | ((unsigned long long) end_pfn << PAGE_SHIFT) - 1); |
Yasunori Goto | 7b78d33 | 2007-10-21 16:41:36 -0700 | [diff] [blame] | 1692 | memory_notify(MEM_CANCEL_OFFLINE, &arg); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1693 | /* pushback to free area */ |
Michal Nazarewicz | 0815f3d | 2012-04-03 15:06:15 +0200 | [diff] [blame] | 1694 | undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); |
David Hildenbrand | 17523d7 | 2018-10-30 15:10:29 -0700 | [diff] [blame] | 1695 | mem_hotplug_done(); |
KAMEZAWA Hiroyuki | 0c0e619 | 2007-10-16 01:26:12 -0700 | [diff] [blame] | 1696 | return ret; |
| 1697 | } |
Badari Pulavarty | 7108878 | 2008-10-18 20:25:58 -0700 | [diff] [blame] | 1698 | |
Wen Congyang | a16cee1 | 2012-10-08 16:33:58 -0700 | [diff] [blame] | 1699 | int offline_pages(unsigned long start_pfn, unsigned long nr_pages) |
| 1700 | { |
Michal Hocko | ecde0f3 | 2017-11-15 17:33:38 -0800 | [diff] [blame] | 1701 | return __offline_pages(start_pfn, start_pfn + nr_pages); |
Wen Congyang | a16cee1 | 2012-10-08 16:33:58 -0700 | [diff] [blame] | 1702 | } |
Rafael J. Wysocki | e2ff394 | 2013-05-08 00:29:49 +0200 | [diff] [blame] | 1703 | #endif /* CONFIG_MEMORY_HOTREMOVE */ |
Wen Congyang | a16cee1 | 2012-10-08 16:33:58 -0700 | [diff] [blame] | 1704 | |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1705 | /** |
| 1706 | * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn) |
| 1707 | * @start_pfn: start pfn of the memory range |
Toshi Kani | e05c4bb | 2013-04-29 15:06:16 -0700 | [diff] [blame] | 1708 | * @end_pfn: end pfn of the memory range |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1709 | * @arg: argument passed to func |
| 1710 | * @func: callback for each memory section walked |
| 1711 | * |
| 1712 | * This function walks through all present mem sections in range |
| 1713 | * [start_pfn, end_pfn) and call func on each mem section. |
| 1714 | * |
| 1715 | * Returns the return value of func. |
| 1716 | */ |
Rafael J. Wysocki | e2ff394 | 2013-05-08 00:29:49 +0200 | [diff] [blame] | 1717 | int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1718 | void *arg, int (*func)(struct memory_block *, void *)) |
Badari Pulavarty | 7108878 | 2008-10-18 20:25:58 -0700 | [diff] [blame] | 1719 | { |
Wen Congyang | e90bdb7 | 2012-10-08 16:34:01 -0700 | [diff] [blame] | 1720 | struct memory_block *mem = NULL; |
| 1721 | struct mem_section *section; |
Wen Congyang | e90bdb7 | 2012-10-08 16:34:01 -0700 | [diff] [blame] | 1722 | unsigned long pfn, section_nr; |
| 1723 | int ret; |
Badari Pulavarty | 7108878 | 2008-10-18 20:25:58 -0700 | [diff] [blame] | 1724 | |
Wen Congyang | e90bdb7 | 2012-10-08 16:34:01 -0700 | [diff] [blame] | 1725 | for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { |
| 1726 | section_nr = pfn_to_section_nr(pfn); |
| 1727 | if (!present_section_nr(section_nr)) |
| 1728 | continue; |
| 1729 | |
| 1730 | section = __nr_to_section(section_nr); |
| 1731 | /* same memblock? */ |
| 1732 | if (mem) |
| 1733 | if ((section_nr >= mem->start_section_nr) && |
| 1734 | (section_nr <= mem->end_section_nr)) |
| 1735 | continue; |
| 1736 | |
| 1737 | mem = find_memory_block_hinted(section, mem); |
| 1738 | if (!mem) |
| 1739 | continue; |
| 1740 | |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1741 | ret = func(mem, arg); |
Wen Congyang | e90bdb7 | 2012-10-08 16:34:01 -0700 | [diff] [blame] | 1742 | if (ret) { |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1743 | kobject_put(&mem->dev.kobj); |
| 1744 | return ret; |
Wen Congyang | e90bdb7 | 2012-10-08 16:34:01 -0700 | [diff] [blame] | 1745 | } |
| 1746 | } |
| 1747 | |
| 1748 | if (mem) |
| 1749 | kobject_put(&mem->dev.kobj); |
| 1750 | |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1751 | return 0; |
| 1752 | } |
| 1753 | |
Rafael J. Wysocki | e2ff394 | 2013-05-08 00:29:49 +0200 | [diff] [blame] | 1754 | #ifdef CONFIG_MEMORY_HOTREMOVE |
Xishi Qiu | d6de9d5 | 2013-11-12 15:07:20 -0800 | [diff] [blame] | 1755 | static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1756 | { |
| 1757 | int ret = !is_memblock_offlined(mem); |
| 1758 | |
Randy Dunlap | 349daa0 | 2013-04-29 15:08:49 -0700 | [diff] [blame] | 1759 | if (unlikely(ret)) { |
| 1760 | phys_addr_t beginpa, endpa; |
| 1761 | |
| 1762 | beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); |
| 1763 | endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1; |
Joe Perches | 756a025 | 2016-03-17 14:19:47 -0700 | [diff] [blame] | 1764 | pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n", |
Randy Dunlap | 349daa0 | 2013-04-29 15:08:49 -0700 | [diff] [blame] | 1765 | &beginpa, &endpa); |
| 1766 | } |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1767 | |
| 1768 | return ret; |
| 1769 | } |
| 1770 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1771 | static int check_cpu_on_node(pg_data_t *pgdat) |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1772 | { |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1773 | int cpu; |
| 1774 | |
| 1775 | for_each_present_cpu(cpu) { |
| 1776 | if (cpu_to_node(cpu) == pgdat->node_id) |
| 1777 | /* |
| 1778 | * the cpu on this node isn't removed, and we can't |
| 1779 | * offline this node. |
| 1780 | */ |
| 1781 | return -EBUSY; |
| 1782 | } |
| 1783 | |
| 1784 | return 0; |
| 1785 | } |
| 1786 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1787 | static void unmap_cpu_on_node(pg_data_t *pgdat) |
Wen Congyang | e13fe86 | 2013-02-22 16:33:31 -0800 | [diff] [blame] | 1788 | { |
| 1789 | #ifdef CONFIG_ACPI_NUMA |
Wen Congyang | e13fe86 | 2013-02-22 16:33:31 -0800 | [diff] [blame] | 1790 | int cpu; |
| 1791 | |
| 1792 | for_each_possible_cpu(cpu) |
| 1793 | if (cpu_to_node(cpu) == pgdat->node_id) |
| 1794 | numa_clear_node(cpu); |
| 1795 | #endif |
| 1796 | } |
| 1797 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1798 | static int check_and_unmap_cpu_on_node(pg_data_t *pgdat) |
Wen Congyang | e13fe86 | 2013-02-22 16:33:31 -0800 | [diff] [blame] | 1799 | { |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1800 | int ret; |
Wen Congyang | e13fe86 | 2013-02-22 16:33:31 -0800 | [diff] [blame] | 1801 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1802 | ret = check_cpu_on_node(pgdat); |
Wen Congyang | e13fe86 | 2013-02-22 16:33:31 -0800 | [diff] [blame] | 1803 | if (ret) |
| 1804 | return ret; |
| 1805 | |
| 1806 | /* |
| 1807 | * the node will be offlined when we come here, so we can clear |
| 1808 | * the cpu_to_node() now. |
| 1809 | */ |
| 1810 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1811 | unmap_cpu_on_node(pgdat); |
Wen Congyang | e13fe86 | 2013-02-22 16:33:31 -0800 | [diff] [blame] | 1812 | return 0; |
| 1813 | } |
| 1814 | |
David Hildenbrand | d98d053 | 2020-01-28 10:50:20 +0100 | [diff] [blame] | 1815 | static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) |
| 1816 | { |
| 1817 | int nid = *(int *)arg; |
| 1818 | |
| 1819 | /* |
| 1820 | * If a memory block belongs to multiple nodes, the stored nid is not |
| 1821 | * reliable. However, such blocks are always online (e.g., cannot get |
| 1822 | * offlined) and, therefore, are still spanned by the node. |
| 1823 | */ |
| 1824 | return mem->nid == nid ? -EEXIST : 0; |
| 1825 | } |
| 1826 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1827 | /** |
| 1828 | * try_offline_node |
Mike Rapoport | e8b098f | 2018-04-05 16:24:57 -0700 | [diff] [blame] | 1829 | * @nid: the node ID |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1830 | * |
| 1831 | * Offline a node if all memory sections and cpus of the node are removed. |
| 1832 | * |
| 1833 | * NOTE: The caller must call lock_device_hotplug() to serialize hotplug |
| 1834 | * and online/offline operations before this call. |
| 1835 | */ |
Wen Congyang | 90b30cd | 2013-02-22 16:33:27 -0800 | [diff] [blame] | 1836 | void try_offline_node(int nid) |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1837 | { |
Wen Congyang | d822b86 | 2013-02-22 16:33:16 -0800 | [diff] [blame] | 1838 | pg_data_t *pgdat = NODE_DATA(nid); |
David Hildenbrand | d98d053 | 2020-01-28 10:50:20 +0100 | [diff] [blame] | 1839 | int rc; |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1840 | |
David Hildenbrand | d98d053 | 2020-01-28 10:50:20 +0100 | [diff] [blame] | 1841 | /* |
| 1842 | * If the node still spans pages (especially ZONE_DEVICE), don't |
| 1843 | * offline it. A node spans memory after move_pfn_range_to_zone(), |
| 1844 | * e.g., after the memory block was onlined. |
| 1845 | */ |
| 1846 | if (pgdat->node_spanned_pages) |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1847 | return; |
David Hildenbrand | d98d053 | 2020-01-28 10:50:20 +0100 | [diff] [blame] | 1848 | |
| 1849 | /* |
| 1850 | * Especially offline memory blocks might not be spanned by the |
| 1851 | * node. They will get spanned by the node once they get onlined. |
| 1852 | * However, they link to the node in sysfs and can get onlined later. |
| 1853 | */ |
| 1854 | rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb); |
| 1855 | if (rc) |
| 1856 | return; |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1857 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1858 | if (check_and_unmap_cpu_on_node(pgdat)) |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1859 | return; |
| 1860 | |
| 1861 | /* |
| 1862 | * all memory/cpu of this node are removed, we can offline this |
| 1863 | * node now. |
| 1864 | */ |
| 1865 | node_set_offline(nid); |
| 1866 | unregister_one_node(nid); |
| 1867 | } |
Wen Congyang | 90b30cd | 2013-02-22 16:33:27 -0800 | [diff] [blame] | 1868 | EXPORT_SYMBOL(try_offline_node); |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1869 | |
David Hildenbrand | 2ad264f | 2020-01-28 10:50:03 +0100 | [diff] [blame] | 1870 | static void __release_memory_resource(resource_size_t start, |
| 1871 | resource_size_t size) |
| 1872 | { |
| 1873 | int ret; |
| 1874 | |
| 1875 | /* |
| 1876 | * When removing memory in the same granularity as it was added, |
| 1877 | * this function never fails. It might only fail if resources |
| 1878 | * have to be adjusted or split. We'll ignore the error, as |
| 1879 | * removing of memory cannot fail. |
| 1880 | */ |
| 1881 | ret = release_mem_region_adjustable(&iomem_resource, start, size); |
| 1882 | if (ret) { |
| 1883 | resource_size_t endres = start + size - 1; |
| 1884 | |
| 1885 | pr_warn("Unable to release resource <%pa-%pa> (%d)\n", |
| 1886 | &start, &endres, ret); |
| 1887 | } |
| 1888 | } |
| 1889 | |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1890 | /** |
| 1891 | * remove_memory |
Mike Rapoport | e8b098f | 2018-04-05 16:24:57 -0700 | [diff] [blame] | 1892 | * @nid: the node ID |
| 1893 | * @start: physical address of the region to remove |
| 1894 | * @size: size of the region to remove |
Toshi Kani | 0f1cfe9 | 2013-09-11 14:21:50 -0700 | [diff] [blame] | 1895 | * |
| 1896 | * NOTE: The caller must call lock_device_hotplug() to serialize hotplug |
| 1897 | * and online/offline operations before this call, as required by |
| 1898 | * try_offline_node(). |
| 1899 | */ |
David Hildenbrand | a3cf10b | 2020-01-28 10:49:58 +0100 | [diff] [blame] | 1900 | void __ref __remove_memory(int nid, u64 start, u64 size) |
Wen Congyang | bbc76be | 2013-02-22 16:32:54 -0800 | [diff] [blame] | 1901 | { |
Rafael J. Wysocki | 242831e | 2013-05-27 12:58:46 +0200 | [diff] [blame] | 1902 | int ret; |
Wen Congyang | 993c1aa | 2013-02-22 16:32:50 -0800 | [diff] [blame] | 1903 | |
Toshi Kani | 27356f5 | 2013-09-11 14:21:49 -0700 | [diff] [blame] | 1904 | BUG_ON(check_hotplug_memory_range(start, size)); |
| 1905 | |
Yasuaki Ishimatsu | 6677e3e | 2013-02-22 16:32:52 -0800 | [diff] [blame] | 1906 | /* |
Rafael J. Wysocki | 242831e | 2013-05-27 12:58:46 +0200 | [diff] [blame] | 1907 | * All memory blocks must be offlined before removing memory. Check |
| 1908 | * whether all memory blocks in question are offline and trigger a BUG() |
| 1909 | * if this is not the case. |
Yasuaki Ishimatsu | 6677e3e | 2013-02-22 16:32:52 -0800 | [diff] [blame] | 1910 | */ |
Rafael J. Wysocki | 242831e | 2013-05-27 12:58:46 +0200 | [diff] [blame] | 1911 | ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, |
Xishi Qiu | d6de9d5 | 2013-11-12 15:07:20 -0800 | [diff] [blame] | 1912 | check_memblock_offlined_cb); |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 1913 | if (ret) |
Rafael J. Wysocki | 242831e | 2013-05-27 12:58:46 +0200 | [diff] [blame] | 1914 | BUG(); |
Yasuaki Ishimatsu | 6677e3e | 2013-02-22 16:32:52 -0800 | [diff] [blame] | 1915 | |
Yasuaki Ishimatsu | 46c66c4 | 2013-02-22 16:32:56 -0800 | [diff] [blame] | 1916 | /* remove memmap entry */ |
| 1917 | firmware_map_remove(start, start + size, "System RAM"); |
Xishi Qiu | f9126ab | 2015-08-14 15:35:16 -0700 | [diff] [blame] | 1918 | memblock_free(start, size); |
| 1919 | memblock_remove(start, size); |
Yasuaki Ishimatsu | 46c66c4 | 2013-02-22 16:32:56 -0800 | [diff] [blame] | 1920 | |
Dan Williams | 9a6873a | 2020-01-30 22:11:17 -0800 | [diff] [blame] | 1921 | /* |
| 1922 | * Memory block device removal under the device_hotplug_lock is |
| 1923 | * a barrier against racing online attempts. |
| 1924 | */ |
David Hildenbrand | d883abb | 2020-01-28 10:50:14 +0100 | [diff] [blame] | 1925 | remove_memory_block_devices(start, size); |
| 1926 | |
Dan Williams | 9a6873a | 2020-01-30 22:11:17 -0800 | [diff] [blame] | 1927 | mem_hotplug_begin(); |
| 1928 | |
Oscar Salvador | 5c1f8f5 | 2020-01-28 10:50:02 +0100 | [diff] [blame] | 1929 | arch_remove_memory(nid, start, size, NULL); |
David Hildenbrand | 2ad264f | 2020-01-28 10:50:03 +0100 | [diff] [blame] | 1930 | __release_memory_resource(start, size); |
Wen Congyang | 24d335c | 2013-02-22 16:32:58 -0800 | [diff] [blame] | 1931 | |
Tang Chen | 60a5a19 | 2013-02-22 16:33:14 -0800 | [diff] [blame] | 1932 | try_offline_node(nid); |
| 1933 | |
Vladimir Davydov | bfc8c90 | 2014-06-04 16:07:18 -0700 | [diff] [blame] | 1934 | mem_hotplug_done(); |
Badari Pulavarty | 7108878 | 2008-10-18 20:25:58 -0700 | [diff] [blame] | 1935 | } |
David Hildenbrand | a3cf10b | 2020-01-28 10:49:58 +0100 | [diff] [blame] | 1936 | |
| 1937 | void remove_memory(int nid, u64 start, u64 size) |
| 1938 | { |
| 1939 | lock_device_hotplug(); |
| 1940 | __remove_memory(nid, start, size); |
| 1941 | unlock_device_hotplug(); |
| 1942 | } |
Badari Pulavarty | 7108878 | 2008-10-18 20:25:58 -0700 | [diff] [blame] | 1943 | EXPORT_SYMBOL_GPL(remove_memory); |
Rafael J. Wysocki | aba6efc | 2013-06-01 22:24:07 +0200 | [diff] [blame] | 1944 | #endif /* CONFIG_MEMORY_HOTREMOVE */ |