blob: 45950a074bdb44fc8f45cbf3682057bbc074a41a [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Andy Whitcroftd41dee32005-06-23 00:07:54 -07002/*
3 * sparse memory mappings.
4 */
Andy Whitcroftd41dee32005-06-23 00:07:54 -07005#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09006#include <linux/slab.h>
Andy Whitcroftd41dee32005-06-23 00:07:54 -07007#include <linux/mmzone.h>
8#include <linux/bootmem.h>
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -07009#include <linux/compiler.h>
Dave Hansen0b0acbe2005-10-29 18:16:55 -070010#include <linux/highmem.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040011#include <linux/export.h>
Dave Hansen28ae55c2005-09-03 15:54:29 -070012#include <linux/spinlock.h>
Dave Hansen0b0acbe2005-10-29 18:16:55 -070013#include <linux/vmalloc.h>
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -070014
Yasunori Goto0c0a4a52008-04-28 02:13:34 -070015#include "internal.h"
Andy Whitcroftd41dee32005-06-23 00:07:54 -070016#include <asm/dma.h>
Christoph Lameter8f6aac42007-10-16 01:24:13 -070017#include <asm/pgalloc.h>
18#include <asm/pgtable.h>
Andy Whitcroftd41dee32005-06-23 00:07:54 -070019
20/*
21 * Permanent SPARSEMEM data:
22 *
23 * 1) mem_section - memory sections, mem_map's for valid memory
24 */
Bob Picco3e347262005-09-03 15:54:28 -070025#ifdef CONFIG_SPARSEMEM_EXTREME
Kirill A. Shutemov83e3c482017-09-29 17:08:16 +030026struct mem_section **mem_section;
Bob Picco3e347262005-09-03 15:54:28 -070027#else
28struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
Ravikiran G Thirumalai22fc6ec2006-01-08 01:01:27 -080029 ____cacheline_internodealigned_in_smp;
Bob Picco3e347262005-09-03 15:54:28 -070030#endif
31EXPORT_SYMBOL(mem_section);
32
Christoph Lameter89689ae2006-12-06 20:31:45 -080033#ifdef NODE_NOT_IN_PAGE_FLAGS
34/*
35 * If we did not store the node number in the page then we have to
36 * do a lookup in the section_to_node_table in order to find which
37 * node the page belongs to.
38 */
39#if MAX_NUMNODES <= 256
40static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
41#else
42static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
43#endif
44
Ian Campbell33dd4e02011-07-25 17:11:51 -070045int page_to_nid(const struct page *page)
Christoph Lameter89689ae2006-12-06 20:31:45 -080046{
47 return section_to_node_table[page_to_section(page)];
48}
49EXPORT_SYMBOL(page_to_nid);
Andy Whitcroft85770ff2007-08-22 14:01:03 -070050
51static void set_section_nid(unsigned long section_nr, int nid)
52{
53 section_to_node_table[section_nr] = nid;
54}
55#else /* !NODE_NOT_IN_PAGE_FLAGS */
56static inline void set_section_nid(unsigned long section_nr, int nid)
57{
58}
Christoph Lameter89689ae2006-12-06 20:31:45 -080059#endif
60
Bob Picco3e347262005-09-03 15:54:28 -070061#ifdef CONFIG_SPARSEMEM_EXTREME
Fabian Frederickbd721ea2016-08-02 14:03:33 -070062static noinline struct mem_section __ref *sparse_index_alloc(int nid)
Bob Picco802f1922005-09-03 15:54:26 -070063{
Dave Hansen28ae55c2005-09-03 15:54:29 -070064 struct mem_section *section = NULL;
65 unsigned long array_size = SECTIONS_PER_ROOT *
66 sizeof(struct mem_section);
Bob Picco802f1922005-09-03 15:54:26 -070067
Michal Hockob95046b2017-09-06 16:20:41 -070068 if (slab_is_available())
69 section = kzalloc_node(array_size, GFP_KERNEL, nid);
70 else
Santosh Shilimkarbb016b82014-01-21 15:50:34 -080071 section = memblock_virt_alloc_node(array_size, nid);
Bob Picco3e347262005-09-03 15:54:28 -070072
Dave Hansen28ae55c2005-09-03 15:54:29 -070073 return section;
Bob Picco802f1922005-09-03 15:54:26 -070074}
Dave Hansen28ae55c2005-09-03 15:54:29 -070075
Yasunori Gotoa3142c82007-05-08 00:23:07 -070076static int __meminit sparse_index_init(unsigned long section_nr, int nid)
Dave Hansen28ae55c2005-09-03 15:54:29 -070077{
Dave Hansen28ae55c2005-09-03 15:54:29 -070078 unsigned long root = SECTION_NR_TO_ROOT(section_nr);
79 struct mem_section *section;
Dave Hansen28ae55c2005-09-03 15:54:29 -070080
81 if (mem_section[root])
82 return -EEXIST;
83
84 section = sparse_index_alloc(nid);
WANG Congaf0cd5a2007-12-17 16:19:58 -080085 if (!section)
86 return -ENOMEM;
Dave Hansen28ae55c2005-09-03 15:54:29 -070087
88 mem_section[root] = section;
Gavin Shanc1c95182012-07-31 16:46:06 -070089
Zhang Yanfei9d1936c2013-05-17 22:10:38 +080090 return 0;
Dave Hansen28ae55c2005-09-03 15:54:29 -070091}
92#else /* !SPARSEMEM_EXTREME */
93static inline int sparse_index_init(unsigned long section_nr, int nid)
94{
95 return 0;
96}
97#endif
98
Zhou Chengming91fd8b92016-07-28 15:48:35 -070099#ifdef CONFIG_SPARSEMEM_EXTREME
Dave Hansen4ca644d2005-10-29 18:16:51 -0700100int __section_nr(struct mem_section* ms)
101{
102 unsigned long root_nr;
Kirill A. Shutemov83e3c482017-09-29 17:08:16 +0300103 struct mem_section *root = NULL;
Dave Hansen4ca644d2005-10-29 18:16:51 -0700104
Mike Kravetz12783b02006-05-20 15:00:05 -0700105 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {
106 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);
Dave Hansen4ca644d2005-10-29 18:16:51 -0700107 if (!root)
108 continue;
109
110 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))
111 break;
112 }
113
Kirill A. Shutemov83e3c482017-09-29 17:08:16 +0300114 VM_BUG_ON(!root);
Gavin Shandb36a462012-07-31 16:46:04 -0700115
Dave Hansen4ca644d2005-10-29 18:16:51 -0700116 return (root_nr * SECTIONS_PER_ROOT) + (ms - root);
117}
Zhou Chengming91fd8b92016-07-28 15:48:35 -0700118#else
119int __section_nr(struct mem_section* ms)
120{
121 return (int)(ms - mem_section[0]);
122}
123#endif
Dave Hansen4ca644d2005-10-29 18:16:51 -0700124
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700125/*
126 * During early boot, before section_mem_map is used for an actual
127 * mem_map, we use section_mem_map to store the section's NUMA
128 * node. This keeps us from having to use another data structure. The
129 * node information is cleared just before we store the real mem_map.
130 */
131static inline unsigned long sparse_encode_early_nid(int nid)
132{
133 return (nid << SECTION_NID_SHIFT);
134}
135
136static inline int sparse_early_nid(struct mem_section *section)
137{
138 return (section->section_mem_map >> SECTION_NID_SHIFT);
139}
140
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700141/* Validate the physical addressing limitations of the model */
142void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
143 unsigned long *end_pfn)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700144{
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700145 unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT);
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700146
Ingo Molnarbead9a32008-04-16 01:40:00 +0200147 /*
148 * Sanity checks - do not allow an architecture to pass
149 * in larger pfns than the maximum scope of sparsemem:
150 */
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700151 if (*start_pfn > max_sparsemem_pfn) {
152 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
153 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
154 *start_pfn, *end_pfn, max_sparsemem_pfn);
155 WARN_ON_ONCE(1);
156 *start_pfn = max_sparsemem_pfn;
157 *end_pfn = max_sparsemem_pfn;
Cyrill Gorcunovef161a92009-03-31 15:19:25 -0700158 } else if (*end_pfn > max_sparsemem_pfn) {
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700159 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
160 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
161 *start_pfn, *end_pfn, max_sparsemem_pfn);
162 WARN_ON_ONCE(1);
163 *end_pfn = max_sparsemem_pfn;
164 }
165}
166
Dave Hansenc4e1be92017-07-06 15:36:44 -0700167/*
168 * There are a number of times that we loop over NR_MEM_SECTIONS,
169 * looking for section_present() on each. But, when we have very
170 * large physical address spaces, NR_MEM_SECTIONS can also be
171 * very large which makes the loops quite long.
172 *
173 * Keeping track of this gives us an easy way to break out of
174 * those loops early.
175 */
176int __highest_present_section_nr;
177static void section_mark_present(struct mem_section *ms)
178{
179 int section_nr = __section_nr(ms);
180
181 if (section_nr > __highest_present_section_nr)
182 __highest_present_section_nr = section_nr;
183
184 ms->section_mem_map |= SECTION_MARKED_PRESENT;
185}
186
187static inline int next_present_section_nr(int section_nr)
188{
189 do {
190 section_nr++;
191 if (present_section_nr(section_nr))
192 return section_nr;
Wei Yangd538c162018-06-07 17:06:39 -0700193 } while ((section_nr <= __highest_present_section_nr));
Dave Hansenc4e1be92017-07-06 15:36:44 -0700194
195 return -1;
196}
197#define for_each_present_section_nr(start, section_nr) \
198 for (section_nr = next_present_section_nr(start-1); \
Qian Cai7b287c42019-03-05 15:50:11 -0800199 ((section_nr != -1) && \
Dave Hansenc4e1be92017-07-06 15:36:44 -0700200 (section_nr <= __highest_present_section_nr)); \
201 section_nr = next_present_section_nr(section_nr))
202
Pavel Tatashin85c77f72018-08-17 15:49:33 -0700203static inline unsigned long first_present_section_nr(void)
204{
205 return next_present_section_nr(-1);
206}
207
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700208/* Record a memory area against a node. */
209void __init memory_present(int nid, unsigned long start, unsigned long end)
210{
211 unsigned long pfn;
Ingo Molnarbead9a32008-04-16 01:40:00 +0200212
Kirill A. Shutemov629a3592017-11-07 11:33:37 +0300213#ifdef CONFIG_SPARSEMEM_EXTREME
214 if (unlikely(!mem_section)) {
215 unsigned long size, align;
216
Baoquan Hed09cfbb2018-01-04 16:18:06 -0800217 size = sizeof(struct mem_section*) * NR_SECTION_ROOTS;
Kirill A. Shutemov629a3592017-11-07 11:33:37 +0300218 align = 1 << (INTERNODE_CACHE_SHIFT);
219 mem_section = memblock_virt_alloc(size, align);
220 }
221#endif
222
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700223 start &= PAGE_SECTION_MASK;
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700224 mminit_validate_memmodel_limits(&start, &end);
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700225 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
226 unsigned long section = pfn_to_section_nr(pfn);
Bob Picco802f1922005-09-03 15:54:26 -0700227 struct mem_section *ms;
228
229 sparse_index_init(section, nid);
Andy Whitcroft85770ff2007-08-22 14:01:03 -0700230 set_section_nid(section, nid);
Bob Picco802f1922005-09-03 15:54:26 -0700231
232 ms = __nr_to_section(section);
Dave Hansenc4e1be92017-07-06 15:36:44 -0700233 if (!ms->section_mem_map) {
Michal Hocko2d070ea2017-07-06 15:37:56 -0700234 ms->section_mem_map = sparse_encode_early_nid(nid) |
235 SECTION_IS_ONLINE;
Dave Hansenc4e1be92017-07-06 15:36:44 -0700236 section_mark_present(ms);
237 }
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700238 }
239}
240
241/*
Andy Whitcroft29751f62005-06-23 00:08:00 -0700242 * Subtle, we encode the real pfn into the mem_map such that
243 * the identity pfn - section_mem_map will return the actual
244 * physical page frame number.
245 */
246static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
247{
Petr Tesarikdef9b712018-01-31 16:20:26 -0800248 unsigned long coded_mem_map =
249 (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
250 BUILD_BUG_ON(SECTION_MAP_LAST_BIT > (1UL<<PFN_SECTION_SHIFT));
251 BUG_ON(coded_mem_map & ~SECTION_MAP_MASK);
252 return coded_mem_map;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700253}
254
255/*
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700256 * Decode mem_map from the coded memmap
Andy Whitcroft29751f62005-06-23 00:08:00 -0700257 */
Andy Whitcroft29751f62005-06-23 00:08:00 -0700258struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
259{
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700260 /* mask off the extra low bits of information */
261 coded_mem_map &= SECTION_MAP_MASK;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700262 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
263}
264
Oscar Salvador4e409872018-08-17 15:47:14 -0700265static void __meminit sparse_init_one_section(struct mem_section *ms,
Mel Gorman5c0e3062007-10-16 01:25:56 -0700266 unsigned long pnum, struct page *mem_map,
267 unsigned long *pageblock_bitmap)
Andy Whitcroft29751f62005-06-23 00:08:00 -0700268{
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700269 ms->section_mem_map &= ~SECTION_MAP_MASK;
Andy Whitcroft540557b2007-10-16 01:24:11 -0700270 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |
271 SECTION_HAS_MEM_MAP;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700272 ms->pageblock_flags = pageblock_bitmap;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700273}
274
Yasunori Goto04753272008-04-28 02:13:31 -0700275unsigned long usemap_size(void)
Mel Gorman5c0e3062007-10-16 01:25:56 -0700276{
Wei Yang60a7a882017-05-03 14:53:51 -0700277 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);
Mel Gorman5c0e3062007-10-16 01:25:56 -0700278}
279
280#ifdef CONFIG_MEMORY_HOTPLUG
281static unsigned long *__kmalloc_section_usemap(void)
282{
283 return kmalloc(usemap_size(), GFP_KERNEL);
284}
285#endif /* CONFIG_MEMORY_HOTPLUG */
286
Yasunori Goto48c90682008-07-23 21:28:15 -0700287#ifdef CONFIG_MEMORY_HOTREMOVE
288static unsigned long * __init
Yinghai Lua4322e12010-02-10 01:20:21 -0800289sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
Johannes Weiner238305b2012-05-29 15:06:36 -0700290 unsigned long size)
Yasunori Goto48c90682008-07-23 21:28:15 -0700291{
Yinghai Lu99ab7b12012-07-11 14:02:53 -0700292 unsigned long goal, limit;
293 unsigned long *p;
294 int nid;
Yasunori Goto48c90682008-07-23 21:28:15 -0700295 /*
296 * A page may contain usemaps for other sections preventing the
297 * page being freed and making a section unremovable while
Li Zhongc800bcd2014-03-31 16:41:58 +0800298 * other sections referencing the usemap remain active. Similarly,
Yasunori Goto48c90682008-07-23 21:28:15 -0700299 * a pgdat can prevent a section being removed. If section A
300 * contains a pgdat and section B contains the usemap, both
301 * sections become inter-dependent. This allocates usemaps
302 * from the same section as the pgdat where possible to avoid
303 * this problem.
304 */
Yinghai Lu07b4e2b2012-07-11 14:02:51 -0700305 goal = __pa(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT);
Yinghai Lu99ab7b12012-07-11 14:02:53 -0700306 limit = goal + (1UL << PA_SECTION_SHIFT);
307 nid = early_pfn_to_nid(goal >> PAGE_SHIFT);
308again:
Santosh Shilimkarbb016b82014-01-21 15:50:34 -0800309 p = memblock_virt_alloc_try_nid_nopanic(size,
310 SMP_CACHE_BYTES, goal, limit,
311 nid);
Yinghai Lu99ab7b12012-07-11 14:02:53 -0700312 if (!p && limit) {
313 limit = 0;
314 goto again;
315 }
316 return p;
Yasunori Goto48c90682008-07-23 21:28:15 -0700317}
318
319static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
320{
321 unsigned long usemap_snr, pgdat_snr;
Kirill A. Shutemov83e3c482017-09-29 17:08:16 +0300322 static unsigned long old_usemap_snr;
323 static unsigned long old_pgdat_snr;
Yasunori Goto48c90682008-07-23 21:28:15 -0700324 struct pglist_data *pgdat = NODE_DATA(nid);
325 int usemap_nid;
326
Kirill A. Shutemov83e3c482017-09-29 17:08:16 +0300327 /* First call */
328 if (!old_usemap_snr) {
329 old_usemap_snr = NR_MEM_SECTIONS;
330 old_pgdat_snr = NR_MEM_SECTIONS;
331 }
332
Yasunori Goto48c90682008-07-23 21:28:15 -0700333 usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT);
334 pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);
335 if (usemap_snr == pgdat_snr)
336 return;
337
338 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)
339 /* skip redundant message */
340 return;
341
342 old_usemap_snr = usemap_snr;
343 old_pgdat_snr = pgdat_snr;
344
345 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
346 if (usemap_nid != nid) {
Joe Perches11705322016-03-17 14:19:50 -0700347 pr_info("node %d must be removed before remove section %ld\n",
348 nid, usemap_snr);
Yasunori Goto48c90682008-07-23 21:28:15 -0700349 return;
350 }
351 /*
352 * There is a circular dependency.
353 * Some platforms allow un-removable section because they will just
354 * gather other removable sections for dynamic partitioning.
355 * Just notify un-removable section's number here.
356 */
Joe Perches11705322016-03-17 14:19:50 -0700357 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
358 usemap_snr, pgdat_snr, nid);
Yasunori Goto48c90682008-07-23 21:28:15 -0700359}
360#else
361static unsigned long * __init
Yinghai Lua4322e12010-02-10 01:20:21 -0800362sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
Johannes Weiner238305b2012-05-29 15:06:36 -0700363 unsigned long size)
Yasunori Goto48c90682008-07-23 21:28:15 -0700364{
Santosh Shilimkarbb016b82014-01-21 15:50:34 -0800365 return memblock_virt_alloc_node_nopanic(size, pgdat->node_id);
Yasunori Goto48c90682008-07-23 21:28:15 -0700366}
367
368static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
369{
370}
371#endif /* CONFIG_MEMORY_HOTREMOVE */
372
Pavel Tatashin35fd1eb2018-08-17 15:49:21 -0700373#ifdef CONFIG_SPARSEMEM_VMEMMAP
Pavel Tatashinafda57b2018-08-17 15:49:30 -0700374static unsigned long __init section_map_size(void)
Pavel Tatashin35fd1eb2018-08-17 15:49:21 -0700375{
376 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
377}
378
379#else
Pavel Tatashinafda57b2018-08-17 15:49:30 -0700380static unsigned long __init section_map_size(void)
Pavel Tatashine131c062018-08-17 15:49:26 -0700381{
382 return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
383}
384
Christoph Hellwig7b73d972017-12-29 08:53:54 +0100385struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid,
386 struct vmem_altmap *altmap)
Andy Whitcroft29751f62005-06-23 00:08:00 -0700387{
Pavel Tatashine131c062018-08-17 15:49:26 -0700388 unsigned long size = section_map_size();
389 struct page *map = sparse_buffer_alloc(size);
Andy Whitcroft29751f62005-06-23 00:08:00 -0700390
Pavel Tatashine131c062018-08-17 15:49:26 -0700391 if (map)
392 return map;
393
Santosh Shilimkarbb016b82014-01-21 15:50:34 -0800394 map = memblock_virt_alloc_try_nid(size,
395 PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
396 BOOTMEM_ALLOC_ACCESSIBLE, nid);
Christoph Lameter8f6aac42007-10-16 01:24:13 -0700397 return map;
398}
399#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
400
Pavel Tatashin35fd1eb2018-08-17 15:49:21 -0700401static void *sparsemap_buf __meminitdata;
402static void *sparsemap_buf_end __meminitdata;
403
Pavel Tatashinafda57b2018-08-17 15:49:30 -0700404static void __init sparse_buffer_init(unsigned long size, int nid)
Pavel Tatashin35fd1eb2018-08-17 15:49:21 -0700405{
406 WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */
407 sparsemap_buf =
408 memblock_virt_alloc_try_nid_raw(size, PAGE_SIZE,
409 __pa(MAX_DMA_ADDRESS),
410 BOOTMEM_ALLOC_ACCESSIBLE, nid);
411 sparsemap_buf_end = sparsemap_buf + size;
412}
413
Pavel Tatashinafda57b2018-08-17 15:49:30 -0700414static void __init sparse_buffer_fini(void)
Pavel Tatashin35fd1eb2018-08-17 15:49:21 -0700415{
416 unsigned long size = sparsemap_buf_end - sparsemap_buf;
417
418 if (sparsemap_buf && size > 0)
419 memblock_free_early(__pa(sparsemap_buf), size);
420 sparsemap_buf = NULL;
421}
422
423void * __meminit sparse_buffer_alloc(unsigned long size)
424{
425 void *ptr = NULL;
426
427 if (sparsemap_buf) {
428 ptr = PTR_ALIGN(sparsemap_buf, size);
429 if (ptr + size > sparsemap_buf_end)
430 ptr = NULL;
431 else
432 sparsemap_buf = ptr + size;
433 }
434 return ptr;
435}
436
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -0700437void __weak __meminit vmemmap_populate_print_last(void)
Yinghai Luc2b91e22008-04-12 01:19:24 -0700438{
439}
Yinghai Lua4322e12010-02-10 01:20:21 -0800440
Pavel Tatashin85c77f72018-08-17 15:49:33 -0700441/*
442 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
443 * And number of present sections in this node is map_count.
444 */
445static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
446 unsigned long pnum_end,
447 unsigned long map_count)
448{
449 unsigned long pnum, usemap_longs, *usemap;
450 struct page *map;
451
452 usemap_longs = BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS);
453 usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nid),
454 usemap_size() *
455 map_count);
456 if (!usemap) {
457 pr_err("%s: node[%d] usemap allocation failed", __func__, nid);
458 goto failed;
459 }
460 sparse_buffer_init(map_count * section_map_size(), nid);
461 for_each_present_section_nr(pnum_begin, pnum) {
462 if (pnum >= pnum_end)
463 break;
464
465 map = sparse_mem_map_populate(pnum, nid, NULL);
466 if (!map) {
467 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
468 __func__, nid);
469 pnum_begin = pnum;
470 goto failed;
471 }
472 check_usemap_section_nr(nid, usemap);
473 sparse_init_one_section(__nr_to_section(pnum), pnum, map, usemap);
474 usemap += usemap_longs;
475 }
476 sparse_buffer_fini();
477 return;
478failed:
479 /* We failed to allocate, mark all the following pnums as not present */
480 for_each_present_section_nr(pnum_begin, pnum) {
481 struct mem_section *ms;
482
483 if (pnum >= pnum_end)
484 break;
485 ms = __nr_to_section(pnum);
486 ms->section_mem_map = 0;
487 }
488}
489
490/*
491 * Allocate the accumulated non-linear sections, allocate a mem_map
492 * for each and record the physical to section mapping.
493 */
Pavel Tatashin2a3cb8b2018-08-17 15:49:37 -0700494void __init sparse_init(void)
Pavel Tatashin85c77f72018-08-17 15:49:33 -0700495{
496 unsigned long pnum_begin = first_present_section_nr();
497 int nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
498 unsigned long pnum_end, map_count = 1;
499
500 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
501 set_pageblock_order();
502
503 for_each_present_section_nr(pnum_begin + 1, pnum_end) {
504 int nid = sparse_early_nid(__nr_to_section(pnum_end));
505
506 if (nid == nid_begin) {
507 map_count++;
508 continue;
509 }
510 /* Init node with sections in range [pnum_begin, pnum_end) */
511 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
512 nid_begin = nid;
513 pnum_begin = pnum_end;
514 map_count = 1;
515 }
516 /* cover the last node */
517 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
518 vmemmap_populate_print_last();
519}
520
Stephen Rothwell193faea2007-06-08 13:46:51 -0700521#ifdef CONFIG_MEMORY_HOTPLUG
Michal Hocko2d070ea2017-07-06 15:37:56 -0700522
523/* Mark all memory sections within the pfn range as online */
524void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
525{
526 unsigned long pfn;
527
528 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
Michal Hockob4ccec42017-09-08 16:13:15 -0700529 unsigned long section_nr = pfn_to_section_nr(pfn);
Michal Hocko2d070ea2017-07-06 15:37:56 -0700530 struct mem_section *ms;
531
532 /* onlining code should never touch invalid ranges */
533 if (WARN_ON(!valid_section_nr(section_nr)))
534 continue;
535
536 ms = __nr_to_section(section_nr);
537 ms->section_mem_map |= SECTION_IS_ONLINE;
538 }
539}
540
541#ifdef CONFIG_MEMORY_HOTREMOVE
542/* Mark all memory sections within the pfn range as online */
543void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
544{
545 unsigned long pfn;
546
547 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
Pavel Tatashin27227c72018-05-11 16:01:50 -0700548 unsigned long section_nr = pfn_to_section_nr(pfn);
Michal Hocko2d070ea2017-07-06 15:37:56 -0700549 struct mem_section *ms;
550
551 /*
552 * TODO this needs some double checking. Offlining code makes
553 * sure to check pfn_valid but those checks might be just bogus
554 */
555 if (WARN_ON(!valid_section_nr(section_nr)))
556 continue;
557
558 ms = __nr_to_section(section_nr);
559 ms->section_mem_map &= ~SECTION_IS_ONLINE;
560 }
561}
562#endif
563
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700564#ifdef CONFIG_SPARSEMEM_VMEMMAP
Christoph Hellwig7b73d972017-12-29 08:53:54 +0100565static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
566 struct vmem_altmap *altmap)
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700567{
568 /* This will make the necessary allocations eventually. */
Christoph Hellwig7b73d972017-12-29 08:53:54 +0100569 return sparse_mem_map_populate(pnum, nid, altmap);
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700570}
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100571static void __kfree_section_memmap(struct page *memmap,
572 struct vmem_altmap *altmap)
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700573{
Johannes Weiner0aad8182013-04-29 15:07:50 -0700574 unsigned long start = (unsigned long)memmap;
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800575 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
Johannes Weiner0aad8182013-04-29 15:07:50 -0700576
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100577 vmemmap_free(start, end, altmap);
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700578}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700579#ifdef CONFIG_MEMORY_HOTREMOVE
Zhang Yanfei81556b02013-11-12 15:07:43 -0800580static void free_map_bootmem(struct page *memmap)
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700581{
Johannes Weiner0aad8182013-04-29 15:07:50 -0700582 unsigned long start = (unsigned long)memmap;
Zhang Yanfei81556b02013-11-12 15:07:43 -0800583 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
Johannes Weiner0aad8182013-04-29 15:07:50 -0700584
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100585 vmemmap_free(start, end, NULL);
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700586}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700587#endif /* CONFIG_MEMORY_HOTREMOVE */
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700588#else
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800589static struct page *__kmalloc_section_memmap(void)
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700590{
591 struct page *page, *ret;
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800592 unsigned long memmap_size = sizeof(struct page) * PAGES_PER_SECTION;
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700593
Yasunori Gotof2d0aa52006-10-28 10:38:32 -0700594 page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700595 if (page)
596 goto got_map_page;
597
598 ret = vmalloc(memmap_size);
599 if (ret)
600 goto got_map_ptr;
601
602 return NULL;
603got_map_page:
604 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
605got_map_ptr:
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700606
607 return ret;
608}
609
Christoph Hellwig7b73d972017-12-29 08:53:54 +0100610static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
611 struct vmem_altmap *altmap)
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700612{
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800613 return __kmalloc_section_memmap();
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700614}
615
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100616static void __kfree_section_memmap(struct page *memmap,
617 struct vmem_altmap *altmap)
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700618{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800619 if (is_vmalloc_addr(memmap))
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700620 vfree(memmap);
621 else
622 free_pages((unsigned long)memmap,
Zhang Yanfei85b35fe2013-11-12 15:07:42 -0800623 get_order(sizeof(struct page) * PAGES_PER_SECTION));
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700624}
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700625
David Rientjes4edd7ce2013-04-29 15:08:22 -0700626#ifdef CONFIG_MEMORY_HOTREMOVE
Zhang Yanfei81556b02013-11-12 15:07:43 -0800627static void free_map_bootmem(struct page *memmap)
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700628{
629 unsigned long maps_section_nr, removing_section_nr, i;
Zhang Yanfei81556b02013-11-12 15:07:43 -0800630 unsigned long magic, nr_pages;
Jianguo Wuae64ffc2012-11-29 13:54:21 -0800631 struct page *page = virt_to_page(memmap);
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700632
Zhang Yanfei81556b02013-11-12 15:07:43 -0800633 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page))
634 >> PAGE_SHIFT;
635
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700636 for (i = 0; i < nr_pages; i++, page++) {
Yasuaki Ishimatsuddffe982017-02-22 15:45:13 -0800637 magic = (unsigned long) page->freelist;
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700638
639 BUG_ON(magic == NODE_INFO);
640
641 maps_section_nr = pfn_to_section_nr(page_to_pfn(page));
Yasuaki Ishimatsu857e5222017-02-22 15:45:10 -0800642 removing_section_nr = page_private(page);
Yasunori Goto0c0a4a52008-04-28 02:13:34 -0700643
644 /*
645 * When this function is called, the removing section is
646 * logical offlined state. This means all pages are isolated
647 * from page allocator. If removing section's memmap is placed
648 * on the same section, it must not be freed.
649 * If it is freed, page allocator may allocate it which will
650 * be removed physically soon.
651 */
652 if (maps_section_nr != removing_section_nr)
653 put_page_bootmem(page);
654 }
655}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700656#endif /* CONFIG_MEMORY_HOTREMOVE */
Yasunori Goto98f3cfc2007-10-16 01:26:14 -0700657#endif /* CONFIG_SPARSEMEM_VMEMMAP */
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700658
Andy Whitcroft29751f62005-06-23 00:08:00 -0700659/*
Andy Whitcroft29751f62005-06-23 00:08:00 -0700660 * returns the number of sections whose mem_maps were properly
661 * set. If this is <=0, then that means that the passed-in
662 * map was not consumed and must be freed.
663 */
Christoph Hellwig7b73d972017-12-29 08:53:54 +0100664int __meminit sparse_add_one_section(struct pglist_data *pgdat,
665 unsigned long start_pfn, struct vmem_altmap *altmap)
Andy Whitcroft29751f62005-06-23 00:08:00 -0700666{
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700667 unsigned long section_nr = pfn_to_section_nr(start_pfn);
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700668 struct mem_section *ms;
669 struct page *memmap;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700670 unsigned long *usemap;
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700671 unsigned long flags;
672 int ret;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700673
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700674 /*
675 * no locking for this, because it does its own
676 * plus, it does a kmalloc
677 */
WANG Congbbd06822007-12-17 16:19:59 -0800678 ret = sparse_index_init(section_nr, pgdat->node_id);
679 if (ret < 0 && ret != -EEXIST)
680 return ret;
Oscar Salvador4e409872018-08-17 15:47:14 -0700681 ret = 0;
Christoph Hellwig7b73d972017-12-29 08:53:54 +0100682 memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, altmap);
WANG Congbbd06822007-12-17 16:19:59 -0800683 if (!memmap)
684 return -ENOMEM;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700685 usemap = __kmalloc_section_usemap();
WANG Congbbd06822007-12-17 16:19:59 -0800686 if (!usemap) {
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100687 __kfree_section_memmap(memmap, altmap);
WANG Congbbd06822007-12-17 16:19:59 -0800688 return -ENOMEM;
689 }
Andy Whitcroft29751f62005-06-23 00:08:00 -0700690
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700691 pgdat_resize_lock(pgdat, &flags);
692
693 ms = __pfn_to_section(start_pfn);
694 if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
695 ret = -EEXIST;
696 goto out;
697 }
Mel Gorman5c0e3062007-10-16 01:25:56 -0700698
Pavel Tatashind0dc12e2018-04-05 16:23:00 -0700699#ifdef CONFIG_DEBUG_VM
700 /*
701 * Poison uninitialized struct pages in order to catch invalid flags
702 * combinations.
703 */
704 memset(memmap, PAGE_POISON_PATTERN, sizeof(struct page) * PAGES_PER_SECTION);
705#endif
Wen Congyang3ac19f82012-12-11 16:00:59 -0800706
Dave Hansenc4e1be92017-07-06 15:36:44 -0700707 section_mark_present(ms);
Oscar Salvador4e409872018-08-17 15:47:14 -0700708 sparse_init_one_section(ms, section_nr, memmap, usemap);
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700709
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700710out:
711 pgdat_resize_unlock(pgdat, &flags);
Oscar Salvador4e409872018-08-17 15:47:14 -0700712 if (ret < 0) {
WANG Congbbd06822007-12-17 16:19:59 -0800713 kfree(usemap);
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100714 __kfree_section_memmap(memmap, altmap);
WANG Congbbd06822007-12-17 16:19:59 -0800715 }
Dave Hansen0b0acbe2005-10-29 18:16:55 -0700716 return ret;
Andy Whitcroft29751f62005-06-23 00:08:00 -0700717}
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700718
Zhang Yanfeif3deb682013-07-08 16:00:10 -0700719#ifdef CONFIG_MEMORY_HOTREMOVE
Wen Congyang95a47742012-12-11 16:00:47 -0800720#ifdef CONFIG_MEMORY_FAILURE
721static void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
722{
723 int i;
724
725 if (!memmap)
726 return;
727
Dan Williams4b94ffd2016-01-15 16:56:22 -0800728 for (i = 0; i < nr_pages; i++) {
Wen Congyang95a47742012-12-11 16:00:47 -0800729 if (PageHWPoison(&memmap[i])) {
Xishi Qiu293c07e2013-02-22 16:34:02 -0800730 atomic_long_sub(1, &num_poisoned_pages);
Wen Congyang95a47742012-12-11 16:00:47 -0800731 ClearPageHWPoison(&memmap[i]);
732 }
733 }
734}
735#else
736static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
737{
738}
739#endif
740
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100741static void free_section_usemap(struct page *memmap, unsigned long *usemap,
742 struct vmem_altmap *altmap)
David Rientjes4edd7ce2013-04-29 15:08:22 -0700743{
744 struct page *usemap_page;
David Rientjes4edd7ce2013-04-29 15:08:22 -0700745
746 if (!usemap)
747 return;
748
749 usemap_page = virt_to_page(usemap);
750 /*
751 * Check to see if allocation came from hot-plug-add
752 */
753 if (PageSlab(usemap_page) || PageCompound(usemap_page)) {
754 kfree(usemap);
755 if (memmap)
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100756 __kfree_section_memmap(memmap, altmap);
David Rientjes4edd7ce2013-04-29 15:08:22 -0700757 return;
758 }
759
760 /*
761 * The usemap came from bootmem. This is packed with other usemaps
762 * on the section which has pgdat at boot time. Just keep it as is now.
763 */
764
Zhang Yanfei81556b02013-11-12 15:07:43 -0800765 if (memmap)
766 free_map_bootmem(memmap);
David Rientjes4edd7ce2013-04-29 15:08:22 -0700767}
768
Dan Williams4b94ffd2016-01-15 16:56:22 -0800769void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100770 unsigned long map_offset, struct vmem_altmap *altmap)
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700771{
772 struct page *memmap = NULL;
Tang Chencd099682013-02-22 16:33:02 -0800773 unsigned long *usemap = NULL, flags;
774 struct pglist_data *pgdat = zone->zone_pgdat;
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700775
Tang Chencd099682013-02-22 16:33:02 -0800776 pgdat_resize_lock(pgdat, &flags);
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700777 if (ms->section_mem_map) {
778 usemap = ms->pageblock_flags;
779 memmap = sparse_decode_mem_map(ms->section_mem_map,
780 __section_nr(ms));
781 ms->section_mem_map = 0;
782 ms->pageblock_flags = NULL;
783 }
Tang Chencd099682013-02-22 16:33:02 -0800784 pgdat_resize_unlock(pgdat, &flags);
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700785
Dan Williams4b94ffd2016-01-15 16:56:22 -0800786 clear_hwpoisoned_pages(memmap + map_offset,
787 PAGES_PER_SECTION - map_offset);
Christoph Hellwig24b6d412017-12-29 08:53:56 +0100788 free_section_usemap(memmap, usemap, altmap);
Badari Pulavartyea01ea92008-04-28 02:12:01 -0700789}
David Rientjes4edd7ce2013-04-29 15:08:22 -0700790#endif /* CONFIG_MEMORY_HOTREMOVE */
791#endif /* CONFIG_MEMORY_HOTPLUG */