blob: f394dd87fa03359df9bb10ab3cc55b3c2e923062 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/page_alloc.c
3 *
4 * Manages the free list, the system allocates free pages here.
5 * Note that kmalloc() lives in slab.c
6 *
7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Swap reorganised 29.12.95, Stephen Tweedie
9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
15 */
16
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/stddef.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/interrupt.h>
21#include <linux/pagemap.h>
KOSAKI Motohiro10ed2732008-03-04 14:28:32 -080022#include <linux/jiffies.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/bootmem.h>
Yinghai Luedbe7d22010-08-25 13:39:16 -070024#include <linux/memblock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/compiler.h>
Randy Dunlap9f158332005-09-13 01:25:16 -070026#include <linux/kernel.h>
Vegard Nossumb1eeab62008-11-25 16:55:53 +010027#include <linux/kmemcheck.h>
Andrey Ryabininb8c73fc2015-02-13 14:39:28 -080028#include <linux/kasan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/module.h>
30#include <linux/suspend.h>
31#include <linux/pagevec.h>
32#include <linux/blkdev.h>
33#include <linux/slab.h>
Dave Hansena238ab52011-05-24 17:12:16 -070034#include <linux/ratelimit.h>
David Rientjes5a3135c22007-10-16 23:25:53 -070035#include <linux/oom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <linux/notifier.h>
37#include <linux/topology.h>
38#include <linux/sysctl.h>
39#include <linux/cpu.h>
40#include <linux/cpuset.h>
Dave Hansenbdc8cb92005-10-29 18:16:53 -070041#include <linux/memory_hotplug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <linux/nodemask.h>
43#include <linux/vmalloc.h>
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -070044#include <linux/vmstat.h>
Christoph Lameter4be38e32006-01-06 00:11:17 -080045#include <linux/mempolicy.h>
Dan Williams4b94ffd2016-01-15 16:56:22 -080046#include <linux/memremap.h>
Yasunori Goto68113782006-06-23 02:03:11 -070047#include <linux/stop_machine.h>
Mel Gormanc7132162006-09-27 01:49:43 -070048#include <linux/sort.h>
49#include <linux/pfn.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070050#include <linux/backing-dev.h>
Akinobu Mita933e3122006-12-08 02:39:45 -080051#include <linux/fault-inject.h>
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -070052#include <linux/page-isolation.h>
Joonsoo Kimeefa8642014-12-12 16:55:46 -080053#include <linux/page_ext.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070054#include <linux/debugobjects.h>
Catalin Marinasdbb1f812009-06-11 13:23:19 +010055#include <linux/kmemleak.h>
Mel Gorman56de7262010-05-24 14:32:30 -070056#include <linux/compaction.h>
Mel Gorman0d3d0622009-09-21 17:02:44 -070057#include <trace/events/kmem.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070058#include <linux/prefetch.h>
Lisa Du6e543d52013-09-11 14:22:36 -070059#include <linux/mm_inline.h>
Michal Nazarewicz041d3a82011-12-29 13:09:50 +010060#include <linux/migrate.h>
Joonsoo Kime30825f2014-12-12 16:55:49 -080061#include <linux/page_ext.h>
David Rientjes949f7ec2013-04-29 15:07:48 -070062#include <linux/hugetlb.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060063#include <linux/sched/rt.h>
Joonsoo Kim48c96a32014-12-12 16:56:01 -080064#include <linux/page_owner.h>
Mel Gorman0e1cc952015-06-30 14:57:27 -070065#include <linux/kthread.h>
Vladimir Davydov49491482016-07-26 15:24:24 -070066#include <linux/memcontrol.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070067
Jiang Liu7ee3d4e2013-07-03 15:03:41 -070068#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070069#include <asm/tlbflush.h>
Andrew Mortonac924c62006-05-15 09:43:59 -070070#include <asm/div64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070071#include "internal.h"
72
Cody P Schaferc8e251f2013-07-03 15:01:29 -070073/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
74static DEFINE_MUTEX(pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -070075#define MIN_PERCPU_PAGELIST_FRACTION (8)
Cody P Schaferc8e251f2013-07-03 15:01:29 -070076
Lee Schermerhorn72812012010-05-26 14:44:56 -070077#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
78DEFINE_PER_CPU(int, numa_node);
79EXPORT_PER_CPU_SYMBOL(numa_node);
80#endif
81
Lee Schermerhorn7aac7892010-05-26 14:45:00 -070082#ifdef CONFIG_HAVE_MEMORYLESS_NODES
83/*
84 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
85 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
86 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
87 * defined in <linux/topology.h>.
88 */
89DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
90EXPORT_PER_CPU_SYMBOL(_numa_mem_);
Joonsoo Kimad2c8142014-10-09 15:26:13 -070091int _node_numa_mem_[MAX_NUMNODES];
Lee Schermerhorn7aac7892010-05-26 14:45:00 -070092#endif
93
Emese Revfy38addce2016-06-20 20:41:19 +020094#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
Kees Cook58bea412016-10-19 00:08:04 +020095volatile unsigned long latent_entropy __latent_entropy;
Emese Revfy38addce2016-06-20 20:41:19 +020096EXPORT_SYMBOL(latent_entropy);
97#endif
98
Linus Torvalds1da177e2005-04-16 15:20:36 -070099/*
Christoph Lameter13808912007-10-16 01:25:27 -0700100 * Array of node states.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 */
Christoph Lameter13808912007-10-16 01:25:27 -0700102nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
103 [N_POSSIBLE] = NODE_MASK_ALL,
104 [N_ONLINE] = { { [0] = 1UL } },
105#ifndef CONFIG_NUMA
106 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
107#ifdef CONFIG_HIGHMEM
108 [N_HIGH_MEMORY] = { { [0] = 1UL } },
109#endif
Lai Jiangshan20b2f522012-12-12 13:52:00 -0800110#ifdef CONFIG_MOVABLE_NODE
111 [N_MEMORY] = { { [0] = 1UL } },
112#endif
Christoph Lameter13808912007-10-16 01:25:27 -0700113 [N_CPU] = { { [0] = 1UL } },
114#endif /* NUMA */
115};
116EXPORT_SYMBOL(node_states);
117
Jiang Liuc3d5f5f2013-07-03 15:03:14 -0700118/* Protect totalram_pages and zone->managed_pages */
119static DEFINE_SPINLOCK(managed_page_count_lock);
120
Ravikiran G Thirumalai6c231b72005-09-06 15:17:45 -0700121unsigned long totalram_pages __read_mostly;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -0700122unsigned long totalreserve_pages __read_mostly;
Pintu Kumare48322a2014-12-18 16:17:15 -0800123unsigned long totalcma_pages __read_mostly;
Johannes Weinerab8fabd2012-01-10 15:07:42 -0800124
Hugh Dickins1b76b022012-05-11 01:00:07 -0700125int percpu_pagelist_fraction;
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +1000126gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -0700128/*
129 * A cached value of the page's pageblock's migratetype, used when the page is
130 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
131 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
132 * Also the migratetype set in the page does not necessarily match the pcplist
133 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
134 * other index - this ensures that it will be put on the correct CMA freelist.
135 */
136static inline int get_pcppage_migratetype(struct page *page)
137{
138 return page->index;
139}
140
141static inline void set_pcppage_migratetype(struct page *page, int migratetype)
142{
143 page->index = migratetype;
144}
145
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800146#ifdef CONFIG_PM_SLEEP
147/*
148 * The following functions are used by the suspend/hibernate code to temporarily
149 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
150 * while devices are suspended. To avoid races with the suspend/hibernate code,
151 * they should always be called with pm_mutex held (gfp_allowed_mask also should
152 * only be modified with pm_mutex held, unless the suspend/hibernate code is
153 * guaranteed not to run in parallel with that modification).
154 */
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100155
156static gfp_t saved_gfp_mask;
157
158void pm_restore_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800159{
160 WARN_ON(!mutex_is_locked(&pm_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100161 if (saved_gfp_mask) {
162 gfp_allowed_mask = saved_gfp_mask;
163 saved_gfp_mask = 0;
164 }
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800165}
166
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100167void pm_restrict_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800168{
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800169 WARN_ON(!mutex_is_locked(&pm_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100170 WARN_ON(saved_gfp_mask);
171 saved_gfp_mask = gfp_allowed_mask;
Mel Gormand0164ad2015-11-06 16:28:21 -0800172 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800173}
Mel Gormanf90ac392012-01-10 15:07:15 -0800174
175bool pm_suspended_storage(void)
176{
Mel Gormand0164ad2015-11-06 16:28:21 -0800177 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
Mel Gormanf90ac392012-01-10 15:07:15 -0800178 return false;
179 return true;
180}
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800181#endif /* CONFIG_PM_SLEEP */
182
Mel Gormand9c23402007-10-16 01:26:01 -0700183#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800184unsigned int pageblock_order __read_mostly;
Mel Gormand9c23402007-10-16 01:26:01 -0700185#endif
186
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800187static void __free_pages_ok(struct page *page, unsigned int order);
David Howellsa226f6c2006-01-06 00:11:08 -0800188
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189/*
190 * results with 256, 32 in the lowmem_reserve sysctl:
191 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
192 * 1G machine -> (16M dma, 784M normal, 224M high)
193 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
194 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
Yaowei Bai84109e12015-02-12 15:00:22 -0800195 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
Andi Kleena2f1b422005-11-05 17:25:53 +0100196 *
197 * TBD: should special case ZONE_DMA32 machines here - in those we normally
198 * don't need any ZONE_NORMAL reservation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 */
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700200int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800201#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700202 256,
Christoph Lameter4b51d662007-02-10 01:43:10 -0800203#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700204#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700205 256,
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700206#endif
Christoph Lametere53ef382006-09-25 23:31:14 -0700207#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700208 32,
Christoph Lametere53ef382006-09-25 23:31:14 -0700209#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700210 32,
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700211};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212
213EXPORT_SYMBOL(totalram_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
Helge Deller15ad7cd2006-12-06 20:40:36 -0800215static char * const zone_names[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800216#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700217 "DMA",
Christoph Lameter4b51d662007-02-10 01:43:10 -0800218#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700219#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700220 "DMA32",
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700221#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700222 "Normal",
Christoph Lametere53ef382006-09-25 23:31:14 -0700223#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700224 "HighMem",
Christoph Lametere53ef382006-09-25 23:31:14 -0700225#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700226 "Movable",
Dan Williams033fbae2015-08-09 15:29:06 -0400227#ifdef CONFIG_ZONE_DEVICE
228 "Device",
229#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700230};
231
Vlastimil Babka60f30352016-03-15 14:56:08 -0700232char * const migratetype_names[MIGRATE_TYPES] = {
233 "Unmovable",
234 "Movable",
235 "Reclaimable",
236 "HighAtomic",
237#ifdef CONFIG_CMA
238 "CMA",
239#endif
240#ifdef CONFIG_MEMORY_ISOLATION
241 "Isolate",
242#endif
243};
244
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800245compound_page_dtor * const compound_page_dtors[] = {
246 NULL,
247 free_compound_page,
248#ifdef CONFIG_HUGETLB_PAGE
249 free_huge_page,
250#endif
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800251#ifdef CONFIG_TRANSPARENT_HUGEPAGE
252 free_transhuge_page,
253#endif
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800254};
255
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256int min_free_kbytes = 1024;
Han Pingtian42aa83c2014-01-23 15:53:28 -0800257int user_min_free_kbytes = -1;
Johannes Weiner795ae7a2016-03-17 14:19:14 -0700258int watermark_scale_factor = 10;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
Jan Beulich2c85f512009-09-21 17:03:07 -0700260static unsigned long __meminitdata nr_kernel_pages;
261static unsigned long __meminitdata nr_all_pages;
Yasunori Gotoa3142c82007-05-08 00:23:07 -0700262static unsigned long __meminitdata dma_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263
Tejun Heo0ee332c2011-12-08 10:22:09 -0800264#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
265static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
266static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
267static unsigned long __initdata required_kernelcore;
268static unsigned long __initdata required_movablecore;
269static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
Taku Izumi342332e2016-03-15 14:55:22 -0700270static bool mirrored_kernelcore;
Mel Gormanc7132162006-09-27 01:49:43 -0700271
Tejun Heo0ee332c2011-12-08 10:22:09 -0800272/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
273int movable_zone;
274EXPORT_SYMBOL(movable_zone);
275#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -0700276
Miklos Szeredi418508c2007-05-23 13:57:55 -0700277#if MAX_NUMNODES > 1
278int nr_node_ids __read_mostly = MAX_NUMNODES;
Christoph Lameter62bc62a2009-06-16 15:32:15 -0700279int nr_online_nodes __read_mostly = 1;
Miklos Szeredi418508c2007-05-23 13:57:55 -0700280EXPORT_SYMBOL(nr_node_ids);
Christoph Lameter62bc62a2009-06-16 15:32:15 -0700281EXPORT_SYMBOL(nr_online_nodes);
Miklos Szeredi418508c2007-05-23 13:57:55 -0700282#endif
283
Mel Gorman9ef9acb2007-10-16 01:25:54 -0700284int page_group_by_mobility_disabled __read_mostly;
285
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700286#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Pavel Tatashin9980b822017-11-15 17:38:41 -0800287
288/*
289 * Determine how many pages need to be initialized durig early boot
290 * (non-deferred initialization).
291 * The value of first_deferred_pfn will be set later, once non-deferred pages
292 * are initialized, but for now set it ULONG_MAX.
293 */
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700294static inline void reset_deferred_meminit(pg_data_t *pgdat)
295{
Pavel Tatashin9980b822017-11-15 17:38:41 -0800296 phys_addr_t start_addr, end_addr;
297 unsigned long max_pgcnt;
298 unsigned long reserved;
Michal Hocko292f70c2017-06-02 14:46:49 -0700299
300 /*
301 * Initialise at least 2G of a node but also take into account that
302 * two large system hashes that can take up 1GB for 0.25TB/node.
303 */
Pavel Tatashin9980b822017-11-15 17:38:41 -0800304 max_pgcnt = max(2UL << (30 - PAGE_SHIFT),
305 (pgdat->node_spanned_pages >> 8));
Michal Hocko292f70c2017-06-02 14:46:49 -0700306
307 /*
308 * Compensate the all the memblock reservations (e.g. crash kernel)
309 * from the initial estimation to make sure we will initialize enough
310 * memory to boot.
311 */
Pavel Tatashin9980b822017-11-15 17:38:41 -0800312 start_addr = PFN_PHYS(pgdat->node_start_pfn);
313 end_addr = PFN_PHYS(pgdat->node_start_pfn + max_pgcnt);
314 reserved = memblock_reserved_memory_within(start_addr, end_addr);
315 max_pgcnt += PHYS_PFN(reserved);
Michal Hocko292f70c2017-06-02 14:46:49 -0700316
Pavel Tatashin9980b822017-11-15 17:38:41 -0800317 pgdat->static_init_pgcnt = min(max_pgcnt, pgdat->node_spanned_pages);
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700318 pgdat->first_deferred_pfn = ULONG_MAX;
319}
320
321/* Returns true if the struct page for the pfn is uninitialised */
Mel Gorman0e1cc952015-06-30 14:57:27 -0700322static inline bool __meminit early_page_uninitialised(unsigned long pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700323{
Mel Gormanef70b6f2016-07-14 12:07:23 -0700324 int nid = early_pfn_to_nid(pfn);
325
326 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700327 return true;
328
329 return false;
330}
331
332/*
333 * Returns false when the remaining initialisation should be deferred until
334 * later in the boot cycle when it can be parallelised.
335 */
336static inline bool update_defer_init(pg_data_t *pgdat,
337 unsigned long pfn, unsigned long zone_end,
338 unsigned long *nr_initialised)
339{
340 /* Always populate low zones for address-contrained allocations */
341 if (zone_end < pgdat_end_pfn(pgdat))
342 return true;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700343 (*nr_initialised)++;
Pavel Tatashin9980b822017-11-15 17:38:41 -0800344 if ((*nr_initialised > pgdat->static_init_pgcnt) &&
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700345 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
346 pgdat->first_deferred_pfn = pfn;
347 return false;
348 }
349
350 return true;
351}
352#else
353static inline void reset_deferred_meminit(pg_data_t *pgdat)
354{
355}
356
357static inline bool early_page_uninitialised(unsigned long pfn)
358{
359 return false;
360}
361
362static inline bool update_defer_init(pg_data_t *pgdat,
363 unsigned long pfn, unsigned long zone_end,
364 unsigned long *nr_initialised)
365{
366 return true;
367}
368#endif
369
Mel Gorman0b423ca2016-05-19 17:14:27 -0700370/* Return a pointer to the bitmap storing bits affecting a block of pages */
371static inline unsigned long *get_pageblock_bitmap(struct page *page,
372 unsigned long pfn)
373{
374#ifdef CONFIG_SPARSEMEM
375 return __pfn_to_section(pfn)->pageblock_flags;
376#else
377 return page_zone(page)->pageblock_flags;
378#endif /* CONFIG_SPARSEMEM */
379}
380
381static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
382{
383#ifdef CONFIG_SPARSEMEM
384 pfn &= (PAGES_PER_SECTION-1);
385 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
386#else
387 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
388 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
389#endif /* CONFIG_SPARSEMEM */
390}
391
392/**
393 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
394 * @page: The page within the block of interest
395 * @pfn: The target page frame number
396 * @end_bitidx: The last bit of interest to retrieve
397 * @mask: mask of bits that the caller is interested in
398 *
399 * Return: pageblock_bits flags
400 */
401static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
402 unsigned long pfn,
403 unsigned long end_bitidx,
404 unsigned long mask)
405{
406 unsigned long *bitmap;
407 unsigned long bitidx, word_bitidx;
408 unsigned long word;
409
410 bitmap = get_pageblock_bitmap(page, pfn);
411 bitidx = pfn_to_bitidx(page, pfn);
412 word_bitidx = bitidx / BITS_PER_LONG;
413 bitidx &= (BITS_PER_LONG-1);
414
415 word = bitmap[word_bitidx];
416 bitidx += end_bitidx;
417 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
418}
419
420unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
421 unsigned long end_bitidx,
422 unsigned long mask)
423{
424 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
425}
426
427static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
428{
429 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
430}
431
432/**
433 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
434 * @page: The page within the block of interest
435 * @flags: The flags to set
436 * @pfn: The target page frame number
437 * @end_bitidx: The last bit of interest
438 * @mask: mask of bits that the caller is interested in
439 */
440void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
441 unsigned long pfn,
442 unsigned long end_bitidx,
443 unsigned long mask)
444{
445 unsigned long *bitmap;
446 unsigned long bitidx, word_bitidx;
447 unsigned long old_word, word;
448
449 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
450
451 bitmap = get_pageblock_bitmap(page, pfn);
452 bitidx = pfn_to_bitidx(page, pfn);
453 word_bitidx = bitidx / BITS_PER_LONG;
454 bitidx &= (BITS_PER_LONG-1);
455
456 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
457
458 bitidx += end_bitidx;
459 mask <<= (BITS_PER_LONG - bitidx - 1);
460 flags <<= (BITS_PER_LONG - bitidx - 1);
461
462 word = READ_ONCE(bitmap[word_bitidx]);
463 for (;;) {
464 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
465 if (word == old_word)
466 break;
467 word = old_word;
468 }
469}
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700470
Minchan Kimee6f5092012-07-31 16:43:50 -0700471void set_pageblock_migratetype(struct page *page, int migratetype)
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700472{
KOSAKI Motohiro5d0f3f72013-11-12 15:08:18 -0800473 if (unlikely(page_group_by_mobility_disabled &&
474 migratetype < MIGRATE_PCPTYPES))
Mel Gorman49255c62009-06-16 15:31:58 -0700475 migratetype = MIGRATE_UNMOVABLE;
476
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700477 set_pageblock_flags_group(page, (unsigned long)migratetype,
478 PB_migrate, PB_migrate_end);
479}
480
Nick Piggin13e74442006-01-06 00:10:58 -0800481#ifdef CONFIG_DEBUG_VM
Dave Hansenc6a57e12005-10-29 18:16:52 -0700482static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483{
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700484 int ret = 0;
485 unsigned seq;
486 unsigned long pfn = page_to_pfn(page);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800487 unsigned long sp, start_pfn;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700488
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700489 do {
490 seq = zone_span_seqbegin(zone);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800491 start_pfn = zone->zone_start_pfn;
492 sp = zone->spanned_pages;
Cody P Schafer108bcc92013-02-22 16:35:23 -0800493 if (!zone_spans_pfn(zone, pfn))
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700494 ret = 1;
495 } while (zone_span_seqretry(zone, seq));
496
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800497 if (ret)
Dave Hansen613813e2014-06-04 16:07:27 -0700498 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
499 pfn, zone_to_nid(zone), zone->name,
500 start_pfn, start_pfn + sp);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800501
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700502 return ret;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700503}
504
505static int page_is_consistent(struct zone *zone, struct page *page)
506{
Andy Whitcroft14e07292007-05-06 14:49:14 -0700507 if (!pfn_valid_within(page_to_pfn(page)))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700508 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 if (zone != page_zone(page))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700510 return 0;
511
512 return 1;
513}
514/*
515 * Temporary debugging check for pages not lying within a given zone.
516 */
517static int bad_range(struct zone *zone, struct page *page)
518{
519 if (page_outside_zone_boundaries(zone, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 return 1;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700521 if (!page_is_consistent(zone, page))
522 return 1;
523
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 return 0;
525}
Nick Piggin13e74442006-01-06 00:10:58 -0800526#else
527static inline int bad_range(struct zone *zone, struct page *page)
528{
529 return 0;
530}
531#endif
532
Kirill A. Shutemovd230dec2014-04-07 15:37:38 -0700533static void bad_page(struct page *page, const char *reason,
534 unsigned long bad_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535{
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800536 static unsigned long resume;
537 static unsigned long nr_shown;
538 static unsigned long nr_unshown;
539
540 /*
541 * Allow a burst of 60 reports, then keep quiet for that minute;
542 * or allow a steady drip of one report per second.
543 */
544 if (nr_shown == 60) {
545 if (time_before(jiffies, resume)) {
546 nr_unshown++;
547 goto out;
548 }
549 if (nr_unshown) {
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700550 pr_alert(
Hugh Dickins1e9e6362009-01-06 14:40:13 -0800551 "BUG: Bad page state: %lu messages suppressed\n",
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800552 nr_unshown);
553 nr_unshown = 0;
554 }
555 nr_shown = 0;
556 }
557 if (nr_shown++ == 0)
558 resume = jiffies + 60 * HZ;
559
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700560 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
Hugh Dickins3dc14742009-01-06 14:40:08 -0800561 current->comm, page_to_pfn(page));
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700562 __dump_page(page, reason);
563 bad_flags &= page->flags;
564 if (bad_flags)
565 pr_alert("bad because of flags: %#lx(%pGp)\n",
566 bad_flags, &bad_flags);
Vlastimil Babka4e462112016-03-15 14:56:21 -0700567 dump_page_owner(page);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700568
Dave Jones4f318882011-10-31 17:07:24 -0700569 print_modules();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 dump_stack();
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800571out:
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800572 /* Leave bad fields for debug, except PageBuddy could make trouble */
Mel Gorman22b751c2013-02-22 16:34:59 -0800573 page_mapcount_reset(page); /* remove PageBuddy */
Rusty Russell373d4d02013-01-21 17:17:39 +1030574 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575}
576
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577/*
578 * Higher-order pages are called "compound pages". They are structured thusly:
579 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800580 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800582 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
583 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800585 * The first tail page's ->compound_dtor holds the offset in array of compound
586 * page destructors. See compound_page_dtors.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800588 * The first tail page's ->compound_order holds the order of allocation.
Hugh Dickins41d78ba2006-02-14 13:52:58 -0800589 * This usage means that zero-order pages may not be compound.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 */
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800591
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800592void free_compound_page(struct page *page)
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800593{
Christoph Lameterd85f3382007-05-06 14:49:39 -0700594 __free_pages_ok(page, compound_order(page));
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800595}
596
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800597void prep_compound_page(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598{
599 int i;
600 int nr_pages = 1 << order;
601
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800602 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
Christoph Lameterd85f3382007-05-06 14:49:39 -0700603 set_compound_order(page, order);
Christoph Lameter6d777952007-05-06 14:49:40 -0700604 __SetPageHead(page);
Andy Whitcroft18229df2008-11-06 12:53:27 -0800605 for (i = 1; i < nr_pages; i++) {
606 struct page *p = page + i;
Youquan Song58a84aa2011-12-08 14:34:18 -0800607 set_page_count(p, 0);
Kirill A. Shutemov1c290f62016-01-15 16:52:07 -0800608 p->mapping = TAIL_MAPPING;
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800609 set_compound_head(p, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 }
Kirill A. Shutemov53f92632016-01-15 16:53:42 -0800611 atomic_set(compound_mapcount_ptr(page), -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612}
613
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800614#ifdef CONFIG_DEBUG_PAGEALLOC
615unsigned int _debug_guardpage_minorder;
Christian Borntraegerea6eabb2016-03-15 14:55:30 -0700616bool _debug_pagealloc_enabled __read_mostly
617 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
Joonsoo Kim505f6d22016-03-17 14:17:56 -0700618EXPORT_SYMBOL(_debug_pagealloc_enabled);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800619bool _debug_guardpage_enabled __read_mostly;
620
Joonsoo Kim031bc572014-12-12 16:55:52 -0800621static int __init early_debug_pagealloc(char *buf)
622{
623 if (!buf)
624 return -EINVAL;
Minfei Huang2a138dc2016-05-20 16:58:13 -0700625 return kstrtobool(buf, &_debug_pagealloc_enabled);
Joonsoo Kim031bc572014-12-12 16:55:52 -0800626}
627early_param("debug_pagealloc", early_debug_pagealloc);
628
Joonsoo Kime30825f2014-12-12 16:55:49 -0800629static bool need_debug_guardpage(void)
630{
Joonsoo Kim031bc572014-12-12 16:55:52 -0800631 /* If we don't use debug_pagealloc, we don't need guard page */
632 if (!debug_pagealloc_enabled())
633 return false;
634
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700635 if (!debug_guardpage_minorder())
636 return false;
637
Joonsoo Kime30825f2014-12-12 16:55:49 -0800638 return true;
639}
640
641static void init_debug_guardpage(void)
642{
Joonsoo Kim031bc572014-12-12 16:55:52 -0800643 if (!debug_pagealloc_enabled())
644 return;
645
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700646 if (!debug_guardpage_minorder())
647 return;
648
Joonsoo Kime30825f2014-12-12 16:55:49 -0800649 _debug_guardpage_enabled = true;
650}
651
652struct page_ext_operations debug_guardpage_ops = {
653 .need = need_debug_guardpage,
654 .init = init_debug_guardpage,
655};
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800656
657static int __init debug_guardpage_minorder_setup(char *buf)
658{
659 unsigned long res;
660
661 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
Joe Perches11705322016-03-17 14:19:50 -0700662 pr_err("Bad debug_guardpage_minorder value\n");
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800663 return 0;
664 }
665 _debug_guardpage_minorder = res;
Joe Perches11705322016-03-17 14:19:50 -0700666 pr_info("Setting debug_guardpage_minorder to %lu\n", res);
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800667 return 0;
668}
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700669early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800670
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700671static inline bool set_page_guard(struct zone *zone, struct page *page,
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800672 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800673{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800674 struct page_ext *page_ext;
675
676 if (!debug_guardpage_enabled())
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700677 return false;
678
679 if (order >= debug_guardpage_minorder())
680 return false;
Joonsoo Kime30825f2014-12-12 16:55:49 -0800681
682 page_ext = lookup_page_ext(page);
Yang Shif86e4272016-06-03 14:55:38 -0700683 if (unlikely(!page_ext))
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700684 return false;
Yang Shif86e4272016-06-03 14:55:38 -0700685
Joonsoo Kime30825f2014-12-12 16:55:49 -0800686 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
687
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800688 INIT_LIST_HEAD(&page->lru);
689 set_page_private(page, order);
690 /* Guard pages are not available for any usage */
691 __mod_zone_freepage_state(zone, -(1 << order), migratetype);
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700692
693 return true;
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800694}
695
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800696static inline void clear_page_guard(struct zone *zone, struct page *page,
697 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800698{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800699 struct page_ext *page_ext;
700
701 if (!debug_guardpage_enabled())
702 return;
703
704 page_ext = lookup_page_ext(page);
Yang Shif86e4272016-06-03 14:55:38 -0700705 if (unlikely(!page_ext))
706 return;
707
Joonsoo Kime30825f2014-12-12 16:55:49 -0800708 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
709
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800710 set_page_private(page, 0);
711 if (!is_migrate_isolate(migratetype))
712 __mod_zone_freepage_state(zone, (1 << order), migratetype);
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800713}
714#else
Joonsoo Kim980ac162016-10-07 16:58:27 -0700715struct page_ext_operations debug_guardpage_ops;
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700716static inline bool set_page_guard(struct zone *zone, struct page *page,
717 unsigned int order, int migratetype) { return false; }
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800718static inline void clear_page_guard(struct zone *zone, struct page *page,
719 unsigned int order, int migratetype) {}
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800720#endif
721
Mel Gorman7aeb09f2014-06-04 16:10:21 -0700722static inline void set_page_order(struct page *page, unsigned int order)
Andrew Morton6aa3001b22006-04-18 22:20:52 -0700723{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700724 set_page_private(page, order);
Nick Piggin676165a2006-04-10 11:21:48 +1000725 __SetPageBuddy(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726}
727
728static inline void rmv_page_order(struct page *page)
729{
Nick Piggin676165a2006-04-10 11:21:48 +1000730 __ClearPageBuddy(page);
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700731 set_page_private(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732}
733
734/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 * This function checks whether a page is free && is the buddy
736 * we can do coalesce a page and its buddy if
Nick Piggin13e74442006-01-06 00:10:58 -0800737 * (a) the buddy is not in a hole &&
Nick Piggin676165a2006-04-10 11:21:48 +1000738 * (b) the buddy is in the buddy system &&
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700739 * (c) a page and its buddy have the same order &&
740 * (d) a page and its buddy are in the same zone.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 *
Wang Sheng-Huicf6fe942013-09-11 14:22:48 -0700742 * For recording whether a page is in the buddy system, we set ->_mapcount
743 * PAGE_BUDDY_MAPCOUNT_VALUE.
744 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
745 * serialized by zone->lock.
Nick Piggin676165a2006-04-10 11:21:48 +1000746 *
747 * For recording page's order, we use page_private(page).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 */
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700749static inline int page_is_buddy(struct page *page, struct page *buddy,
Mel Gorman7aeb09f2014-06-04 16:10:21 -0700750 unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751{
Andy Whitcroft14e07292007-05-06 14:49:14 -0700752 if (!pfn_valid_within(page_to_pfn(buddy)))
Nick Piggin13e74442006-01-06 00:10:58 -0800753 return 0;
Nick Piggin13e74442006-01-06 00:10:58 -0800754
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800755 if (page_is_guard(buddy) && page_order(buddy) == order) {
Mel Gormand34c5fa2014-06-04 16:10:10 -0700756 if (page_zone_id(page) != page_zone_id(buddy))
757 return 0;
758
Weijie Yang4c5018c2015-02-10 14:11:39 -0800759 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
760
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800761 return 1;
762 }
763
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700764 if (PageBuddy(buddy) && page_order(buddy) == order) {
Mel Gormand34c5fa2014-06-04 16:10:10 -0700765 /*
766 * zone check is done late to avoid uselessly
767 * calculating zone/node ids for pages that could
768 * never merge.
769 */
770 if (page_zone_id(page) != page_zone_id(buddy))
771 return 0;
772
Weijie Yang4c5018c2015-02-10 14:11:39 -0800773 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
774
Andrew Morton6aa3001b22006-04-18 22:20:52 -0700775 return 1;
Nick Piggin676165a2006-04-10 11:21:48 +1000776 }
Andrew Morton6aa3001b22006-04-18 22:20:52 -0700777 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778}
779
780/*
781 * Freeing function for a buddy system allocator.
782 *
783 * The concept of a buddy system is to maintain direct-mapped table
784 * (containing bit values) for memory blocks of various "orders".
785 * The bottom level table contains the map for the smallest allocatable
786 * units of memory (here, pages), and each level above it describes
787 * pairs of units from the levels below, hence, "buddies".
788 * At a high level, all that happens here is marking the table entry
789 * at the bottom level available, and propagating the changes upward
790 * as necessary, plus some accounting needed to play nicely with other
791 * parts of the VM system.
792 * At each level, we keep a list of pages, which are heads of continuous
Wang Sheng-Huicf6fe942013-09-11 14:22:48 -0700793 * free pages of length of (1 << order) and marked with _mapcount
794 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
795 * field.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 * So when we are allocating or freeing one, we can derive the state of the
Michal Nazarewicz5f63b722012-01-11 15:16:11 +0100797 * other. That is, if we allocate a small block, and both were
798 * free, the remainder of the region must be split into blocks.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 * If a block is freed, and its buddy is also free, then this
Michal Nazarewicz5f63b722012-01-11 15:16:11 +0100800 * triggers coalescing into a block of larger size.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +0100802 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 */
804
Nick Piggin48db57f2006-01-08 01:00:42 -0800805static inline void __free_one_page(struct page *page,
Mel Gormandc4b0ca2014-06-04 16:10:17 -0700806 unsigned long pfn,
Mel Gormaned0ae212009-06-16 15:32:07 -0700807 struct zone *zone, unsigned int order,
808 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809{
810 unsigned long page_idx;
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700811 unsigned long combined_idx;
KyongHo Cho43506fa2011-01-13 15:47:24 -0800812 unsigned long uninitialized_var(buddy_idx);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700813 struct page *buddy;
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700814 unsigned int max_order;
815
816 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817
Cody P Schaferd29bb972013-02-22 16:35:25 -0800818 VM_BUG_ON(!zone_is_initialized(zone));
Kirill A. Shutemov6e9f0d52015-02-11 15:25:50 -0800819 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820
Mel Gormaned0ae212009-06-16 15:32:07 -0700821 VM_BUG_ON(migratetype == -1);
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700822 if (likely(!is_migrate_isolate(migratetype)))
Joonsoo Kim8f82b552014-11-13 15:19:18 -0800823 __mod_zone_freepage_state(zone, 1 << order, migratetype);
Mel Gormaned0ae212009-06-16 15:32:07 -0700824
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700825 page_idx = pfn & ((1 << MAX_ORDER) - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826
Sasha Levin309381fea2014-01-23 15:52:54 -0800827 VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
828 VM_BUG_ON_PAGE(bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700830continue_merging:
Joonsoo Kim3c605092014-11-13 15:19:21 -0800831 while (order < max_order - 1) {
KyongHo Cho43506fa2011-01-13 15:47:24 -0800832 buddy_idx = __find_buddy_index(page_idx, order);
833 buddy = page + (buddy_idx - page_idx);
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700834 if (!page_is_buddy(page, buddy, order))
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700835 goto done_merging;
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800836 /*
837 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
838 * merge with it and move up one order.
839 */
840 if (page_is_guard(buddy)) {
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800841 clear_page_guard(zone, buddy, order, migratetype);
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -0800842 } else {
843 list_del(&buddy->lru);
844 zone->free_area[order].nr_free--;
845 rmv_page_order(buddy);
846 }
KyongHo Cho43506fa2011-01-13 15:47:24 -0800847 combined_idx = buddy_idx & page_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 page = page + (combined_idx - page_idx);
849 page_idx = combined_idx;
850 order++;
851 }
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700852 if (max_order < MAX_ORDER) {
853 /* If we are here, it means order is >= pageblock_order.
854 * We want to prevent merge between freepages on isolate
855 * pageblock and normal pageblock. Without this, pageblock
856 * isolation could cause incorrect freepage or CMA accounting.
857 *
858 * We don't want to hit this code for the more frequent
859 * low-order merging.
860 */
861 if (unlikely(has_isolate_pageblock(zone))) {
862 int buddy_mt;
863
864 buddy_idx = __find_buddy_index(page_idx, order);
865 buddy = page + (buddy_idx - page_idx);
866 buddy_mt = get_pageblock_migratetype(buddy);
867
868 if (migratetype != buddy_mt
869 && (is_migrate_isolate(migratetype) ||
870 is_migrate_isolate(buddy_mt)))
871 goto done_merging;
872 }
873 max_order++;
874 goto continue_merging;
875 }
876
877done_merging:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 set_page_order(page, order);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700879
880 /*
881 * If this is not the largest possible page, check if the buddy
882 * of the next-highest order is free. If it is, it's possible
883 * that pages are being freed that will coalesce soon. In case,
884 * that is happening, add the free page to the tail of the list
885 * so it's less likely to be used soon and more likely to be merged
886 * as a higher order page
887 */
Mel Gormanb7f50cf2010-10-26 14:21:11 -0700888 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700889 struct page *higher_page, *higher_buddy;
KyongHo Cho43506fa2011-01-13 15:47:24 -0800890 combined_idx = buddy_idx & page_idx;
891 higher_page = page + (combined_idx - page_idx);
892 buddy_idx = __find_buddy_index(combined_idx, order + 1);
Li Haifeng0ba8f2d2012-09-17 14:09:21 -0700893 higher_buddy = higher_page + (buddy_idx - combined_idx);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700894 if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
895 list_add_tail(&page->lru,
896 &zone->free_area[order].free_list[migratetype]);
897 goto out;
898 }
899 }
900
901 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
902out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 zone->free_area[order].nr_free++;
904}
905
Mel Gorman7bfec6f2016-05-19 17:14:15 -0700906/*
907 * A bad page could be due to a number of fields. Instead of multiple branches,
908 * try and check multiple fields with one check. The caller must do a detailed
909 * check if necessary.
910 */
911static inline bool page_expected_state(struct page *page,
912 unsigned long check_flags)
913{
914 if (unlikely(atomic_read(&page->_mapcount) != -1))
915 return false;
916
917 if (unlikely((unsigned long)page->mapping |
918 page_ref_count(page) |
919#ifdef CONFIG_MEMCG
920 (unsigned long)page->mem_cgroup |
921#endif
922 (page->flags & check_flags)))
923 return false;
924
925 return true;
926}
927
Mel Gormanbb552ac2016-05-19 17:14:18 -0700928static void free_pages_check_bad(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929{
Mel Gorman7bfec6f2016-05-19 17:14:15 -0700930 const char *bad_reason;
931 unsigned long bad_flags;
932
Mel Gorman7bfec6f2016-05-19 17:14:15 -0700933 bad_reason = NULL;
934 bad_flags = 0;
Dave Hansenf0b791a2014-01-23 15:52:49 -0800935
Kirill A. Shutemov53f92632016-01-15 16:53:42 -0800936 if (unlikely(atomic_read(&page->_mapcount) != -1))
Dave Hansenf0b791a2014-01-23 15:52:49 -0800937 bad_reason = "nonzero mapcount";
938 if (unlikely(page->mapping != NULL))
939 bad_reason = "non-NULL mapping";
Joonsoo Kimfe896d12016-03-17 14:19:26 -0700940 if (unlikely(page_ref_count(page) != 0))
Joonsoo Kim0139aa72016-05-19 17:10:49 -0700941 bad_reason = "nonzero _refcount";
Dave Hansenf0b791a2014-01-23 15:52:49 -0800942 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
943 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
944 bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
945 }
Johannes Weiner9edad6e2014-12-10 15:44:58 -0800946#ifdef CONFIG_MEMCG
947 if (unlikely(page->mem_cgroup))
948 bad_reason = "page still charged to cgroup";
949#endif
Mel Gorman7bfec6f2016-05-19 17:14:15 -0700950 bad_page(page, bad_reason, bad_flags);
Mel Gormanbb552ac2016-05-19 17:14:18 -0700951}
952
953static inline int free_pages_check(struct page *page)
954{
Mel Gormanda838d42016-05-19 17:14:21 -0700955 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
Mel Gormanbb552ac2016-05-19 17:14:18 -0700956 return 0;
Mel Gormanbb552ac2016-05-19 17:14:18 -0700957
958 /* Something has gone sideways, find it */
959 free_pages_check_bad(page);
Mel Gorman7bfec6f2016-05-19 17:14:15 -0700960 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961}
962
Mel Gorman4db75482016-05-19 17:14:32 -0700963static int free_tail_pages_check(struct page *head_page, struct page *page)
964{
965 int ret = 1;
966
967 /*
968 * We rely page->lru.next never has bit 0 set, unless the page
969 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
970 */
971 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
972
973 if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
974 ret = 0;
975 goto out;
976 }
977 switch (page - head_page) {
978 case 1:
979 /* the first tail page: ->mapping is compound_mapcount() */
980 if (unlikely(compound_mapcount(page))) {
981 bad_page(page, "nonzero compound_mapcount", 0);
982 goto out;
983 }
984 break;
985 case 2:
986 /*
987 * the second tail page: ->mapping is
988 * page_deferred_list().next -- ignore value.
989 */
990 break;
991 default:
992 if (page->mapping != TAIL_MAPPING) {
993 bad_page(page, "corrupted mapping in tail page", 0);
994 goto out;
995 }
996 break;
997 }
998 if (unlikely(!PageTail(page))) {
999 bad_page(page, "PageTail not set", 0);
1000 goto out;
1001 }
1002 if (unlikely(compound_head(page) != head_page)) {
1003 bad_page(page, "compound_head not consistent", 0);
1004 goto out;
1005 }
1006 ret = 0;
1007out:
1008 page->mapping = NULL;
1009 clear_compound_head(page);
1010 return ret;
1011}
1012
Mel Gormane2769db2016-05-19 17:14:38 -07001013static __always_inline bool free_pages_prepare(struct page *page,
1014 unsigned int order, bool check_free)
1015{
1016 int bad = 0;
1017
1018 VM_BUG_ON_PAGE(PageTail(page), page);
1019
1020 trace_mm_page_free(page, order);
1021 kmemcheck_free_shadow(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001022
1023 /*
1024 * Check tail pages before head page information is cleared to
1025 * avoid checking PageCompound for order-0 pages.
1026 */
1027 if (unlikely(order)) {
1028 bool compound = PageCompound(page);
1029 int i;
1030
1031 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1032
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001033 if (compound)
1034 ClearPageDoubleMap(page);
Mel Gormane2769db2016-05-19 17:14:38 -07001035 for (i = 1; i < (1 << order); i++) {
1036 if (compound)
1037 bad += free_tail_pages_check(page, page + i);
1038 if (unlikely(free_pages_check(page + i))) {
1039 bad++;
1040 continue;
1041 }
1042 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1043 }
1044 }
Minchan Kimbda807d2016-07-26 15:23:05 -07001045 if (PageMappingFlags(page))
Mel Gormane2769db2016-05-19 17:14:38 -07001046 page->mapping = NULL;
Vladimir Davydovc4159a72016-08-08 23:03:12 +03001047 if (memcg_kmem_enabled() && PageKmemcg(page))
Vladimir Davydov49491482016-07-26 15:24:24 -07001048 memcg_kmem_uncharge(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001049 if (check_free)
1050 bad += free_pages_check(page);
1051 if (bad)
1052 return false;
1053
1054 page_cpupid_reset_last(page);
1055 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1056 reset_page_owner(page, order);
1057
1058 if (!PageHighMem(page)) {
1059 debug_check_no_locks_freed(page_address(page),
1060 PAGE_SIZE << order);
1061 debug_check_no_obj_freed(page_address(page),
1062 PAGE_SIZE << order);
1063 }
1064 arch_free_page(page, order);
1065 kernel_poison_pages(page, 1 << order, 0);
1066 kernel_map_pages(page, 1 << order, 0);
seokhoon.yoon29b52de2016-05-20 16:58:47 -07001067 kasan_free_pages(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001068
1069 return true;
1070}
Mel Gorman4db75482016-05-19 17:14:32 -07001071
1072#ifdef CONFIG_DEBUG_VM
1073static inline bool free_pcp_prepare(struct page *page)
1074{
Mel Gormane2769db2016-05-19 17:14:38 -07001075 return free_pages_prepare(page, 0, true);
Mel Gorman4db75482016-05-19 17:14:32 -07001076}
1077
1078static inline bool bulkfree_pcp_prepare(struct page *page)
1079{
1080 return false;
1081}
1082#else
1083static bool free_pcp_prepare(struct page *page)
1084{
Mel Gormane2769db2016-05-19 17:14:38 -07001085 return free_pages_prepare(page, 0, false);
Mel Gorman4db75482016-05-19 17:14:32 -07001086}
1087
1088static bool bulkfree_pcp_prepare(struct page *page)
1089{
1090 return free_pages_check(page);
1091}
1092#endif /* CONFIG_DEBUG_VM */
1093
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094/*
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001095 * Frees a number of pages from the PCP lists
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096 * Assumes all pages on list are in same zone, and of same order.
Renaud Lienhart207f36e2005-09-10 00:26:59 -07001097 * count is the number of pages to free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098 *
1099 * If the zone was previously in an "all pages pinned" state then look to
1100 * see if this freeing clears that state.
1101 *
1102 * And clear the zone's pages_scanned counter, to hold off the "all pages are
1103 * pinned" detection logic.
1104 */
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001105static void free_pcppages_bulk(struct zone *zone, int count,
1106 struct per_cpu_pages *pcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107{
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001108 int migratetype = 0;
Mel Gormana6f9edd2009-09-21 17:03:20 -07001109 int batch_free = 0;
Mel Gorman0d5d8232014-08-06 16:07:16 -07001110 unsigned long nr_scanned;
Mel Gorman37779992016-05-19 17:13:58 -07001111 bool isolated_pageblocks;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001112
Nick Pigginc54ad302006-01-06 00:10:56 -08001113 spin_lock(&zone->lock);
Mel Gorman37779992016-05-19 17:13:58 -07001114 isolated_pageblocks = has_isolate_pageblock(zone);
Mel Gorman599d0c92016-07-28 15:45:31 -07001115 nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
Mel Gorman0d5d8232014-08-06 16:07:16 -07001116 if (nr_scanned)
Mel Gorman599d0c92016-07-28 15:45:31 -07001117 __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
Mel Gormanf2260e62009-06-16 15:32:13 -07001118
Mel Gormane5b31ac2016-05-19 17:14:24 -07001119 while (count) {
Nick Piggin48db57f2006-01-08 01:00:42 -08001120 struct page *page;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001121 struct list_head *list;
Nick Piggin48db57f2006-01-08 01:00:42 -08001122
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001123 /*
Mel Gormana6f9edd2009-09-21 17:03:20 -07001124 * Remove pages from lists in a round-robin fashion. A
1125 * batch_free count is maintained that is incremented when an
1126 * empty list is encountered. This is so more pages are freed
1127 * off fuller lists instead of spinning excessively around empty
1128 * lists
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001129 */
1130 do {
Mel Gormana6f9edd2009-09-21 17:03:20 -07001131 batch_free++;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001132 if (++migratetype == MIGRATE_PCPTYPES)
1133 migratetype = 0;
1134 list = &pcp->lists[migratetype];
1135 } while (list_empty(list));
1136
Namhyung Kim1d168712011-03-22 16:32:45 -07001137 /* This is the only non-empty list. Free them all. */
1138 if (batch_free == MIGRATE_PCPTYPES)
Mel Gormane5b31ac2016-05-19 17:14:24 -07001139 batch_free = count;
Namhyung Kim1d168712011-03-22 16:32:45 -07001140
Mel Gormana6f9edd2009-09-21 17:03:20 -07001141 do {
Bartlomiej Zolnierkiewicz770c8aa2012-10-08 16:31:57 -07001142 int mt; /* migratetype of the to-be-freed page */
1143
Geliang Tanga16601c2016-01-14 15:20:30 -08001144 page = list_last_entry(list, struct page, lru);
Mel Gormana6f9edd2009-09-21 17:03:20 -07001145 /* must delete as __free_one_page list manipulates */
1146 list_del(&page->lru);
Vlastimil Babkaaa016d12015-09-08 15:01:22 -07001147
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07001148 mt = get_pcppage_migratetype(page);
Vlastimil Babkaaa016d12015-09-08 15:01:22 -07001149 /* MIGRATE_ISOLATE page should not go to pcplists */
1150 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1151 /* Pageblock could have been isolated meanwhile */
Mel Gorman37779992016-05-19 17:13:58 -07001152 if (unlikely(isolated_pageblocks))
Joonsoo Kim51bb1a42014-11-13 15:19:14 -08001153 mt = get_pageblock_migratetype(page);
Joonsoo Kim51bb1a42014-11-13 15:19:14 -08001154
Mel Gorman4db75482016-05-19 17:14:32 -07001155 if (bulkfree_pcp_prepare(page))
1156 continue;
1157
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001158 __free_one_page(page, page_to_pfn(page), zone, 0, mt);
Bartlomiej Zolnierkiewicz770c8aa2012-10-08 16:31:57 -07001159 trace_mm_page_pcpu_drain(page, 0, mt);
Mel Gormane5b31ac2016-05-19 17:14:24 -07001160 } while (--count && --batch_free && !list_empty(list));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 }
Nick Pigginc54ad302006-01-06 00:10:56 -08001162 spin_unlock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163}
1164
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001165static void free_one_page(struct zone *zone,
1166 struct page *page, unsigned long pfn,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07001167 unsigned int order,
Mel Gormaned0ae212009-06-16 15:32:07 -07001168 int migratetype)
Nick Piggin48db57f2006-01-08 01:00:42 -08001169{
Mel Gorman0d5d8232014-08-06 16:07:16 -07001170 unsigned long nr_scanned;
Christoph Lameter006d22d2006-09-25 23:31:48 -07001171 spin_lock(&zone->lock);
Mel Gorman599d0c92016-07-28 15:45:31 -07001172 nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
Mel Gorman0d5d8232014-08-06 16:07:16 -07001173 if (nr_scanned)
Mel Gorman599d0c92016-07-28 15:45:31 -07001174 __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
Mel Gormanf2260e62009-06-16 15:32:13 -07001175
Joonsoo Kimad53f922014-11-13 15:19:11 -08001176 if (unlikely(has_isolate_pageblock(zone) ||
1177 is_migrate_isolate(migratetype))) {
1178 migratetype = get_pfnblock_migratetype(page, pfn);
Joonsoo Kimad53f922014-11-13 15:19:11 -08001179 }
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001180 __free_one_page(page, pfn, zone, order, migratetype);
Christoph Lameter006d22d2006-09-25 23:31:48 -07001181 spin_unlock(&zone->lock);
Nick Piggin48db57f2006-01-08 01:00:42 -08001182}
1183
Robin Holt1e8ce832015-06-30 14:56:45 -07001184static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1185 unsigned long zone, int nid)
1186{
Robin Holt1e8ce832015-06-30 14:56:45 -07001187 set_page_links(page, zone, nid, pfn);
Robin Holt1e8ce832015-06-30 14:56:45 -07001188 init_page_count(page);
1189 page_mapcount_reset(page);
1190 page_cpupid_reset_last(page);
Robin Holt1e8ce832015-06-30 14:56:45 -07001191
Robin Holt1e8ce832015-06-30 14:56:45 -07001192 INIT_LIST_HEAD(&page->lru);
1193#ifdef WANT_PAGE_VIRTUAL
1194 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1195 if (!is_highmem_idx(zone))
1196 set_page_address(page, __va(pfn << PAGE_SHIFT));
1197#endif
1198}
1199
1200static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone,
1201 int nid)
1202{
1203 return __init_single_page(pfn_to_page(pfn), pfn, zone, nid);
1204}
1205
Mel Gorman7e18adb2015-06-30 14:57:05 -07001206#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1207static void init_reserved_page(unsigned long pfn)
1208{
1209 pg_data_t *pgdat;
1210 int nid, zid;
1211
1212 if (!early_page_uninitialised(pfn))
1213 return;
1214
1215 nid = early_pfn_to_nid(pfn);
1216 pgdat = NODE_DATA(nid);
1217
1218 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1219 struct zone *zone = &pgdat->node_zones[zid];
1220
1221 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1222 break;
1223 }
1224 __init_single_pfn(pfn, zid, nid);
1225}
1226#else
1227static inline void init_reserved_page(unsigned long pfn)
1228{
1229}
1230#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1231
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001232/*
1233 * Initialised pages do not have PageReserved set. This function is
1234 * called for each range allocated by the bootmem allocator and
1235 * marks the pages PageReserved. The remaining valid pages are later
1236 * sent to the buddy page allocator.
1237 */
Stefan Bader4b50bcc2016-05-20 16:58:38 -07001238void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001239{
1240 unsigned long start_pfn = PFN_DOWN(start);
1241 unsigned long end_pfn = PFN_UP(end);
1242
Mel Gorman7e18adb2015-06-30 14:57:05 -07001243 for (; start_pfn < end_pfn; start_pfn++) {
1244 if (pfn_valid(start_pfn)) {
1245 struct page *page = pfn_to_page(start_pfn);
1246
1247 init_reserved_page(start_pfn);
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -08001248
1249 /* Avoid false-positive PageTail() */
1250 INIT_LIST_HEAD(&page->lru);
1251
Mel Gorman7e18adb2015-06-30 14:57:05 -07001252 SetPageReserved(page);
1253 }
1254 }
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001255}
1256
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001257static void __free_pages_ok(struct page *page, unsigned int order)
1258{
1259 unsigned long flags;
Minchan Kim95e34412012-10-08 16:32:11 -07001260 int migratetype;
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001261 unsigned long pfn = page_to_pfn(page);
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001262
Mel Gormane2769db2016-05-19 17:14:38 -07001263 if (!free_pages_prepare(page, order, true))
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001264 return;
1265
Mel Gormancfc47a22014-06-04 16:10:19 -07001266 migratetype = get_pfnblock_migratetype(page, pfn);
Nick Pigginc54ad302006-01-06 00:10:56 -08001267 local_irq_save(flags);
Christoph Lameterf8891e52006-06-30 01:55:45 -07001268 __count_vm_events(PGFREE, 1 << order);
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001269 free_one_page(page_zone(page), page, pfn, order, migratetype);
Nick Pigginc54ad302006-01-06 00:10:56 -08001270 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271}
1272
Li Zhang949698a2016-05-19 17:11:37 -07001273static void __init __free_pages_boot_core(struct page *page, unsigned int order)
David Howellsa226f6c2006-01-06 00:11:08 -08001274{
Johannes Weinerc3993072012-01-10 15:08:10 -08001275 unsigned int nr_pages = 1 << order;
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001276 struct page *p = page;
Johannes Weinerc3993072012-01-10 15:08:10 -08001277 unsigned int loop;
David Howellsa226f6c2006-01-06 00:11:08 -08001278
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001279 prefetchw(p);
1280 for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1281 prefetchw(p + 1);
Johannes Weinerc3993072012-01-10 15:08:10 -08001282 __ClearPageReserved(p);
1283 set_page_count(p, 0);
David Howellsa226f6c2006-01-06 00:11:08 -08001284 }
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001285 __ClearPageReserved(p);
1286 set_page_count(p, 0);
Johannes Weinerc3993072012-01-10 15:08:10 -08001287
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001288 page_zone(page)->managed_pages += nr_pages;
Johannes Weinerc3993072012-01-10 15:08:10 -08001289 set_page_refcounted(page);
1290 __free_pages(page, order);
David Howellsa226f6c2006-01-06 00:11:08 -08001291}
1292
Mel Gorman75a592a2015-06-30 14:56:59 -07001293#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
1294 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
Mel Gorman7ace9912015-08-06 15:46:13 -07001295
Mel Gorman75a592a2015-06-30 14:56:59 -07001296static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1297
1298int __meminit early_pfn_to_nid(unsigned long pfn)
1299{
Mel Gorman7ace9912015-08-06 15:46:13 -07001300 static DEFINE_SPINLOCK(early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001301 int nid;
1302
Mel Gorman7ace9912015-08-06 15:46:13 -07001303 spin_lock(&early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001304 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
Mel Gorman7ace9912015-08-06 15:46:13 -07001305 if (nid < 0)
Mel Gormane4568d32016-07-14 12:07:20 -07001306 nid = first_online_node;
Mel Gorman7ace9912015-08-06 15:46:13 -07001307 spin_unlock(&early_pfn_lock);
1308
1309 return nid;
Mel Gorman75a592a2015-06-30 14:56:59 -07001310}
1311#endif
1312
1313#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1314static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
1315 struct mminit_pfnnid_cache *state)
1316{
1317 int nid;
1318
1319 nid = __early_pfn_to_nid(pfn, state);
1320 if (nid >= 0 && nid != node)
1321 return false;
1322 return true;
1323}
1324
1325/* Only safe to use early in boot when initialisation is single-threaded */
1326static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1327{
1328 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache);
1329}
1330
1331#else
1332
1333static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1334{
1335 return true;
1336}
1337static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
1338 struct mminit_pfnnid_cache *state)
1339{
1340 return true;
1341}
1342#endif
1343
1344
Mel Gorman0e1cc952015-06-30 14:57:27 -07001345void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001346 unsigned int order)
1347{
1348 if (early_page_uninitialised(pfn))
1349 return;
Li Zhang949698a2016-05-19 17:11:37 -07001350 return __free_pages_boot_core(page, order);
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001351}
1352
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001353/*
1354 * Check that the whole (or subset of) a pageblock given by the interval of
1355 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1356 * with the migration of free compaction scanner. The scanners then need to
1357 * use only pfn_valid_within() check for arches that allow holes within
1358 * pageblocks.
1359 *
1360 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1361 *
1362 * It's possible on some configurations to have a setup like node0 node1 node0
1363 * i.e. it's possible that all pages within a zones range of pages do not
1364 * belong to a single zone. We assume that a border between node0 and node1
1365 * can occur within a single pageblock, but not a node0 node1 node0
1366 * interleaving within a single pageblock. It is therefore sufficient to check
1367 * the first and last page of a pageblock and avoid checking each individual
1368 * page in a pageblock.
1369 */
1370struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1371 unsigned long end_pfn, struct zone *zone)
1372{
1373 struct page *start_page;
1374 struct page *end_page;
1375
1376 /* end_pfn is one past the range we are checking */
1377 end_pfn--;
1378
1379 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1380 return NULL;
1381
1382 start_page = pfn_to_page(start_pfn);
1383
1384 if (page_zone(start_page) != zone)
1385 return NULL;
1386
1387 end_page = pfn_to_page(end_pfn);
1388
1389 /* This gives a shorter code than deriving page_zone(end_page) */
1390 if (page_zone_id(start_page) != page_zone_id(end_page))
1391 return NULL;
1392
1393 return start_page;
1394}
1395
1396void set_zone_contiguous(struct zone *zone)
1397{
1398 unsigned long block_start_pfn = zone->zone_start_pfn;
1399 unsigned long block_end_pfn;
1400
1401 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1402 for (; block_start_pfn < zone_end_pfn(zone);
1403 block_start_pfn = block_end_pfn,
1404 block_end_pfn += pageblock_nr_pages) {
1405
1406 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1407
1408 if (!__pageblock_pfn_to_page(block_start_pfn,
1409 block_end_pfn, zone))
1410 return;
David Hildenbrande254aa02020-05-07 18:35:46 -07001411 cond_resched();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001412 }
1413
1414 /* We confirm that there is no hole */
1415 zone->contiguous = true;
1416}
1417
1418void clear_zone_contiguous(struct zone *zone)
1419{
1420 zone->contiguous = false;
1421}
1422
Mel Gorman7e18adb2015-06-30 14:57:05 -07001423#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Mel Gorman0e1cc952015-06-30 14:57:27 -07001424static void __init deferred_free_range(struct page *page,
Mel Gormana4de83d2015-06-30 14:57:16 -07001425 unsigned long pfn, int nr_pages)
1426{
1427 int i;
1428
1429 if (!page)
1430 return;
1431
1432 /* Free a large naturally-aligned chunk if possible */
Xishi Qiue7801492016-10-07 16:58:09 -07001433 if (nr_pages == pageblock_nr_pages &&
1434 (pfn & (pageblock_nr_pages - 1)) == 0) {
Mel Gormanac5d2532015-06-30 14:57:20 -07001435 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Xishi Qiue7801492016-10-07 16:58:09 -07001436 __free_pages_boot_core(page, pageblock_order);
Mel Gormana4de83d2015-06-30 14:57:16 -07001437 return;
1438 }
1439
Xishi Qiue7801492016-10-07 16:58:09 -07001440 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1441 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1442 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Li Zhang949698a2016-05-19 17:11:37 -07001443 __free_pages_boot_core(page, 0);
Xishi Qiue7801492016-10-07 16:58:09 -07001444 }
Mel Gormana4de83d2015-06-30 14:57:16 -07001445}
1446
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001447/* Completion tracking for deferred_init_memmap() threads */
1448static atomic_t pgdat_init_n_undone __initdata;
1449static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1450
1451static inline void __init pgdat_init_report_one_done(void)
1452{
1453 if (atomic_dec_and_test(&pgdat_init_n_undone))
1454 complete(&pgdat_init_all_done_comp);
1455}
Mel Gorman0e1cc952015-06-30 14:57:27 -07001456
Mel Gorman7e18adb2015-06-30 14:57:05 -07001457/* Initialise remaining memory on a node */
Mel Gorman0e1cc952015-06-30 14:57:27 -07001458static int __init deferred_init_memmap(void *data)
Mel Gorman7e18adb2015-06-30 14:57:05 -07001459{
Mel Gorman0e1cc952015-06-30 14:57:27 -07001460 pg_data_t *pgdat = data;
1461 int nid = pgdat->node_id;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001462 struct mminit_pfnnid_cache nid_init_state = { };
1463 unsigned long start = jiffies;
1464 unsigned long nr_pages = 0;
1465 unsigned long walk_start, walk_end;
1466 int i, zid;
1467 struct zone *zone;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001468 unsigned long first_init_pfn = pgdat->first_deferred_pfn;
Mel Gorman0e1cc952015-06-30 14:57:27 -07001469 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001470
Mel Gorman0e1cc952015-06-30 14:57:27 -07001471 if (first_init_pfn == ULONG_MAX) {
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001472 pgdat_init_report_one_done();
Mel Gorman0e1cc952015-06-30 14:57:27 -07001473 return 0;
1474 }
1475
1476 /* Bind memory initialisation thread to a local node if possible */
1477 if (!cpumask_empty(cpumask))
1478 set_cpus_allowed_ptr(current, cpumask);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001479
1480 /* Sanity check boundaries */
1481 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1482 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1483 pgdat->first_deferred_pfn = ULONG_MAX;
1484
1485 /* Only the highest zone is deferred so find it */
1486 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1487 zone = pgdat->node_zones + zid;
1488 if (first_init_pfn < zone_end_pfn(zone))
1489 break;
1490 }
1491
1492 for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) {
1493 unsigned long pfn, end_pfn;
Mel Gorman54608c32015-06-30 14:57:09 -07001494 struct page *page = NULL;
Mel Gormana4de83d2015-06-30 14:57:16 -07001495 struct page *free_base_page = NULL;
1496 unsigned long free_base_pfn = 0;
1497 int nr_to_free = 0;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001498
1499 end_pfn = min(walk_end, zone_end_pfn(zone));
1500 pfn = first_init_pfn;
1501 if (pfn < walk_start)
1502 pfn = walk_start;
1503 if (pfn < zone->zone_start_pfn)
1504 pfn = zone->zone_start_pfn;
1505
1506 for (; pfn < end_pfn; pfn++) {
Mel Gorman54608c32015-06-30 14:57:09 -07001507 if (!pfn_valid_within(pfn))
Mel Gormana4de83d2015-06-30 14:57:16 -07001508 goto free_range;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001509
Mel Gorman54608c32015-06-30 14:57:09 -07001510 /*
1511 * Ensure pfn_valid is checked every
Xishi Qiue7801492016-10-07 16:58:09 -07001512 * pageblock_nr_pages for memory holes
Mel Gorman54608c32015-06-30 14:57:09 -07001513 */
Xishi Qiue7801492016-10-07 16:58:09 -07001514 if ((pfn & (pageblock_nr_pages - 1)) == 0) {
Mel Gorman54608c32015-06-30 14:57:09 -07001515 if (!pfn_valid(pfn)) {
1516 page = NULL;
Mel Gormana4de83d2015-06-30 14:57:16 -07001517 goto free_range;
Mel Gorman54608c32015-06-30 14:57:09 -07001518 }
1519 }
1520
1521 if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) {
1522 page = NULL;
Mel Gormana4de83d2015-06-30 14:57:16 -07001523 goto free_range;
Mel Gorman54608c32015-06-30 14:57:09 -07001524 }
1525
1526 /* Minimise pfn page lookups and scheduler checks */
Xishi Qiue7801492016-10-07 16:58:09 -07001527 if (page && (pfn & (pageblock_nr_pages - 1)) != 0) {
Mel Gorman54608c32015-06-30 14:57:09 -07001528 page++;
1529 } else {
Mel Gormana4de83d2015-06-30 14:57:16 -07001530 nr_pages += nr_to_free;
1531 deferred_free_range(free_base_page,
1532 free_base_pfn, nr_to_free);
1533 free_base_page = NULL;
1534 free_base_pfn = nr_to_free = 0;
1535
Mel Gorman54608c32015-06-30 14:57:09 -07001536 page = pfn_to_page(pfn);
1537 cond_resched();
1538 }
Mel Gorman7e18adb2015-06-30 14:57:05 -07001539
1540 if (page->flags) {
1541 VM_BUG_ON(page_zone(page) != zone);
Mel Gormana4de83d2015-06-30 14:57:16 -07001542 goto free_range;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001543 }
1544
1545 __init_single_page(page, pfn, zid, nid);
Mel Gormana4de83d2015-06-30 14:57:16 -07001546 if (!free_base_page) {
1547 free_base_page = page;
1548 free_base_pfn = pfn;
1549 nr_to_free = 0;
1550 }
1551 nr_to_free++;
1552
1553 /* Where possible, batch up pages for a single free */
1554 continue;
1555free_range:
1556 /* Free the current block of pages to allocator */
1557 nr_pages += nr_to_free;
1558 deferred_free_range(free_base_page, free_base_pfn,
1559 nr_to_free);
1560 free_base_page = NULL;
1561 free_base_pfn = nr_to_free = 0;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001562 }
Xishi Qiue7801492016-10-07 16:58:09 -07001563 /* Free the last block of pages to allocator */
1564 nr_pages += nr_to_free;
1565 deferred_free_range(free_base_page, free_base_pfn, nr_to_free);
Mel Gormana4de83d2015-06-30 14:57:16 -07001566
Mel Gorman7e18adb2015-06-30 14:57:05 -07001567 first_init_pfn = max(end_pfn, first_init_pfn);
1568 }
1569
1570 /* Sanity check that the next zone really is unpopulated */
1571 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1572
Mel Gorman0e1cc952015-06-30 14:57:27 -07001573 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
Mel Gorman7e18adb2015-06-30 14:57:05 -07001574 jiffies_to_msecs(jiffies - start));
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001575
1576 pgdat_init_report_one_done();
Mel Gorman0e1cc952015-06-30 14:57:27 -07001577 return 0;
1578}
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001579#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
Mel Gorman0e1cc952015-06-30 14:57:27 -07001580
1581void __init page_alloc_init_late(void)
1582{
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001583 struct zone *zone;
1584
1585#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Mel Gorman0e1cc952015-06-30 14:57:27 -07001586 int nid;
1587
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001588 /* There will be num_node_state(N_MEMORY) threads */
1589 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
Mel Gorman0e1cc952015-06-30 14:57:27 -07001590 for_each_node_state(nid, N_MEMORY) {
Mel Gorman0e1cc952015-06-30 14:57:27 -07001591 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
1592 }
1593
1594 /* Block until all are initialised */
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001595 wait_for_completion(&pgdat_init_all_done_comp);
Mel Gorman4248b0d2015-08-06 15:46:20 -07001596
1597 /* Reinit limits that are based on free pages after the kernel is up */
1598 files_maxfiles_init();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001599#endif
Pavel Tatashin87395ee2017-08-18 15:16:05 -07001600#ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
1601 /* Discard memblock private memory */
1602 memblock_discard();
1603#endif
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001604
1605 for_each_populated_zone(zone)
1606 set_zone_contiguous(zone);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001607}
Mel Gorman7e18adb2015-06-30 14:57:05 -07001608
Michal Nazarewicz47118af2011-12-29 13:09:50 +01001609#ifdef CONFIG_CMA
Li Zhong9cf510a2013-08-23 13:52:52 +08001610/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01001611void __init init_cma_reserved_pageblock(struct page *page)
1612{
1613 unsigned i = pageblock_nr_pages;
1614 struct page *p = page;
1615
1616 do {
1617 __ClearPageReserved(p);
1618 set_page_count(p, 0);
1619 } while (++p, --i);
1620
Michal Nazarewicz47118af2011-12-29 13:09:50 +01001621 set_pageblock_migratetype(page, MIGRATE_CMA);
Michal Nazarewiczdc783272014-07-02 15:22:35 -07001622
1623 if (pageblock_order >= MAX_ORDER) {
1624 i = pageblock_nr_pages;
1625 p = page;
1626 do {
1627 set_page_refcounted(p);
1628 __free_pages(p, MAX_ORDER - 1);
1629 p += MAX_ORDER_NR_PAGES;
1630 } while (i -= MAX_ORDER_NR_PAGES);
1631 } else {
1632 set_page_refcounted(page);
1633 __free_pages(page, pageblock_order);
1634 }
1635
Jiang Liu3dcc0572013-07-03 15:03:21 -07001636 adjust_managed_page_count(page, pageblock_nr_pages);
Michal Nazarewicz47118af2011-12-29 13:09:50 +01001637}
1638#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639
1640/*
1641 * The order of subdivision here is critical for the IO subsystem.
1642 * Please do not alter this order without good reasons and regression
1643 * testing. Specifically, as large blocks of memory are subdivided,
1644 * the order in which smaller blocks are delivered depends on the order
1645 * they're subdivided in this function. This is the primary factor
1646 * influencing the order in which pages are delivered to the IO
1647 * subsystem according to empirical testing, and this is also justified
1648 * by considering the behavior of a buddy system containing a single
1649 * large block of memory acted on by a series of small allocations.
1650 * This behavior is a critical factor in sglist merging's success.
1651 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +01001652 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653 */
Nick Piggin085cc7d2006-01-06 00:11:01 -08001654static inline void expand(struct zone *zone, struct page *page,
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001655 int low, int high, struct free_area *area,
1656 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657{
1658 unsigned long size = 1 << high;
1659
1660 while (high > low) {
1661 area--;
1662 high--;
1663 size >>= 1;
Sasha Levin309381fea2014-01-23 15:52:54 -08001664 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08001665
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07001666 /*
1667 * Mark as guard pages (or page), that will allow to
1668 * merge back to allocator when buddy will be freed.
1669 * Corresponding page table entries will not be touched,
1670 * pages will stay not present in virtual address space
1671 */
1672 if (set_page_guard(zone, &page[size], high, migratetype))
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08001673 continue;
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07001674
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001675 list_add(&page[size].lru, &area->free_list[migratetype]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 area->nr_free++;
1677 set_page_order(&page[size], high);
1678 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679}
1680
Vlastimil Babka4e611802016-05-19 17:14:41 -07001681static void check_new_page_bad(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682{
Vlastimil Babka4e611802016-05-19 17:14:41 -07001683 const char *bad_reason = NULL;
1684 unsigned long bad_flags = 0;
Dave Hansenf0b791a2014-01-23 15:52:49 -08001685
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08001686 if (unlikely(atomic_read(&page->_mapcount) != -1))
Dave Hansenf0b791a2014-01-23 15:52:49 -08001687 bad_reason = "nonzero mapcount";
1688 if (unlikely(page->mapping != NULL))
1689 bad_reason = "non-NULL mapping";
Joonsoo Kimfe896d12016-03-17 14:19:26 -07001690 if (unlikely(page_ref_count(page) != 0))
Dave Hansenf0b791a2014-01-23 15:52:49 -08001691 bad_reason = "nonzero _count";
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07001692 if (unlikely(page->flags & __PG_HWPOISON)) {
1693 bad_reason = "HWPoisoned (hardware-corrupted)";
1694 bad_flags = __PG_HWPOISON;
Naoya Horiguchie570f562016-05-20 16:58:50 -07001695 /* Don't complain about hwpoisoned pages */
1696 page_mapcount_reset(page); /* remove PageBuddy */
1697 return;
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07001698 }
Dave Hansenf0b791a2014-01-23 15:52:49 -08001699 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
1700 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
1701 bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
1702 }
Johannes Weiner9edad6e2014-12-10 15:44:58 -08001703#ifdef CONFIG_MEMCG
1704 if (unlikely(page->mem_cgroup))
1705 bad_reason = "page still charged to cgroup";
1706#endif
Vlastimil Babka4e611802016-05-19 17:14:41 -07001707 bad_page(page, bad_reason, bad_flags);
1708}
1709
1710/*
1711 * This page is about to be returned from the page allocator
1712 */
1713static inline int check_new_page(struct page *page)
1714{
1715 if (likely(page_expected_state(page,
1716 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
1717 return 0;
1718
1719 check_new_page_bad(page);
1720 return 1;
Wu Fengguang2a7684a2009-09-16 11:50:12 +02001721}
1722
Laura Abbott1414c7f2016-03-15 14:56:30 -07001723static inline bool free_pages_prezeroed(bool poisoned)
1724{
1725 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
1726 page_poisoning_enabled() && poisoned;
1727}
1728
Mel Gorman479f8542016-05-19 17:14:35 -07001729#ifdef CONFIG_DEBUG_VM
1730static bool check_pcp_refill(struct page *page)
1731{
1732 return false;
1733}
1734
1735static bool check_new_pcp(struct page *page)
1736{
1737 return check_new_page(page);
1738}
1739#else
1740static bool check_pcp_refill(struct page *page)
1741{
1742 return check_new_page(page);
1743}
1744static bool check_new_pcp(struct page *page)
1745{
1746 return false;
1747}
1748#endif /* CONFIG_DEBUG_VM */
1749
1750static bool check_new_pages(struct page *page, unsigned int order)
1751{
1752 int i;
1753 for (i = 0; i < (1 << order); i++) {
1754 struct page *p = page + i;
1755
1756 if (unlikely(check_new_page(p)))
1757 return true;
1758 }
1759
1760 return false;
1761}
1762
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07001763inline void post_alloc_hook(struct page *page, unsigned int order,
1764 gfp_t gfp_flags)
1765{
1766 set_page_private(page, 0);
1767 set_page_refcounted(page);
1768
1769 arch_alloc_page(page, order);
1770 kernel_map_pages(page, 1 << order, 1);
1771 kernel_poison_pages(page, 1 << order, 1);
1772 kasan_alloc_pages(page, order);
1773 set_page_owner(page, order, gfp_flags);
1774}
1775
Mel Gorman479f8542016-05-19 17:14:35 -07001776static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Mel Gormanc6038442016-05-19 17:13:38 -07001777 unsigned int alloc_flags)
Wu Fengguang2a7684a2009-09-16 11:50:12 +02001778{
1779 int i;
Laura Abbott1414c7f2016-03-15 14:56:30 -07001780 bool poisoned = true;
Wu Fengguang2a7684a2009-09-16 11:50:12 +02001781
1782 for (i = 0; i < (1 << order); i++) {
1783 struct page *p = page + i;
Laura Abbott1414c7f2016-03-15 14:56:30 -07001784 if (poisoned)
1785 poisoned &= page_is_poisoned(p);
Wu Fengguang2a7684a2009-09-16 11:50:12 +02001786 }
Hugh Dickins689bceb2005-11-21 21:32:20 -08001787
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07001788 post_alloc_hook(page, order, gfp_flags);
Nick Piggin17cf4402006-03-22 00:08:41 -08001789
Laura Abbott1414c7f2016-03-15 14:56:30 -07001790 if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO))
Anisse Astierf4d28972015-06-24 16:56:36 -07001791 for (i = 0; i < (1 << order); i++)
1792 clear_highpage(page + i);
Nick Piggin17cf4402006-03-22 00:08:41 -08001793
1794 if (order && (gfp_flags & __GFP_COMP))
1795 prep_compound_page(page, order);
1796
Vlastimil Babka75379192015-02-11 15:25:38 -08001797 /*
Michal Hocko2f064f32015-08-21 14:11:51 -07001798 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
Vlastimil Babka75379192015-02-11 15:25:38 -08001799 * allocate the page. The expectation is that the caller is taking
1800 * steps that will free more memory. The caller should avoid the page
1801 * being used for !PFMEMALLOC purposes.
1802 */
Michal Hocko2f064f32015-08-21 14:11:51 -07001803 if (alloc_flags & ALLOC_NO_WATERMARKS)
1804 set_page_pfmemalloc(page);
1805 else
1806 clear_page_pfmemalloc(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807}
1808
Mel Gorman56fd56b2007-10-16 01:25:58 -07001809/*
1810 * Go through the free lists for the given migratetype and remove
1811 * the smallest available page from the freelists
1812 */
Mel Gorman728ec982009-06-16 15:32:04 -07001813static inline
1814struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
Mel Gorman56fd56b2007-10-16 01:25:58 -07001815 int migratetype)
1816{
1817 unsigned int current_order;
Pintu Kumarb8af2942013-09-11 14:20:34 -07001818 struct free_area *area;
Mel Gorman56fd56b2007-10-16 01:25:58 -07001819 struct page *page;
1820
1821 /* Find a page of the appropriate size in the preferred list */
1822 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
1823 area = &(zone->free_area[current_order]);
Geliang Tanga16601c2016-01-14 15:20:30 -08001824 page = list_first_entry_or_null(&area->free_list[migratetype],
Mel Gorman56fd56b2007-10-16 01:25:58 -07001825 struct page, lru);
Geliang Tanga16601c2016-01-14 15:20:30 -08001826 if (!page)
1827 continue;
Mel Gorman56fd56b2007-10-16 01:25:58 -07001828 list_del(&page->lru);
1829 rmv_page_order(page);
1830 area->nr_free--;
Mel Gorman56fd56b2007-10-16 01:25:58 -07001831 expand(zone, page, order, current_order, area, migratetype);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07001832 set_pcppage_migratetype(page, migratetype);
Mel Gorman56fd56b2007-10-16 01:25:58 -07001833 return page;
1834 }
1835
1836 return NULL;
1837}
1838
1839
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001840/*
1841 * This array describes the order lists are fallen back to when
1842 * the free lists for the desirable migrate type are depleted
1843 */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01001844static int fallbacks[MIGRATE_TYPES][4] = {
Mel Gorman974a7862015-11-06 16:28:34 -08001845 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
1846 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
1847 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
Joonsoo Kimdc676472015-04-14 15:45:15 -07001848#ifdef CONFIG_CMA
Mel Gorman974a7862015-11-06 16:28:34 -08001849 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01001850#endif
Minchan Kim194159f2013-02-22 16:33:58 -08001851#ifdef CONFIG_MEMORY_ISOLATION
Mel Gorman974a7862015-11-06 16:28:34 -08001852 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */
Minchan Kim194159f2013-02-22 16:33:58 -08001853#endif
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001854};
1855
Joonsoo Kimdc676472015-04-14 15:45:15 -07001856#ifdef CONFIG_CMA
1857static struct page *__rmqueue_cma_fallback(struct zone *zone,
1858 unsigned int order)
1859{
1860 return __rmqueue_smallest(zone, order, MIGRATE_CMA);
1861}
1862#else
1863static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
1864 unsigned int order) { return NULL; }
1865#endif
1866
Mel Gormanc361be52007-10-16 01:25:51 -07001867/*
1868 * Move the free pages in a range to the free lists of the requested type.
Mel Gormand9c23402007-10-16 01:26:01 -07001869 * Note that start_page and end_pages are not aligned on a pageblock
Mel Gormanc361be52007-10-16 01:25:51 -07001870 * boundary. If alignment is required, use move_freepages_block()
1871 */
Minchan Kim435b4052012-10-08 16:32:16 -07001872int move_freepages(struct zone *zone,
Adrian Bunkb69a7282008-07-23 21:28:12 -07001873 struct page *start_page, struct page *end_page,
1874 int migratetype)
Mel Gormanc361be52007-10-16 01:25:51 -07001875{
1876 struct page *page;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08001877 unsigned int order;
Mel Gormand1003132007-10-16 01:26:00 -07001878 int pages_moved = 0;
Mel Gormanc361be52007-10-16 01:25:51 -07001879
1880#ifndef CONFIG_HOLES_IN_ZONE
1881 /*
1882 * page_zone is not safe to call in this context when
1883 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
1884 * anyway as we check zone boundaries in move_freepages_block().
1885 * Remove at a later date when no bug reports exist related to
Mel Gormanac0e5b72007-10-16 01:25:58 -07001886 * grouping pages by mobility
Mel Gormanc361be52007-10-16 01:25:51 -07001887 */
Mel Gorman97ee4ba2014-10-09 15:28:28 -07001888 VM_BUG_ON(page_zone(start_page) != page_zone(end_page));
Mel Gormanc361be52007-10-16 01:25:51 -07001889#endif
1890
1891 for (page = start_page; page <= end_page;) {
1892 if (!pfn_valid_within(page_to_pfn(page))) {
1893 page++;
1894 continue;
1895 }
1896
Ard Biesheuvel22cccef2017-01-10 16:58:00 -08001897 /* Make sure we are not inadvertently changing nodes */
1898 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
1899
Mel Gormanc361be52007-10-16 01:25:51 -07001900 if (!PageBuddy(page)) {
1901 page++;
1902 continue;
1903 }
1904
1905 order = page_order(page);
Kirill A. Shutemov84be48d2011-03-22 16:33:41 -07001906 list_move(&page->lru,
1907 &zone->free_area[order].free_list[migratetype]);
Mel Gormanc361be52007-10-16 01:25:51 -07001908 page += 1 << order;
Mel Gormand1003132007-10-16 01:26:00 -07001909 pages_moved += 1 << order;
Mel Gormanc361be52007-10-16 01:25:51 -07001910 }
1911
Mel Gormand1003132007-10-16 01:26:00 -07001912 return pages_moved;
Mel Gormanc361be52007-10-16 01:25:51 -07001913}
1914
Minchan Kimee6f5092012-07-31 16:43:50 -07001915int move_freepages_block(struct zone *zone, struct page *page,
Linus Torvalds68e3e922012-06-03 20:05:57 -07001916 int migratetype)
Mel Gormanc361be52007-10-16 01:25:51 -07001917{
1918 unsigned long start_pfn, end_pfn;
1919 struct page *start_page, *end_page;
1920
1921 start_pfn = page_to_pfn(page);
Mel Gormand9c23402007-10-16 01:26:01 -07001922 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
Mel Gormanc361be52007-10-16 01:25:51 -07001923 start_page = pfn_to_page(start_pfn);
Mel Gormand9c23402007-10-16 01:26:01 -07001924 end_page = start_page + pageblock_nr_pages - 1;
1925 end_pfn = start_pfn + pageblock_nr_pages - 1;
Mel Gormanc361be52007-10-16 01:25:51 -07001926
1927 /* Do not cross zone boundaries */
Cody P Schafer108bcc92013-02-22 16:35:23 -08001928 if (!zone_spans_pfn(zone, start_pfn))
Mel Gormanc361be52007-10-16 01:25:51 -07001929 start_page = page;
Cody P Schafer108bcc92013-02-22 16:35:23 -08001930 if (!zone_spans_pfn(zone, end_pfn))
Mel Gormanc361be52007-10-16 01:25:51 -07001931 return 0;
1932
1933 return move_freepages(zone, start_page, end_page, migratetype);
1934}
1935
Mel Gorman2f66a682009-09-21 17:02:31 -07001936static void change_pageblock_range(struct page *pageblock_page,
1937 int start_order, int migratetype)
1938{
1939 int nr_pageblocks = 1 << (start_order - pageblock_order);
1940
1941 while (nr_pageblocks--) {
1942 set_pageblock_migratetype(pageblock_page, migratetype);
1943 pageblock_page += pageblock_nr_pages;
1944 }
1945}
1946
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07001947/*
Vlastimil Babka9c0415e2015-02-11 15:28:21 -08001948 * When we are falling back to another migratetype during allocation, try to
1949 * steal extra free pages from the same pageblocks to satisfy further
1950 * allocations, instead of polluting multiple pageblocks.
1951 *
1952 * If we are stealing a relatively large buddy page, it is likely there will
1953 * be more free pages in the pageblock, so try to steal them all. For
1954 * reclaimable and unmovable allocations, we steal regardless of page size,
1955 * as fragmentation caused by those allocations polluting movable pageblocks
1956 * is worse than movable allocations stealing from unmovable and reclaimable
1957 * pageblocks.
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07001958 */
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07001959static bool can_steal_fallback(unsigned int order, int start_mt)
1960{
1961 /*
1962 * Leaving this order check is intended, although there is
1963 * relaxed order check in next check. The reason is that
1964 * we can actually steal whole pageblock if this condition met,
1965 * but, below check doesn't guarantee it and that is just heuristic
1966 * so could be changed anytime.
1967 */
1968 if (order >= pageblock_order)
1969 return true;
1970
1971 if (order >= pageblock_order / 2 ||
1972 start_mt == MIGRATE_RECLAIMABLE ||
1973 start_mt == MIGRATE_UNMOVABLE ||
1974 page_group_by_mobility_disabled)
1975 return true;
1976
1977 return false;
1978}
1979
1980/*
1981 * This function implements actual steal behaviour. If order is large enough,
1982 * we can steal whole pageblock. If not, we first move freepages in this
1983 * pageblock and check whether half of pages are moved or not. If half of
1984 * pages are moved, we can change migratetype of pageblock and permanently
1985 * use it's pages as requested migratetype in the future.
1986 */
1987static void steal_suitable_fallback(struct zone *zone, struct page *page,
1988 int start_type)
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07001989{
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08001990 unsigned int current_order = page_order(page);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07001991 int pages;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07001992
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07001993 /* Take ownership for orders >= pageblock_order */
1994 if (current_order >= pageblock_order) {
1995 change_pageblock_range(page, current_order, start_type);
Vlastimil Babka3a1086f2015-02-11 15:28:18 -08001996 return;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07001997 }
1998
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07001999 pages = move_freepages_block(zone, page, start_type);
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002000
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002001 /* Claim the whole block if over half of it is free */
2002 if (pages >= (1 << (pageblock_order-1)) ||
2003 page_group_by_mobility_disabled)
2004 set_pageblock_migratetype(page, start_type);
2005}
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002006
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002007/*
2008 * Check whether there is a suitable fallback freepage with requested order.
2009 * If only_stealable is true, this function returns fallback_mt only if
2010 * we can steal other freepages all together. This would help to reduce
2011 * fragmentation due to mixed migratetype pages in one pageblock.
2012 */
2013int find_suitable_fallback(struct free_area *area, unsigned int order,
2014 int migratetype, bool only_stealable, bool *can_steal)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002015{
2016 int i;
2017 int fallback_mt;
2018
2019 if (area->nr_free == 0)
2020 return -1;
2021
2022 *can_steal = false;
2023 for (i = 0;; i++) {
2024 fallback_mt = fallbacks[migratetype][i];
Mel Gorman974a7862015-11-06 16:28:34 -08002025 if (fallback_mt == MIGRATE_TYPES)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002026 break;
2027
2028 if (list_empty(&area->free_list[fallback_mt]))
2029 continue;
2030
2031 if (can_steal_fallback(order, migratetype))
2032 *can_steal = true;
2033
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002034 if (!only_stealable)
2035 return fallback_mt;
2036
2037 if (*can_steal)
2038 return fallback_mt;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002039 }
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002040
2041 return -1;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002042}
2043
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002044/*
2045 * Reserve a pageblock for exclusive use of high-order atomic allocations if
2046 * there are no empty page blocks that contain a page with a suitable order
2047 */
2048static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2049 unsigned int alloc_order)
2050{
2051 int mt;
2052 unsigned long max_managed, flags;
2053
2054 /*
2055 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2056 * Check is race-prone but harmless.
2057 */
2058 max_managed = (zone->managed_pages / 100) + pageblock_nr_pages;
2059 if (zone->nr_reserved_highatomic >= max_managed)
2060 return;
2061
2062 spin_lock_irqsave(&zone->lock, flags);
2063
2064 /* Recheck the nr_reserved_highatomic limit under the lock */
2065 if (zone->nr_reserved_highatomic >= max_managed)
2066 goto out_unlock;
2067
2068 /* Yoink! */
2069 mt = get_pageblock_migratetype(page);
2070 if (mt != MIGRATE_HIGHATOMIC &&
2071 !is_migrate_isolate(mt) && !is_migrate_cma(mt)) {
2072 zone->nr_reserved_highatomic += pageblock_nr_pages;
2073 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2074 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC);
2075 }
2076
2077out_unlock:
2078 spin_unlock_irqrestore(&zone->lock, flags);
2079}
2080
2081/*
2082 * Used when an allocation is about to fail under memory pressure. This
2083 * potentially hurts the reliability of high-order allocations when under
2084 * intense memory pressure but failed atomic allocations should be easier
2085 * to recover from than an OOM.
2086 */
2087static void unreserve_highatomic_pageblock(const struct alloc_context *ac)
2088{
2089 struct zonelist *zonelist = ac->zonelist;
2090 unsigned long flags;
2091 struct zoneref *z;
2092 struct zone *zone;
2093 struct page *page;
2094 int order;
2095
2096 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
2097 ac->nodemask) {
2098 /* Preserve at least one pageblock */
2099 if (zone->nr_reserved_highatomic <= pageblock_nr_pages)
2100 continue;
2101
2102 spin_lock_irqsave(&zone->lock, flags);
2103 for (order = 0; order < MAX_ORDER; order++) {
2104 struct free_area *area = &(zone->free_area[order]);
2105
Geliang Tanga16601c2016-01-14 15:20:30 -08002106 page = list_first_entry_or_null(
2107 &area->free_list[MIGRATE_HIGHATOMIC],
2108 struct page, lru);
2109 if (!page)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002110 continue;
2111
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002112 /*
Minchan Kim39f56772016-12-12 16:42:08 -08002113 * In page freeing path, migratetype change is racy so
2114 * we can counter several free pages in a pageblock
2115 * in this loop althoug we changed the pageblock type
2116 * from highatomic to ac->migratetype. So we should
2117 * adjust the count once.
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002118 */
Minchan Kim39f56772016-12-12 16:42:08 -08002119 if (get_pageblock_migratetype(page) ==
2120 MIGRATE_HIGHATOMIC) {
2121 /*
2122 * It should never happen but changes to
2123 * locking could inadvertently allow a per-cpu
2124 * drain to add pages to MIGRATE_HIGHATOMIC
2125 * while unreserving so be safe and watch for
2126 * underflows.
2127 */
2128 zone->nr_reserved_highatomic -= min(
2129 pageblock_nr_pages,
2130 zone->nr_reserved_highatomic);
2131 }
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002132
2133 /*
2134 * Convert to ac->migratetype and avoid the normal
2135 * pageblock stealing heuristics. Minimally, the caller
2136 * is doing the work and needs the pages. More
2137 * importantly, if the block was always converted to
2138 * MIGRATE_UNMOVABLE or another type then the number
2139 * of pageblocks that cannot be completely freed
2140 * may increase.
2141 */
2142 set_pageblock_migratetype(page, ac->migratetype);
2143 move_freepages_block(zone, page, ac->migratetype);
2144 spin_unlock_irqrestore(&zone->lock, flags);
2145 return;
2146 }
2147 spin_unlock_irqrestore(&zone->lock, flags);
2148 }
2149}
2150
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002151/* Remove an element from the buddy allocator from the fallback list */
Mel Gorman0ac3a402009-06-16 15:32:06 -07002152static inline struct page *
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002153__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002154{
Pintu Kumarb8af2942013-09-11 14:20:34 -07002155 struct free_area *area;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002156 unsigned int current_order;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002157 struct page *page;
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002158 int fallback_mt;
2159 bool can_steal;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002160
2161 /* Find the largest possible block of pages in the other list */
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002162 for (current_order = MAX_ORDER-1;
2163 current_order >= order && current_order <= MAX_ORDER-1;
2164 --current_order) {
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002165 area = &(zone->free_area[current_order]);
2166 fallback_mt = find_suitable_fallback(area, current_order,
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002167 start_migratetype, false, &can_steal);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002168 if (fallback_mt == -1)
2169 continue;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002170
Geliang Tanga16601c2016-01-14 15:20:30 -08002171 page = list_first_entry(&area->free_list[fallback_mt],
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002172 struct page, lru);
2173 if (can_steal)
2174 steal_suitable_fallback(zone, page, start_migratetype);
Mel Gormane0104872007-10-16 01:25:53 -07002175
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002176 /* Remove the page from the freelists */
2177 area->nr_free--;
2178 list_del(&page->lru);
2179 rmv_page_order(page);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002180
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002181 expand(zone, page, order, current_order, area,
2182 start_migratetype);
2183 /*
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002184 * The pcppage_migratetype may differ from pageblock's
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002185 * migratetype depending on the decisions in
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002186 * find_suitable_fallback(). This is OK as long as it does not
2187 * differ for MIGRATE_CMA pageblocks. Those can be used as
2188 * fallback only via special __rmqueue_cma_fallback() function
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002189 */
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002190 set_pcppage_migratetype(page, start_migratetype);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002191
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002192 trace_mm_page_alloc_extfrag(page, order, current_order,
2193 start_migratetype, fallback_mt);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002194
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002195 return page;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002196 }
2197
Mel Gorman728ec982009-06-16 15:32:04 -07002198 return NULL;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002199}
2200
Mel Gorman56fd56b2007-10-16 01:25:58 -07002201/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002202 * Do the hard work of removing an element from the buddy allocator.
2203 * Call me with the zone->lock already held.
2204 */
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002205static struct page *__rmqueue(struct zone *zone, unsigned int order,
Mel Gorman6ac02062016-01-14 15:20:28 -08002206 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002207{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002208 struct page *page;
2209
Mel Gorman56fd56b2007-10-16 01:25:58 -07002210 page = __rmqueue_smallest(zone, order, migratetype);
Mel Gorman974a7862015-11-06 16:28:34 -08002211 if (unlikely(!page)) {
Joonsoo Kimdc676472015-04-14 15:45:15 -07002212 if (migratetype == MIGRATE_MOVABLE)
2213 page = __rmqueue_cma_fallback(zone, order);
2214
2215 if (!page)
2216 page = __rmqueue_fallback(zone, order, migratetype);
Mel Gorman728ec982009-06-16 15:32:04 -07002217 }
2218
Mel Gorman0d3d0622009-09-21 17:02:44 -07002219 trace_mm_page_alloc_zone_locked(page, order, migratetype);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002220 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002221}
2222
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002223/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224 * Obtain a specified number of elements from the buddy allocator, all under
2225 * a single hold of the lock, for efficiency. Add them to the supplied list.
2226 * Returns the number of new pages which were placed at *list.
2227 */
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002228static int rmqueue_bulk(struct zone *zone, unsigned int order,
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002229 unsigned long count, struct list_head *list,
Mel Gormanb745bc82014-06-04 16:10:22 -07002230 int migratetype, bool cold)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002231{
Mel Gorman44919a22016-12-12 16:44:41 -08002232 int i, alloced = 0;
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002233
Nick Pigginc54ad302006-01-06 00:10:56 -08002234 spin_lock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002235 for (i = 0; i < count; ++i) {
Mel Gorman6ac02062016-01-14 15:20:28 -08002236 struct page *page = __rmqueue(zone, order, migratetype);
Nick Piggin085cc7d2006-01-06 00:11:01 -08002237 if (unlikely(page == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002238 break;
Mel Gorman81eabcb2007-12-17 16:20:05 -08002239
Mel Gorman479f8542016-05-19 17:14:35 -07002240 if (unlikely(check_pcp_refill(page)))
2241 continue;
2242
Mel Gorman81eabcb2007-12-17 16:20:05 -08002243 /*
2244 * Split buddy pages returned by expand() are received here
2245 * in physical page order. The page is added to the callers and
2246 * list and the list head then moves forward. From the callers
2247 * perspective, the linked list is ordered by page number in
2248 * some conditions. This is useful for IO devices that can
2249 * merge IO requests if the physical pages are ordered
2250 * properly.
2251 */
Mel Gormanb745bc82014-06-04 16:10:22 -07002252 if (likely(!cold))
Mel Gormane084b2d2009-07-29 15:02:04 -07002253 list_add(&page->lru, list);
2254 else
2255 list_add_tail(&page->lru, list);
Mel Gorman81eabcb2007-12-17 16:20:05 -08002256 list = &page->lru;
Mel Gorman44919a22016-12-12 16:44:41 -08002257 alloced++;
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002258 if (is_migrate_cma(get_pcppage_migratetype(page)))
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07002259 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2260 -(1 << order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002261 }
Mel Gorman44919a22016-12-12 16:44:41 -08002262
2263 /*
2264 * i pages were removed from the buddy list even if some leak due
2265 * to check_pcp_refill failing so adjust NR_FREE_PAGES based
2266 * on i. Do not confuse with 'alloced' which is the number of
2267 * pages added to the pcp list.
2268 */
Mel Gormanf2260e62009-06-16 15:32:13 -07002269 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
Nick Pigginc54ad302006-01-06 00:10:56 -08002270 spin_unlock(&zone->lock);
Mel Gorman44919a22016-12-12 16:44:41 -08002271 return alloced;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002272}
2273
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002274#ifdef CONFIG_NUMA
Christoph Lameter8fce4d82006-03-09 17:33:54 -08002275/*
Christoph Lameter4037d452007-05-09 02:35:14 -07002276 * Called from the vmstat counter updater to drain pagesets of this
2277 * currently executing processor on remote nodes after they have
2278 * expired.
2279 *
Christoph Lameter879336c2006-03-22 00:09:08 -08002280 * Note that this function must be called with the thread pinned to
2281 * a single processor.
Christoph Lameter8fce4d82006-03-09 17:33:54 -08002282 */
Christoph Lameter4037d452007-05-09 02:35:14 -07002283void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002284{
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002285 unsigned long flags;
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07002286 int to_drain, batch;
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002287
Christoph Lameter4037d452007-05-09 02:35:14 -07002288 local_irq_save(flags);
Jason Low4db0c3c2015-04-15 16:14:08 -07002289 batch = READ_ONCE(pcp->batch);
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07002290 to_drain = min(pcp->count, batch);
KOSAKI Motohiro2a135152012-07-31 16:42:53 -07002291 if (to_drain > 0) {
2292 free_pcppages_bulk(zone, to_drain, pcp);
2293 pcp->count -= to_drain;
2294 }
Christoph Lameter4037d452007-05-09 02:35:14 -07002295 local_irq_restore(flags);
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002296}
2297#endif
2298
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002299/*
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002300 * Drain pcplists of the indicated processor and zone.
2301 *
2302 * The processor must either be the current processor and the
2303 * thread pinned to the current processor or a processor that
2304 * is not online.
2305 */
2306static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2307{
2308 unsigned long flags;
2309 struct per_cpu_pageset *pset;
2310 struct per_cpu_pages *pcp;
2311
2312 local_irq_save(flags);
2313 pset = per_cpu_ptr(zone->pageset, cpu);
2314
2315 pcp = &pset->pcp;
2316 if (pcp->count) {
2317 free_pcppages_bulk(zone, pcp->count, pcp);
2318 pcp->count = 0;
2319 }
2320 local_irq_restore(flags);
2321}
2322
2323/*
2324 * Drain pcplists of all zones on the indicated processor.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002325 *
2326 * The processor must either be the current processor and the
2327 * thread pinned to the current processor or a processor that
2328 * is not online.
2329 */
2330static void drain_pages(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002331{
2332 struct zone *zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002333
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07002334 for_each_populated_zone(zone) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002335 drain_pages_zone(cpu, zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002336 }
2337}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002339/*
2340 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002341 *
2342 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
2343 * the single zone's pages.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002344 */
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002345void drain_local_pages(struct zone *zone)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002346{
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002347 int cpu = smp_processor_id();
2348
2349 if (zone)
2350 drain_pages_zone(cpu, zone);
2351 else
2352 drain_pages(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002353}
2354
2355/*
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002356 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
2357 *
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002358 * When zone parameter is non-NULL, spill just the single zone's pages.
2359 *
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002360 * Note that this code is protected against sending an IPI to an offline
2361 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
2362 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
2363 * nothing keeps CPUs from showing up after we populated the cpumask and
2364 * before the call to on_each_cpu_mask().
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002365 */
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002366void drain_all_pages(struct zone *zone)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002367{
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002368 int cpu;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002369
2370 /*
2371 * Allocate in the BSS so we wont require allocation in
2372 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
2373 */
2374 static cpumask_t cpus_with_pcps;
2375
2376 /*
2377 * We don't care about racing with CPU hotplug event
2378 * as offline notification will cause the notified
2379 * cpu to drain that CPU pcps and on_each_cpu_mask
2380 * disables preemption as part of its processing
2381 */
2382 for_each_online_cpu(cpu) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002383 struct per_cpu_pageset *pcp;
2384 struct zone *z;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002385 bool has_pcps = false;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002386
2387 if (zone) {
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002388 pcp = per_cpu_ptr(zone->pageset, cpu);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002389 if (pcp->pcp.count)
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002390 has_pcps = true;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002391 } else {
2392 for_each_populated_zone(z) {
2393 pcp = per_cpu_ptr(z->pageset, cpu);
2394 if (pcp->pcp.count) {
2395 has_pcps = true;
2396 break;
2397 }
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002398 }
2399 }
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002400
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07002401 if (has_pcps)
2402 cpumask_set_cpu(cpu, &cpus_with_pcps);
2403 else
2404 cpumask_clear_cpu(cpu, &cpus_with_pcps);
2405 }
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002406 on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages,
2407 zone, 1);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002408}
2409
Rafael J. Wysocki296699d2007-07-29 23:27:18 +02002410#ifdef CONFIG_HIBERNATION
Linus Torvalds1da177e2005-04-16 15:20:36 -07002411
2412void mark_free_pages(struct zone *zone)
2413{
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07002414 unsigned long pfn, max_zone_pfn;
2415 unsigned long flags;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002416 unsigned int order, t;
Geliang Tang86760a22016-01-14 15:20:33 -08002417 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418
Xishi Qiu8080fc02013-09-11 14:21:45 -07002419 if (zone_is_empty(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420 return;
2421
2422 spin_lock_irqsave(&zone->lock, flags);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07002423
Cody P Schafer108bcc92013-02-22 16:35:23 -08002424 max_zone_pfn = zone_end_pfn(zone);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07002425 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
2426 if (pfn_valid(pfn)) {
Geliang Tang86760a22016-01-14 15:20:33 -08002427 page = pfn_to_page(pfn);
Joonsoo Kimba6b0972016-05-19 17:12:16 -07002428
2429 if (page_zone(page) != zone)
2430 continue;
2431
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07002432 if (!swsusp_page_is_forbidden(page))
2433 swsusp_unset_page_free(page);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07002434 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002435
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002436 for_each_migratetype_order(order, t) {
Geliang Tang86760a22016-01-14 15:20:33 -08002437 list_for_each_entry(page,
2438 &zone->free_area[order].free_list[t], lru) {
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07002439 unsigned long i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440
Geliang Tang86760a22016-01-14 15:20:33 -08002441 pfn = page_to_pfn(page);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07002442 for (i = 0; i < (1UL << order); i++)
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07002443 swsusp_set_page_free(pfn_to_page(pfn + i));
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07002444 }
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002445 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002446 spin_unlock_irqrestore(&zone->lock, flags);
2447}
Mel Gormane2c55dc2007-10-16 01:25:50 -07002448#endif /* CONFIG_PM */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449
2450/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451 * Free a 0-order page
Mel Gormanb745bc82014-06-04 16:10:22 -07002452 * cold == true ? free a cold page : free a hot page
Linus Torvalds1da177e2005-04-16 15:20:36 -07002453 */
Mel Gormanb745bc82014-06-04 16:10:22 -07002454void free_hot_cold_page(struct page *page, bool cold)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002455{
2456 struct zone *zone = page_zone(page);
2457 struct per_cpu_pages *pcp;
2458 unsigned long flags;
Mel Gormandc4b0ca2014-06-04 16:10:17 -07002459 unsigned long pfn = page_to_pfn(page);
Mel Gorman5f8dcc22009-09-21 17:03:19 -07002460 int migratetype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461
Mel Gorman4db75482016-05-19 17:14:32 -07002462 if (!free_pcp_prepare(page))
Hugh Dickins689bceb2005-11-21 21:32:20 -08002463 return;
2464
Mel Gormandc4b0ca2014-06-04 16:10:17 -07002465 migratetype = get_pfnblock_migratetype(page, pfn);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002466 set_pcppage_migratetype(page, migratetype);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467 local_irq_save(flags);
Christoph Lameterf8891e52006-06-30 01:55:45 -07002468 __count_vm_event(PGFREE);
Mel Gormanda456f12009-06-16 15:32:08 -07002469
Mel Gorman5f8dcc22009-09-21 17:03:19 -07002470 /*
2471 * We only track unmovable, reclaimable and movable on pcp lists.
2472 * Free ISOLATE pages back to the allocator because they are being
2473 * offlined but treat RESERVE as movable pages so we can get those
2474 * areas back if necessary. Otherwise, we may have to free
2475 * excessively into the page allocator
2476 */
2477 if (migratetype >= MIGRATE_PCPTYPES) {
Minchan Kim194159f2013-02-22 16:33:58 -08002478 if (unlikely(is_migrate_isolate(migratetype))) {
Mel Gormandc4b0ca2014-06-04 16:10:17 -07002479 free_one_page(zone, page, pfn, 0, migratetype);
Mel Gorman5f8dcc22009-09-21 17:03:19 -07002480 goto out;
2481 }
2482 migratetype = MIGRATE_MOVABLE;
2483 }
2484
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09002485 pcp = &this_cpu_ptr(zone->pageset)->pcp;
Mel Gormanb745bc82014-06-04 16:10:22 -07002486 if (!cold)
Mel Gorman5f8dcc22009-09-21 17:03:19 -07002487 list_add(&page->lru, &pcp->lists[migratetype]);
Mel Gormanb745bc82014-06-04 16:10:22 -07002488 else
2489 list_add_tail(&page->lru, &pcp->lists[migratetype]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490 pcp->count++;
Nick Piggin48db57f2006-01-08 01:00:42 -08002491 if (pcp->count >= pcp->high) {
Jason Low4db0c3c2015-04-15 16:14:08 -07002492 unsigned long batch = READ_ONCE(pcp->batch);
Cody P Schafer998d39c2013-07-03 15:01:32 -07002493 free_pcppages_bulk(zone, batch, pcp);
2494 pcp->count -= batch;
Nick Piggin48db57f2006-01-08 01:00:42 -08002495 }
Mel Gorman5f8dcc22009-09-21 17:03:19 -07002496
2497out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002498 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002499}
2500
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08002501/*
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08002502 * Free a list of 0-order pages
2503 */
Mel Gormanb745bc82014-06-04 16:10:22 -07002504void free_hot_cold_page_list(struct list_head *list, bool cold)
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08002505{
2506 struct page *page, *next;
2507
2508 list_for_each_entry_safe(page, next, list, lru) {
Konstantin Khlebnikovb413d482012-01-10 15:07:09 -08002509 trace_mm_page_free_batched(page, cold);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08002510 free_hot_cold_page(page, cold);
2511 }
2512}
2513
2514/*
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08002515 * split_page takes a non-compound higher-order page, and splits it into
2516 * n (1<<order) sub-pages: page[0..n]
2517 * Each sub-page must be freed individually.
2518 *
2519 * Note: this is probably too low level an operation for use in drivers.
2520 * Please consult with lkml before using this in your driver.
2521 */
2522void split_page(struct page *page, unsigned int order)
2523{
2524 int i;
2525
Sasha Levin309381fea2014-01-23 15:52:54 -08002526 VM_BUG_ON_PAGE(PageCompound(page), page);
2527 VM_BUG_ON_PAGE(!page_count(page), page);
Vegard Nossumb1eeab62008-11-25 16:55:53 +01002528
2529#ifdef CONFIG_KMEMCHECK
2530 /*
2531 * Split shadow pages too, because free(page[0]) would
2532 * otherwise free the whole shadow.
2533 */
2534 if (kmemcheck_page_is_tracked(page))
2535 split_page(virt_to_page(page[0].shadow), order);
2536#endif
2537
Joonsoo Kima9627bc2016-07-26 15:23:49 -07002538 for (i = 1; i < (1 << order); i++)
Nick Piggin7835e982006-03-22 00:08:40 -08002539 set_page_refcounted(page + i);
Joonsoo Kima9627bc2016-07-26 15:23:49 -07002540 split_page_owner(page, order);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08002541}
K. Y. Srinivasan5853ff22013-03-25 15:47:38 -07002542EXPORT_SYMBOL_GPL(split_page);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08002543
Joonsoo Kim3c605092014-11-13 15:19:21 -08002544int __isolate_free_page(struct page *page, unsigned int order)
Mel Gorman748446b2010-05-24 14:32:27 -07002545{
Mel Gorman748446b2010-05-24 14:32:27 -07002546 unsigned long watermark;
2547 struct zone *zone;
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07002548 int mt;
Mel Gorman748446b2010-05-24 14:32:27 -07002549
2550 BUG_ON(!PageBuddy(page));
2551
2552 zone = page_zone(page);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08002553 mt = get_pageblock_migratetype(page);
Mel Gorman748446b2010-05-24 14:32:27 -07002554
Minchan Kim194159f2013-02-22 16:33:58 -08002555 if (!is_migrate_isolate(mt)) {
Vlastimil Babka8348faf2016-10-07 16:58:00 -07002556 /*
2557 * Obey watermarks as if the page was being allocated. We can
2558 * emulate a high-order watermark check with a raised order-0
2559 * watermark, because we already know our high-order page
2560 * exists.
2561 */
2562 watermark = min_wmark_pages(zone) + (1UL << order);
Vlastimil Babka984fdba2016-10-07 16:57:57 -07002563 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08002564 return 0;
2565
Mel Gorman8fb74b92013-01-11 14:32:16 -08002566 __mod_zone_freepage_state(zone, -(1UL << order), mt);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08002567 }
Mel Gorman748446b2010-05-24 14:32:27 -07002568
2569 /* Remove page from free list */
2570 list_del(&page->lru);
2571 zone->free_area[order].nr_free--;
2572 rmv_page_order(page);
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07002573
zhong jiang400bc7f2016-07-28 15:45:07 -07002574 /*
2575 * Set the pageblock if the isolated page is at least half of a
2576 * pageblock
2577 */
Mel Gorman748446b2010-05-24 14:32:27 -07002578 if (order >= pageblock_order - 1) {
2579 struct page *endpage = page + (1 << order) - 1;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002580 for (; page < endpage; page += pageblock_nr_pages) {
2581 int mt = get_pageblock_migratetype(page);
Minchan Kim194159f2013-02-22 16:33:58 -08002582 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt))
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002583 set_pageblock_migratetype(page,
2584 MIGRATE_MOVABLE);
2585 }
Mel Gorman748446b2010-05-24 14:32:27 -07002586 }
2587
Joonsoo Kimf3a14ce2015-07-17 16:24:15 -07002588
Mel Gorman8fb74b92013-01-11 14:32:16 -08002589 return 1UL << order;
Mel Gorman1fb3f8c2012-10-08 16:29:12 -07002590}
2591
2592/*
Mel Gorman060e7412016-05-19 17:13:27 -07002593 * Update NUMA hit/miss statistics
2594 *
2595 * Must be called with interrupts disabled.
Mel Gorman060e7412016-05-19 17:13:27 -07002596 */
2597static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
2598 gfp_t flags)
2599{
2600#ifdef CONFIG_NUMA
Mel Gorman060e7412016-05-19 17:13:27 -07002601 enum zone_stat_item local_stat = NUMA_LOCAL;
2602
Michal Hockofae478c2017-01-10 16:57:39 -08002603 if (z->node != numa_node_id())
Mel Gorman060e7412016-05-19 17:13:27 -07002604 local_stat = NUMA_OTHER;
Mel Gorman060e7412016-05-19 17:13:27 -07002605
Michal Hockofae478c2017-01-10 16:57:39 -08002606 if (z->node == preferred_zone->node)
Mel Gorman060e7412016-05-19 17:13:27 -07002607 __inc_zone_state(z, NUMA_HIT);
Michal Hockofae478c2017-01-10 16:57:39 -08002608 else {
Mel Gorman060e7412016-05-19 17:13:27 -07002609 __inc_zone_state(z, NUMA_MISS);
2610 __inc_zone_state(preferred_zone, NUMA_FOREIGN);
2611 }
Michal Hockofae478c2017-01-10 16:57:39 -08002612 __inc_zone_state(z, local_stat);
Mel Gorman060e7412016-05-19 17:13:27 -07002613#endif
2614}
2615
2616/*
Vlastimil Babka75379192015-02-11 15:25:38 -08002617 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002618 */
Mel Gorman0a15c3e2009-06-16 15:32:05 -07002619static inline
2620struct page *buffered_rmqueue(struct zone *preferred_zone,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002621 struct zone *zone, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07002622 gfp_t gfp_flags, unsigned int alloc_flags,
2623 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002624{
2625 unsigned long flags;
Hugh Dickins689bceb2005-11-21 21:32:20 -08002626 struct page *page;
Mel Gormanb745bc82014-06-04 16:10:22 -07002627 bool cold = ((gfp_flags & __GFP_COLD) != 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002628
Nick Piggin48db57f2006-01-08 01:00:42 -08002629 if (likely(order == 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630 struct per_cpu_pages *pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07002631 struct list_head *list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002632
Linus Torvalds1da177e2005-04-16 15:20:36 -07002633 local_irq_save(flags);
Mel Gorman479f8542016-05-19 17:14:35 -07002634 do {
2635 pcp = &this_cpu_ptr(zone->pageset)->pcp;
2636 list = &pcp->lists[migratetype];
2637 if (list_empty(list)) {
2638 pcp->count += rmqueue_bulk(zone, 0,
2639 pcp->batch, list,
2640 migratetype, cold);
2641 if (unlikely(list_empty(list)))
2642 goto failed;
2643 }
Mel Gormanb92a6ed2007-10-16 01:25:50 -07002644
Mel Gorman479f8542016-05-19 17:14:35 -07002645 if (cold)
2646 page = list_last_entry(list, struct page, lru);
2647 else
2648 page = list_first_entry(list, struct page, lru);
Mel Gorman5f8dcc22009-09-21 17:03:19 -07002649
Vlastimil Babka83b93552016-06-03 14:55:52 -07002650 list_del(&page->lru);
2651 pcp->count--;
2652
2653 } while (check_new_pcp(page));
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002654 } else {
Michal Hocko0f352e52016-03-17 14:19:32 -07002655 /*
2656 * We most definitely don't want callers attempting to
2657 * allocate greater than order-1 page units with __GFP_NOFAIL.
2658 */
2659 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002660 spin_lock_irqsave(&zone->lock, flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002661
Mel Gorman479f8542016-05-19 17:14:35 -07002662 do {
2663 page = NULL;
2664 if (alloc_flags & ALLOC_HARDER) {
2665 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
2666 if (page)
2667 trace_mm_page_alloc_zone_locked(page, order, migratetype);
2668 }
2669 if (!page)
2670 page = __rmqueue(zone, order, migratetype);
2671 } while (page && check_new_pages(page, order));
Nick Piggina74609f2006-01-06 00:11:20 -08002672 spin_unlock(&zone->lock);
2673 if (!page)
2674 goto failed;
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07002675 __mod_zone_freepage_state(zone, -(1 << order),
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002676 get_pcppage_migratetype(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677 }
2678
Mel Gorman16709d12016-07-28 15:46:56 -07002679 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
Andi Kleen78afd562011-03-22 16:33:12 -07002680 zone_statistics(preferred_zone, zone, gfp_flags);
Nick Piggina74609f2006-01-06 00:11:20 -08002681 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002682
Sasha Levin309381fea2014-01-23 15:52:54 -08002683 VM_BUG_ON_PAGE(bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002684 return page;
Nick Piggina74609f2006-01-06 00:11:20 -08002685
2686failed:
2687 local_irq_restore(flags);
Nick Piggina74609f2006-01-06 00:11:20 -08002688 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002689}
2690
Akinobu Mita933e3122006-12-08 02:39:45 -08002691#ifdef CONFIG_FAIL_PAGE_ALLOC
2692
Akinobu Mitab2588c42011-07-26 16:09:03 -07002693static struct {
Akinobu Mita933e3122006-12-08 02:39:45 -08002694 struct fault_attr attr;
2695
Viresh Kumar621a5f72015-09-26 15:04:07 -07002696 bool ignore_gfp_highmem;
Mel Gorman71baba42015-11-06 16:28:28 -08002697 bool ignore_gfp_reclaim;
Akinobu Mita54114992007-07-15 23:40:23 -07002698 u32 min_order;
Akinobu Mita933e3122006-12-08 02:39:45 -08002699} fail_page_alloc = {
2700 .attr = FAULT_ATTR_INITIALIZER,
Mel Gorman71baba42015-11-06 16:28:28 -08002701 .ignore_gfp_reclaim = true,
Viresh Kumar621a5f72015-09-26 15:04:07 -07002702 .ignore_gfp_highmem = true,
Akinobu Mita54114992007-07-15 23:40:23 -07002703 .min_order = 1,
Akinobu Mita933e3122006-12-08 02:39:45 -08002704};
2705
2706static int __init setup_fail_page_alloc(char *str)
2707{
2708 return setup_fault_attr(&fail_page_alloc.attr, str);
2709}
2710__setup("fail_page_alloc=", setup_fail_page_alloc);
2711
Gavin Shandeaf3862012-07-31 16:41:51 -07002712static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08002713{
Akinobu Mita54114992007-07-15 23:40:23 -07002714 if (order < fail_page_alloc.min_order)
Gavin Shandeaf3862012-07-31 16:41:51 -07002715 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08002716 if (gfp_mask & __GFP_NOFAIL)
Gavin Shandeaf3862012-07-31 16:41:51 -07002717 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08002718 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
Gavin Shandeaf3862012-07-31 16:41:51 -07002719 return false;
Mel Gorman71baba42015-11-06 16:28:28 -08002720 if (fail_page_alloc.ignore_gfp_reclaim &&
2721 (gfp_mask & __GFP_DIRECT_RECLAIM))
Gavin Shandeaf3862012-07-31 16:41:51 -07002722 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08002723
2724 return should_fail(&fail_page_alloc.attr, 1 << order);
2725}
2726
2727#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
2728
2729static int __init fail_page_alloc_debugfs(void)
2730{
Al Virof4ae40a2011-07-24 04:33:43 -04002731 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
Akinobu Mita933e3122006-12-08 02:39:45 -08002732 struct dentry *dir;
Akinobu Mita933e3122006-12-08 02:39:45 -08002733
Akinobu Mitadd48c082011-08-03 16:21:01 -07002734 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
2735 &fail_page_alloc.attr);
2736 if (IS_ERR(dir))
2737 return PTR_ERR(dir);
Akinobu Mita933e3122006-12-08 02:39:45 -08002738
Akinobu Mitab2588c42011-07-26 16:09:03 -07002739 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
Mel Gorman71baba42015-11-06 16:28:28 -08002740 &fail_page_alloc.ignore_gfp_reclaim))
Akinobu Mitab2588c42011-07-26 16:09:03 -07002741 goto fail;
2742 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
2743 &fail_page_alloc.ignore_gfp_highmem))
2744 goto fail;
2745 if (!debugfs_create_u32("min-order", mode, dir,
2746 &fail_page_alloc.min_order))
2747 goto fail;
Akinobu Mita933e3122006-12-08 02:39:45 -08002748
Akinobu Mitab2588c42011-07-26 16:09:03 -07002749 return 0;
2750fail:
Akinobu Mitadd48c082011-08-03 16:21:01 -07002751 debugfs_remove_recursive(dir);
Akinobu Mita933e3122006-12-08 02:39:45 -08002752
Akinobu Mitab2588c42011-07-26 16:09:03 -07002753 return -ENOMEM;
Akinobu Mita933e3122006-12-08 02:39:45 -08002754}
2755
2756late_initcall(fail_page_alloc_debugfs);
2757
2758#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
2759
2760#else /* CONFIG_FAIL_PAGE_ALLOC */
2761
Gavin Shandeaf3862012-07-31 16:41:51 -07002762static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08002763{
Gavin Shandeaf3862012-07-31 16:41:51 -07002764 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08002765}
2766
2767#endif /* CONFIG_FAIL_PAGE_ALLOC */
2768
Linus Torvalds1da177e2005-04-16 15:20:36 -07002769/*
Mel Gorman97a16fc2015-11-06 16:28:40 -08002770 * Return true if free base pages are above 'mark'. For high-order checks it
2771 * will return true of the order-0 watermark is reached and there is at least
2772 * one free page of a suitable size. Checking now avoids taking the zone lock
2773 * to check in the allocation paths if no pages are free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002774 */
Michal Hocko86a294a2016-05-20 16:57:12 -07002775bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
2776 int classzone_idx, unsigned int alloc_flags,
2777 long free_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002778{
Christoph Lameterd23ad422007-02-10 01:43:02 -08002779 long min = mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002780 int o;
Mel Gormanc6038442016-05-19 17:13:38 -07002781 const bool alloc_harder = (alloc_flags & ALLOC_HARDER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002782
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002783 /* free_pages may go negative - that's OK */
Michal Hockodf0a6da2012-01-10 15:08:02 -08002784 free_pages -= (1 << order) - 1;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002785
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002786 if (alloc_flags & ALLOC_HIGH)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002787 min -= min / 2;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002788
2789 /*
2790 * If the caller does not have rights to ALLOC_HARDER then subtract
2791 * the high-atomic reserves. This will over-estimate the size of the
2792 * atomic reserve but it avoids a search.
2793 */
Mel Gorman97a16fc2015-11-06 16:28:40 -08002794 if (likely(!alloc_harder))
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002795 free_pages -= z->nr_reserved_highatomic;
2796 else
Linus Torvalds1da177e2005-04-16 15:20:36 -07002797 min -= min / 4;
Mel Gormane2b19192015-11-06 16:28:09 -08002798
Bartlomiej Zolnierkiewiczd95ea5d2012-10-08 16:32:05 -07002799#ifdef CONFIG_CMA
2800 /* If allocation can't use CMA areas don't use free CMA pages */
2801 if (!(alloc_flags & ALLOC_CMA))
Mel Gorman97a16fc2015-11-06 16:28:40 -08002802 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
Bartlomiej Zolnierkiewiczd95ea5d2012-10-08 16:32:05 -07002803#endif
Tomasz Stanislawski026b0812013-06-12 14:05:02 -07002804
Mel Gorman97a16fc2015-11-06 16:28:40 -08002805 /*
2806 * Check watermarks for an order-0 allocation request. If these
2807 * are not met, then a high-order request also cannot go ahead
2808 * even if a suitable page happened to be free.
2809 */
2810 if (free_pages <= min + z->lowmem_reserve[classzone_idx])
Mel Gorman88f5acf2011-01-13 15:45:41 -08002811 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002812
Mel Gorman97a16fc2015-11-06 16:28:40 -08002813 /* If this is an order-0 request then the watermark is fine */
2814 if (!order)
2815 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002816
Mel Gorman97a16fc2015-11-06 16:28:40 -08002817 /* For a high-order request, check at least one suitable page is free */
2818 for (o = order; o < MAX_ORDER; o++) {
2819 struct free_area *area = &z->free_area[o];
2820 int mt;
2821
2822 if (!area->nr_free)
2823 continue;
2824
Mel Gorman97a16fc2015-11-06 16:28:40 -08002825 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
2826 if (!list_empty(&area->free_list[mt]))
2827 return true;
2828 }
2829
2830#ifdef CONFIG_CMA
2831 if ((alloc_flags & ALLOC_CMA) &&
2832 !list_empty(&area->free_list[MIGRATE_CMA])) {
2833 return true;
2834 }
2835#endif
Vlastimil Babka685cce52017-11-15 17:38:30 -08002836 if (alloc_harder &&
2837 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
2838 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839 }
Mel Gorman97a16fc2015-11-06 16:28:40 -08002840 return false;
Mel Gorman88f5acf2011-01-13 15:45:41 -08002841}
2842
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002843bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
Mel Gormanc6038442016-05-19 17:13:38 -07002844 int classzone_idx, unsigned int alloc_flags)
Mel Gorman88f5acf2011-01-13 15:45:41 -08002845{
2846 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
2847 zone_page_state(z, NR_FREE_PAGES));
2848}
2849
Mel Gorman48ee5f32016-05-19 17:14:07 -07002850static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
2851 unsigned long mark, int classzone_idx, unsigned int alloc_flags)
2852{
2853 long free_pages = zone_page_state(z, NR_FREE_PAGES);
2854 long cma_pages = 0;
2855
2856#ifdef CONFIG_CMA
2857 /* If allocation can't use CMA areas don't use free CMA pages */
2858 if (!(alloc_flags & ALLOC_CMA))
2859 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
2860#endif
2861
2862 /*
2863 * Fast check for order-0 only. If this fails then the reserves
2864 * need to be calculated. There is a corner case where the check
2865 * passes but only the high-order atomic reserve are free. If
2866 * the caller is !atomic then it'll uselessly search the free
2867 * list. That corner case is then slower but it is harmless.
2868 */
2869 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
2870 return true;
2871
2872 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
2873 free_pages);
2874}
2875
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002876bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
Mel Gormane2b19192015-11-06 16:28:09 -08002877 unsigned long mark, int classzone_idx)
Mel Gorman88f5acf2011-01-13 15:45:41 -08002878{
2879 long free_pages = zone_page_state(z, NR_FREE_PAGES);
2880
2881 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
2882 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
2883
Mel Gormane2b19192015-11-06 16:28:09 -08002884 return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
Mel Gorman88f5acf2011-01-13 15:45:41 -08002885 free_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002886}
2887
Paul Jackson9276b1bc2006-12-06 20:31:48 -08002888#ifdef CONFIG_NUMA
David Rientjes957f8222012-10-08 16:33:24 -07002889static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
2890{
Gavin Shand1e80422017-02-24 14:59:33 -08002891 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
Mel Gorman5f7a75a2014-06-04 16:07:15 -07002892 RECLAIM_DISTANCE;
David Rientjes957f8222012-10-08 16:33:24 -07002893}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08002894#else /* CONFIG_NUMA */
David Rientjes957f8222012-10-08 16:33:24 -07002895static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
2896{
2897 return true;
2898}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08002899#endif /* CONFIG_NUMA */
2900
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002901/*
Paul Jackson0798e512006-12-06 20:31:38 -08002902 * get_page_from_freelist goes through the zonelist trying to allocate
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002903 * a page.
2904 */
2905static struct page *
Vlastimil Babkaa9263752015-02-11 15:25:41 -08002906get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
2907 const struct alloc_context *ac)
Martin Hicks753ee722005-06-21 17:14:41 -07002908{
Mel Gormanc33d6c02016-05-19 17:14:10 -07002909 struct zoneref *z = ac->preferred_zoneref;
Mel Gorman5117f452009-06-16 15:31:59 -07002910 struct zone *zone;
Mel Gorman3b8c0be2016-07-28 15:46:53 -07002911 struct pglist_data *last_pgdat_dirty_limit = NULL;
2912
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002913 /*
Paul Jackson9276b1bc2006-12-06 20:31:48 -08002914 * Scan zonelist, looking for a zone with enough free.
Vladimir Davydov344736f2014-10-20 15:50:30 +04002915 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002916 */
Mel Gormanc33d6c02016-05-19 17:14:10 -07002917 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08002918 ac->nodemask) {
Mel Gormanbe06af02016-05-19 17:13:47 -07002919 struct page *page;
Johannes Weinere085dbc2013-09-11 14:20:46 -07002920 unsigned long mark;
2921
Mel Gorman664eedd2014-06-04 16:10:08 -07002922 if (cpusets_enabled() &&
2923 (alloc_flags & ALLOC_CPUSET) &&
Vlastimil Babka002f2902016-05-19 17:14:30 -07002924 !__cpuset_zone_allowed(zone, gfp_mask))
Mel Gormancd38b112011-07-25 17:12:29 -07002925 continue;
Johannes Weinera756cf52012-01-10 15:07:49 -08002926 /*
2927 * When allocating a page cache page for writing, we
Mel Gorman281e3722016-07-28 15:46:11 -07002928 * want to get it from a node that is within its dirty
2929 * limit, such that no single node holds more than its
Johannes Weinera756cf52012-01-10 15:07:49 -08002930 * proportional share of globally allowed dirty pages.
Mel Gorman281e3722016-07-28 15:46:11 -07002931 * The dirty limits take into account the node's
Johannes Weinera756cf52012-01-10 15:07:49 -08002932 * lowmem reserves and high watermark so that kswapd
2933 * should be able to balance it without having to
2934 * write pages from its LRU list.
2935 *
Johannes Weinera756cf52012-01-10 15:07:49 -08002936 * XXX: For now, allow allocations to potentially
Mel Gorman281e3722016-07-28 15:46:11 -07002937 * exceed the per-node dirty limit in the slowpath
Mel Gormanc9ab0c42015-11-06 16:28:12 -08002938 * (spread_dirty_pages unset) before going into reclaim,
Johannes Weinera756cf52012-01-10 15:07:49 -08002939 * which is important when on a NUMA setup the allowed
Mel Gorman281e3722016-07-28 15:46:11 -07002940 * nodes are together not big enough to reach the
Johannes Weinera756cf52012-01-10 15:07:49 -08002941 * global limit. The proper fix for these situations
Mel Gorman281e3722016-07-28 15:46:11 -07002942 * will require awareness of nodes in the
Johannes Weinera756cf52012-01-10 15:07:49 -08002943 * dirty-throttling and the flusher threads.
2944 */
Mel Gorman3b8c0be2016-07-28 15:46:53 -07002945 if (ac->spread_dirty_pages) {
2946 if (last_pgdat_dirty_limit == zone->zone_pgdat)
2947 continue;
2948
2949 if (!node_dirty_ok(zone->zone_pgdat)) {
2950 last_pgdat_dirty_limit = zone->zone_pgdat;
2951 continue;
2952 }
2953 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002954
Johannes Weinere085dbc2013-09-11 14:20:46 -07002955 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
Mel Gorman48ee5f32016-05-19 17:14:07 -07002956 if (!zone_watermark_fast(zone, order, mark,
Mel Gorman93ea9962016-05-19 17:14:13 -07002957 ac_classzone_idx(ac), alloc_flags)) {
Mel Gormanfa5e0842009-06-16 15:33:22 -07002958 int ret;
2959
Mel Gorman5dab2912014-06-04 16:10:14 -07002960 /* Checked here to keep the fast path fast */
2961 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
2962 if (alloc_flags & ALLOC_NO_WATERMARKS)
2963 goto try_this_zone;
2964
Mel Gormana5f5f912016-07-28 15:46:32 -07002965 if (node_reclaim_mode == 0 ||
Mel Gormanc33d6c02016-05-19 17:14:10 -07002966 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
Mel Gormancd38b112011-07-25 17:12:29 -07002967 continue;
2968
Mel Gormana5f5f912016-07-28 15:46:32 -07002969 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
Mel Gormanfa5e0842009-06-16 15:33:22 -07002970 switch (ret) {
Mel Gormana5f5f912016-07-28 15:46:32 -07002971 case NODE_RECLAIM_NOSCAN:
Mel Gormanfa5e0842009-06-16 15:33:22 -07002972 /* did not scan */
Mel Gormancd38b112011-07-25 17:12:29 -07002973 continue;
Mel Gormana5f5f912016-07-28 15:46:32 -07002974 case NODE_RECLAIM_FULL:
Mel Gormanfa5e0842009-06-16 15:33:22 -07002975 /* scanned but unreclaimable */
Mel Gormancd38b112011-07-25 17:12:29 -07002976 continue;
Mel Gormanfa5e0842009-06-16 15:33:22 -07002977 default:
2978 /* did we reclaim enough */
Mel Gormanfed27192013-04-29 15:07:57 -07002979 if (zone_watermark_ok(zone, order, mark,
Mel Gorman93ea9962016-05-19 17:14:13 -07002980 ac_classzone_idx(ac), alloc_flags))
Mel Gormanfed27192013-04-29 15:07:57 -07002981 goto try_this_zone;
2982
Mel Gormanfed27192013-04-29 15:07:57 -07002983 continue;
Paul Jackson0798e512006-12-06 20:31:38 -08002984 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002985 }
2986
Mel Gormanfa5e0842009-06-16 15:33:22 -07002987try_this_zone:
Mel Gormanc33d6c02016-05-19 17:14:10 -07002988 page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order,
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002989 gfp_mask, alloc_flags, ac->migratetype);
Vlastimil Babka75379192015-02-11 15:25:38 -08002990 if (page) {
Mel Gorman479f8542016-05-19 17:14:35 -07002991 prep_new_page(page, order, gfp_mask, alloc_flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002992
2993 /*
2994 * If this is a high-order atomic allocation then check
2995 * if the pageblock should be reserved for the future
2996 */
2997 if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
2998 reserve_highatomic_pageblock(page, zone, order);
2999
Vlastimil Babka75379192015-02-11 15:25:38 -08003000 return page;
3001 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07003002 }
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003003
Mel Gorman4ffeaf32014-08-06 16:07:22 -07003004 return NULL;
Martin Hicks753ee722005-06-21 17:14:41 -07003005}
3006
David Rientjes29423e772011-03-22 16:30:47 -07003007/*
3008 * Large machines with many possible nodes should not always dump per-node
3009 * meminfo in irq context.
3010 */
3011static inline bool should_suppress_show_mem(void)
3012{
3013 bool ret = false;
3014
3015#if NODES_SHIFT > 8
3016 ret = in_interrupt();
3017#endif
3018 return ret;
3019}
3020
Dave Hansena238ab52011-05-24 17:12:16 -07003021static DEFINE_RATELIMIT_STATE(nopage_rs,
3022 DEFAULT_RATELIMIT_INTERVAL,
3023 DEFAULT_RATELIMIT_BURST);
3024
Michal Hocko7877cdc2016-10-07 17:01:55 -07003025void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
Dave Hansena238ab52011-05-24 17:12:16 -07003026{
Dave Hansena238ab52011-05-24 17:12:16 -07003027 unsigned int filter = SHOW_MEM_FILTER_NODES;
Michal Hocko7877cdc2016-10-07 17:01:55 -07003028 struct va_format vaf;
3029 va_list args;
Dave Hansena238ab52011-05-24 17:12:16 -07003030
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08003031 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
3032 debug_guardpage_minorder() > 0)
Dave Hansena238ab52011-05-24 17:12:16 -07003033 return;
3034
3035 /*
3036 * This documents exceptions given to allocations in certain
3037 * contexts that are allowed to allocate outside current's set
3038 * of allowed nodes.
3039 */
3040 if (!(gfp_mask & __GFP_NOMEMALLOC))
3041 if (test_thread_flag(TIF_MEMDIE) ||
3042 (current->flags & (PF_MEMALLOC | PF_EXITING)))
3043 filter &= ~SHOW_MEM_FILTER_NODES;
Mel Gormand0164ad2015-11-06 16:28:21 -08003044 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
Dave Hansena238ab52011-05-24 17:12:16 -07003045 filter &= ~SHOW_MEM_FILTER_NODES;
3046
Michal Hocko7877cdc2016-10-07 17:01:55 -07003047 pr_warn("%s: ", current->comm);
Joe Perches3ee9a4f2011-10-31 17:08:35 -07003048
Michal Hocko7877cdc2016-10-07 17:01:55 -07003049 va_start(args, fmt);
3050 vaf.fmt = fmt;
3051 vaf.va = &args;
3052 pr_cont("%pV", &vaf);
3053 va_end(args);
Joe Perches3ee9a4f2011-10-31 17:08:35 -07003054
Michal Hocko7877cdc2016-10-07 17:01:55 -07003055 pr_cont(", mode:%#x(%pGg)\n", gfp_mask, &gfp_mask);
Joe Perches3ee9a4f2011-10-31 17:08:35 -07003056
Dave Hansena238ab52011-05-24 17:12:16 -07003057 dump_stack();
3058 if (!should_suppress_show_mem())
3059 show_mem(filter);
3060}
3061
Mel Gorman11e33f62009-06-16 15:31:57 -07003062static inline struct page *
3063__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003064 const struct alloc_context *ac, unsigned long *did_some_progress)
Mel Gorman11e33f62009-06-16 15:31:57 -07003065{
David Rientjes6e0fc462015-09-08 15:00:36 -07003066 struct oom_control oc = {
3067 .zonelist = ac->zonelist,
3068 .nodemask = ac->nodemask,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07003069 .memcg = NULL,
David Rientjes6e0fc462015-09-08 15:00:36 -07003070 .gfp_mask = gfp_mask,
3071 .order = order,
David Rientjes6e0fc462015-09-08 15:00:36 -07003072 };
Linus Torvalds1da177e2005-04-16 15:20:36 -07003073 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074
Johannes Weiner9879de72015-01-26 12:58:32 -08003075 *did_some_progress = 0;
3076
Johannes Weiner9879de72015-01-26 12:58:32 -08003077 /*
Johannes Weinerdc564012015-06-24 16:57:19 -07003078 * Acquire the oom lock. If that fails, somebody else is
3079 * making progress for us.
Johannes Weiner9879de72015-01-26 12:58:32 -08003080 */
Johannes Weinerdc564012015-06-24 16:57:19 -07003081 if (!mutex_trylock(&oom_lock)) {
Johannes Weiner9879de72015-01-26 12:58:32 -08003082 *did_some_progress = 1;
Mel Gorman11e33f62009-06-16 15:31:57 -07003083 schedule_timeout_uninterruptible(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003084 return NULL;
3085 }
Jens Axboe6b1de912005-11-17 21:35:02 +01003086
Mel Gorman11e33f62009-06-16 15:31:57 -07003087 /*
3088 * Go through the zonelist yet one more time, keep very high watermark
3089 * here, this is only to catch a parallel oom killing, we must fail if
3090 * we're still under heavy pressure.
3091 */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003092 page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order,
3093 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003094 if (page)
Mel Gorman11e33f62009-06-16 15:31:57 -07003095 goto out;
3096
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -08003097 if (!(gfp_mask & __GFP_NOFAIL)) {
Johannes Weiner9879de72015-01-26 12:58:32 -08003098 /* Coredumps can quickly deplete all memory reserves */
3099 if (current->flags & PF_DUMPCORE)
3100 goto out;
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -08003101 /* The OOM killer will not help higher order allocs */
3102 if (order > PAGE_ALLOC_COSTLY_ORDER)
3103 goto out;
David Rientjes03668b32010-08-09 17:18:54 -07003104 /* The OOM killer does not needlessly kill tasks for lowmem */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003105 if (ac->high_zoneidx < ZONE_NORMAL)
David Rientjes03668b32010-08-09 17:18:54 -07003106 goto out;
Johannes Weiner90839052015-06-24 16:57:21 -07003107 if (pm_suspended_storage())
3108 goto out;
Michal Hocko3da88fb2016-05-19 17:13:09 -07003109 /*
3110 * XXX: GFP_NOFS allocations should rather fail than rely on
3111 * other request to make a forward progress.
3112 * We are in an unfortunate situation where out_of_memory cannot
3113 * do much for this context but let's try it to at least get
3114 * access to memory reserved if the current task is killed (see
3115 * out_of_memory). Once filesystems are ready to handle allocation
3116 * failures more gracefully we should just bail out here.
3117 */
3118
David Rientjes4167e9b2015-04-14 15:46:55 -07003119 /* The OOM killer may not free memory on a specific node */
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -08003120 if (gfp_mask & __GFP_THISNODE)
3121 goto out;
3122 }
Mel Gorman11e33f62009-06-16 15:31:57 -07003123 /* Exhausted what can be done so it's blamo time */
Michal Hocko5020e282016-01-14 15:20:36 -08003124 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
Michal Hockoc32b3cb2015-02-11 15:26:24 -08003125 *did_some_progress = 1;
Michal Hocko5020e282016-01-14 15:20:36 -08003126
3127 if (gfp_mask & __GFP_NOFAIL) {
3128 page = get_page_from_freelist(gfp_mask, order,
3129 ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac);
3130 /*
3131 * fallback to ignore cpuset restriction if our nodes
3132 * are depleted
3133 */
3134 if (!page)
3135 page = get_page_from_freelist(gfp_mask, order,
3136 ALLOC_NO_WATERMARKS, ac);
3137 }
3138 }
Mel Gorman11e33f62009-06-16 15:31:57 -07003139out:
Johannes Weinerdc564012015-06-24 16:57:19 -07003140 mutex_unlock(&oom_lock);
Mel Gorman11e33f62009-06-16 15:31:57 -07003141 return page;
3142}
3143
Michal Hocko33c2d212016-05-20 16:57:06 -07003144/*
3145 * Maximum number of compaction retries wit a progress before OOM
3146 * killer is consider as the only way to move forward.
3147 */
3148#define MAX_COMPACT_RETRIES 16
3149
Mel Gorman56de7262010-05-24 14:32:30 -07003150#ifdef CONFIG_COMPACTION
3151/* Try memory compaction for high-order allocations before reclaim */
3152static struct page *
3153__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07003154 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07003155 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07003156{
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07003157 struct page *page;
Vlastimil Babka4e434d42017-05-08 15:59:46 -07003158 unsigned int noreclaim_flag = current->flags & PF_MEMALLOC;
Vlastimil Babka53853e22014-10-09 15:27:02 -07003159
Mel Gorman66199712012-01-12 17:19:41 -08003160 if (!order)
Mel Gorman56de7262010-05-24 14:32:30 -07003161 return NULL;
3162
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08003163 current->flags |= PF_MEMALLOC;
Michal Hockoc5d01d02016-05-20 16:56:53 -07003164 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
Vlastimil Babkac3486f52016-07-28 15:49:30 -07003165 prio);
Vlastimil Babka4e434d42017-05-08 15:59:46 -07003166 current->flags = (current->flags & ~PF_MEMALLOC) | noreclaim_flag;
Mel Gorman56de7262010-05-24 14:32:30 -07003167
Michal Hockoc5d01d02016-05-20 16:56:53 -07003168 if (*compact_result <= COMPACT_INACTIVE)
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07003169 return NULL;
Mel Gorman56de7262010-05-24 14:32:30 -07003170
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07003171 /*
3172 * At least in one zone compaction wasn't deferred or skipped, so let's
3173 * count a compaction stall
3174 */
3175 count_vm_event(COMPACTSTALL);
3176
Vlastimil Babka31a6c192016-07-28 15:49:13 -07003177 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07003178
3179 if (page) {
3180 struct zone *zone = page_zone(page);
3181
3182 zone->compact_blockskip_flush = false;
3183 compaction_defer_reset(zone, order, true);
3184 count_vm_event(COMPACTSUCCESS);
3185 return page;
3186 }
3187
3188 /*
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07003189 * It's bad if compaction run occurs and fails. The most likely reason
3190 * is that pages exist, but not enough to satisfy watermarks.
3191 */
3192 count_vm_event(COMPACTFAIL);
3193
3194 cond_resched();
3195
Mel Gorman56de7262010-05-24 14:32:30 -07003196 return NULL;
3197}
Michal Hocko33c2d212016-05-20 16:57:06 -07003198
Vlastimil Babka32508452016-10-07 17:00:28 -07003199static inline bool
3200should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
3201 enum compact_result compact_result,
3202 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07003203 int *compaction_retries)
Vlastimil Babka32508452016-10-07 17:00:28 -07003204{
3205 int max_retries = MAX_COMPACT_RETRIES;
Vlastimil Babkac2033b02016-10-07 17:00:34 -07003206 int min_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07003207
3208 if (!order)
3209 return false;
3210
Vlastimil Babkad9436492016-10-07 17:00:31 -07003211 if (compaction_made_progress(compact_result))
3212 (*compaction_retries)++;
3213
Vlastimil Babka32508452016-10-07 17:00:28 -07003214 /*
3215 * compaction considers all the zone as desperately out of memory
3216 * so it doesn't really make much sense to retry except when the
3217 * failure could be caused by insufficient priority
3218 */
Vlastimil Babkad9436492016-10-07 17:00:31 -07003219 if (compaction_failed(compact_result))
3220 goto check_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07003221
3222 /*
3223 * make sure the compaction wasn't deferred or didn't bail out early
3224 * due to locks contention before we declare that we should give up.
3225 * But do not retry if the given zonelist is not suitable for
3226 * compaction.
3227 */
3228 if (compaction_withdrawn(compact_result))
3229 return compaction_zonelist_suitable(ac, order, alloc_flags);
3230
3231 /*
3232 * !costly requests are much more important than __GFP_REPEAT
3233 * costly ones because they are de facto nofail and invoke OOM
3234 * killer to move on while costly can fail and users are ready
3235 * to cope with that. 1/4 retries is rather arbitrary but we
3236 * would need much more detailed feedback from compaction to
3237 * make a better decision.
3238 */
3239 if (order > PAGE_ALLOC_COSTLY_ORDER)
3240 max_retries /= 4;
Vlastimil Babkad9436492016-10-07 17:00:31 -07003241 if (*compaction_retries <= max_retries)
Vlastimil Babka32508452016-10-07 17:00:28 -07003242 return true;
3243
Vlastimil Babkad9436492016-10-07 17:00:31 -07003244 /*
3245 * Make sure there are attempts at the highest priority if we exhausted
3246 * all retries or failed at the lower priorities.
3247 */
3248check_priority:
Vlastimil Babkac2033b02016-10-07 17:00:34 -07003249 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
3250 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
3251 if (*compact_priority > min_priority) {
Vlastimil Babkad9436492016-10-07 17:00:31 -07003252 (*compact_priority)--;
3253 *compaction_retries = 0;
3254 return true;
3255 }
Vlastimil Babka32508452016-10-07 17:00:28 -07003256 return false;
3257}
Mel Gorman56de7262010-05-24 14:32:30 -07003258#else
3259static inline struct page *
3260__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07003261 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07003262 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07003263{
Michal Hocko33c2d212016-05-20 16:57:06 -07003264 *compact_result = COMPACT_SKIPPED;
Mel Gorman56de7262010-05-24 14:32:30 -07003265 return NULL;
3266}
Michal Hocko33c2d212016-05-20 16:57:06 -07003267
3268static inline bool
Michal Hocko86a294a2016-05-20 16:57:12 -07003269should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
3270 enum compact_result compact_result,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07003271 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07003272 int *compaction_retries)
Michal Hocko33c2d212016-05-20 16:57:06 -07003273{
Michal Hocko31e49bf2016-05-20 16:57:15 -07003274 struct zone *zone;
3275 struct zoneref *z;
3276
3277 if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
3278 return false;
3279
3280 /*
3281 * There are setups with compaction disabled which would prefer to loop
3282 * inside the allocator rather than hit the oom killer prematurely.
3283 * Let's give them a good hope and keep retrying while the order-0
3284 * watermarks are OK.
3285 */
3286 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3287 ac->nodemask) {
3288 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
3289 ac_classzone_idx(ac), alloc_flags))
3290 return true;
3291 }
Michal Hocko33c2d212016-05-20 16:57:06 -07003292 return false;
3293}
Vlastimil Babka32508452016-10-07 17:00:28 -07003294#endif /* CONFIG_COMPACTION */
Mel Gorman56de7262010-05-24 14:32:30 -07003295
Marek Szyprowskibba90712012-01-25 12:09:52 +01003296/* Perform direct synchronous page reclaim */
3297static int
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003298__perform_reclaim(gfp_t gfp_mask, unsigned int order,
3299 const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07003300{
Mel Gorman11e33f62009-06-16 15:31:57 -07003301 struct reclaim_state reclaim_state;
Marek Szyprowskibba90712012-01-25 12:09:52 +01003302 int progress;
Mel Gorman11e33f62009-06-16 15:31:57 -07003303
3304 cond_resched();
3305
3306 /* We now go into synchronous reclaim */
3307 cpuset_memory_pressure_bump();
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08003308 current->flags |= PF_MEMALLOC;
Mel Gorman11e33f62009-06-16 15:31:57 -07003309 lockdep_set_current_reclaim_state(gfp_mask);
3310 reclaim_state.reclaimed_slab = 0;
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08003311 current->reclaim_state = &reclaim_state;
Mel Gorman11e33f62009-06-16 15:31:57 -07003312
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003313 progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
3314 ac->nodemask);
Mel Gorman11e33f62009-06-16 15:31:57 -07003315
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08003316 current->reclaim_state = NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07003317 lockdep_clear_current_reclaim_state();
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08003318 current->flags &= ~PF_MEMALLOC;
Mel Gorman11e33f62009-06-16 15:31:57 -07003319
3320 cond_resched();
3321
Marek Szyprowskibba90712012-01-25 12:09:52 +01003322 return progress;
3323}
3324
3325/* The really slow allocator path where we enter direct reclaim */
3326static inline struct page *
3327__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07003328 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003329 unsigned long *did_some_progress)
Marek Szyprowskibba90712012-01-25 12:09:52 +01003330{
3331 struct page *page = NULL;
3332 bool drained = false;
3333
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003334 *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07003335 if (unlikely(!(*did_some_progress)))
3336 return NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07003337
Mel Gorman9ee493c2010-09-09 16:38:18 -07003338retry:
Vlastimil Babka31a6c192016-07-28 15:49:13 -07003339 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07003340
3341 /*
3342 * If an allocation failed after direct reclaim, it could be because
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003343 * pages are pinned on the per-cpu lists or in high alloc reserves.
3344 * Shrink them them and try again
Mel Gorman9ee493c2010-09-09 16:38:18 -07003345 */
3346 if (!page && !drained) {
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003347 unreserve_highatomic_pageblock(ac);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003348 drain_all_pages(NULL);
Mel Gorman9ee493c2010-09-09 16:38:18 -07003349 drained = true;
3350 goto retry;
3351 }
3352
Mel Gorman11e33f62009-06-16 15:31:57 -07003353 return page;
3354}
3355
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003356static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07003357{
3358 struct zoneref *z;
3359 struct zone *zone;
Mel Gormane1a55632016-07-28 15:46:26 -07003360 pg_data_t *last_pgdat = NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07003361
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003362 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
Mel Gormane1a55632016-07-28 15:46:26 -07003363 ac->high_zoneidx, ac->nodemask) {
3364 if (last_pgdat != zone->zone_pgdat)
Mel Gorman52e9f872016-07-28 15:46:29 -07003365 wakeup_kswapd(zone, order, ac->high_zoneidx);
Mel Gormane1a55632016-07-28 15:46:26 -07003366 last_pgdat = zone->zone_pgdat;
3367 }
Mel Gorman11e33f62009-06-16 15:31:57 -07003368}
3369
Mel Gormanc6038442016-05-19 17:13:38 -07003370static inline unsigned int
Peter Zijlstra341ce062009-06-16 15:32:02 -07003371gfp_to_alloc_flags(gfp_t gfp_mask)
3372{
Mel Gormanc6038442016-05-19 17:13:38 -07003373 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
Peter Zijlstra341ce062009-06-16 15:32:02 -07003374
Mel Gormana56f57f2009-06-16 15:32:02 -07003375 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
Namhyung Kime6223a32010-10-26 14:21:59 -07003376 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
Mel Gormana56f57f2009-06-16 15:32:02 -07003377
Peter Zijlstra341ce062009-06-16 15:32:02 -07003378 /*
3379 * The caller may dip into page reserves a bit more if the caller
3380 * cannot run direct reclaim, or if the caller has realtime scheduling
3381 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
Mel Gormand0164ad2015-11-06 16:28:21 -08003382 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
Peter Zijlstra341ce062009-06-16 15:32:02 -07003383 */
Namhyung Kime6223a32010-10-26 14:21:59 -07003384 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
Peter Zijlstra341ce062009-06-16 15:32:02 -07003385
Mel Gormand0164ad2015-11-06 16:28:21 -08003386 if (gfp_mask & __GFP_ATOMIC) {
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08003387 /*
David Rientjesb104a352014-07-30 16:08:24 -07003388 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
3389 * if it can't schedule.
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08003390 */
David Rientjesb104a352014-07-30 16:08:24 -07003391 if (!(gfp_mask & __GFP_NOMEMALLOC))
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08003392 alloc_flags |= ALLOC_HARDER;
Peter Zijlstra341ce062009-06-16 15:32:02 -07003393 /*
David Rientjesb104a352014-07-30 16:08:24 -07003394 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
Vladimir Davydov344736f2014-10-20 15:50:30 +04003395 * comment for __cpuset_node_allowed().
Peter Zijlstra341ce062009-06-16 15:32:02 -07003396 */
3397 alloc_flags &= ~ALLOC_CPUSET;
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08003398 } else if (unlikely(rt_task(current)) && !in_interrupt())
Peter Zijlstra341ce062009-06-16 15:32:02 -07003399 alloc_flags |= ALLOC_HARDER;
3400
Bartlomiej Zolnierkiewiczd95ea5d2012-10-08 16:32:05 -07003401#ifdef CONFIG_CMA
David Rientjes43e7a342014-10-09 15:27:25 -07003402 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
Bartlomiej Zolnierkiewiczd95ea5d2012-10-08 16:32:05 -07003403 alloc_flags |= ALLOC_CMA;
3404#endif
Peter Zijlstra341ce062009-06-16 15:32:02 -07003405 return alloc_flags;
3406}
3407
Mel Gorman072bb0a2012-07-31 16:43:58 -07003408bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
3409{
Vlastimil Babka31a6c192016-07-28 15:49:13 -07003410 if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
3411 return false;
3412
3413 if (gfp_mask & __GFP_MEMALLOC)
3414 return true;
3415 if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
3416 return true;
3417 if (!in_interrupt() &&
3418 ((current->flags & PF_MEMALLOC) ||
3419 unlikely(test_thread_flag(TIF_MEMDIE))))
3420 return true;
3421
3422 return false;
Mel Gorman072bb0a2012-07-31 16:43:58 -07003423}
3424
Michal Hocko0a0337e2016-05-20 16:57:00 -07003425/*
Michal Hocko0a0337e2016-05-20 16:57:00 -07003426 * Checks whether it makes sense to retry the reclaim to make a forward progress
3427 * for the given allocation request.
3428 * The reclaim feedback represented by did_some_progress (any progress during
Michal Hocko7854ea62016-05-20 16:57:09 -07003429 * the last reclaim round) and no_progress_loops (number of reclaim rounds without
3430 * any progress in a row) is considered as well as the reclaimable pages on the
3431 * applicable zone list (with a backoff mechanism which is a function of
3432 * no_progress_loops).
Michal Hocko0a0337e2016-05-20 16:57:00 -07003433 *
3434 * Returns true if a retry is viable or false to enter the oom path.
3435 */
3436static inline bool
3437should_reclaim_retry(gfp_t gfp_mask, unsigned order,
3438 struct alloc_context *ac, int alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07003439 bool did_some_progress, int *no_progress_loops)
Michal Hocko0a0337e2016-05-20 16:57:00 -07003440{
3441 struct zone *zone;
3442 struct zoneref *z;
3443
3444 /*
Vlastimil Babka423b4522016-10-07 17:00:40 -07003445 * Costly allocations might have made a progress but this doesn't mean
3446 * their order will become available due to high fragmentation so
3447 * always increment the no progress counter for them
3448 */
3449 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
3450 *no_progress_loops = 0;
3451 else
3452 (*no_progress_loops)++;
3453
3454 /*
Michal Hocko0a0337e2016-05-20 16:57:00 -07003455 * Make sure we converge to OOM if we cannot make any progress
3456 * several times in the row.
3457 */
Vlastimil Babka423b4522016-10-07 17:00:40 -07003458 if (*no_progress_loops > MAX_RECLAIM_RETRIES)
Michal Hocko0a0337e2016-05-20 16:57:00 -07003459 return false;
3460
Michal Hocko0a0337e2016-05-20 16:57:00 -07003461 /*
Mel Gormanbca67592016-07-28 15:47:05 -07003462 * Keep reclaiming pages while there is a chance this will lead
3463 * somewhere. If none of the target zones can satisfy our allocation
3464 * request even if all reclaimable pages are considered then we are
3465 * screwed and have to go OOM.
Michal Hocko0a0337e2016-05-20 16:57:00 -07003466 */
3467 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3468 ac->nodemask) {
3469 unsigned long available;
Michal Hockoede37712016-05-20 16:57:03 -07003470 unsigned long reclaimable;
Michal Hocko0a0337e2016-05-20 16:57:00 -07003471
Mel Gorman5a1c84b2016-07-28 15:47:31 -07003472 available = reclaimable = zone_reclaimable_pages(zone);
Vlastimil Babka423b4522016-10-07 17:00:40 -07003473 available -= DIV_ROUND_UP((*no_progress_loops) * available,
Michal Hocko0a0337e2016-05-20 16:57:00 -07003474 MAX_RECLAIM_RETRIES);
Mel Gorman5a1c84b2016-07-28 15:47:31 -07003475 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
Michal Hocko0a0337e2016-05-20 16:57:00 -07003476
3477 /*
3478 * Would the allocation succeed if we reclaimed the whole
Mel Gorman5a1c84b2016-07-28 15:47:31 -07003479 * available?
Michal Hocko0a0337e2016-05-20 16:57:00 -07003480 */
Mel Gorman5a1c84b2016-07-28 15:47:31 -07003481 if (__zone_watermark_ok(zone, order, min_wmark_pages(zone),
3482 ac_classzone_idx(ac), alloc_flags, available)) {
Michal Hockoede37712016-05-20 16:57:03 -07003483 /*
3484 * If we didn't make any progress and have a lot of
3485 * dirty + writeback pages then we should wait for
3486 * an IO to complete to slow down the reclaim and
3487 * prevent from pre mature OOM
3488 */
3489 if (!did_some_progress) {
Mel Gorman11fb9982016-07-28 15:46:20 -07003490 unsigned long write_pending;
Michal Hockoede37712016-05-20 16:57:03 -07003491
Mel Gorman5a1c84b2016-07-28 15:47:31 -07003492 write_pending = zone_page_state_snapshot(zone,
3493 NR_ZONE_WRITE_PENDING);
Michal Hockoede37712016-05-20 16:57:03 -07003494
Mel Gorman11fb9982016-07-28 15:46:20 -07003495 if (2 * write_pending > reclaimable) {
Michal Hockoede37712016-05-20 16:57:03 -07003496 congestion_wait(BLK_RW_ASYNC, HZ/10);
3497 return true;
3498 }
3499 }
Mel Gorman5a1c84b2016-07-28 15:47:31 -07003500
Michal Hockoede37712016-05-20 16:57:03 -07003501 /*
3502 * Memory allocation/reclaim might be called from a WQ
3503 * context and the current implementation of the WQ
3504 * concurrency control doesn't recognize that
3505 * a particular WQ is congested if the worker thread is
3506 * looping without ever sleeping. Therefore we have to
3507 * do a short sleep here rather than calling
3508 * cond_resched().
3509 */
3510 if (current->flags & PF_WQ_WORKER)
3511 schedule_timeout_uninterruptible(1);
3512 else
3513 cond_resched();
3514
Michal Hocko0a0337e2016-05-20 16:57:00 -07003515 return true;
3516 }
3517 }
3518
3519 return false;
3520}
3521
Mel Gorman11e33f62009-06-16 15:31:57 -07003522static inline struct page *
3523__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003524 struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07003525{
Mel Gormand0164ad2015-11-06 16:28:21 -08003526 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
Mel Gorman11e33f62009-06-16 15:31:57 -07003527 struct page *page = NULL;
Mel Gormanc6038442016-05-19 17:13:38 -07003528 unsigned int alloc_flags;
Mel Gorman11e33f62009-06-16 15:31:57 -07003529 unsigned long did_some_progress;
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003530 enum compact_priority compact_priority;
Michal Hockoc5d01d02016-05-20 16:56:53 -07003531 enum compact_result compact_result;
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003532 int compaction_retries;
3533 int no_progress_loops;
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003534 unsigned int cpuset_mems_cookie;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003535
Christoph Lameter952f3b52006-12-06 20:33:26 -08003536 /*
Mel Gorman72807a72009-06-16 15:32:18 -07003537 * In the slowpath, we sanity check order to avoid ever trying to
3538 * reclaim >= MAX_ORDER areas which will never succeed. Callers may
3539 * be using allocators in order of preference for an area that is
3540 * too large.
3541 */
Mel Gorman1fc28b72009-07-29 15:04:08 -07003542 if (order >= MAX_ORDER) {
3543 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
Mel Gorman72807a72009-06-16 15:32:18 -07003544 return NULL;
Mel Gorman1fc28b72009-07-29 15:04:08 -07003545 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003546
Christoph Lameter952f3b52006-12-06 20:33:26 -08003547 /*
Mel Gormand0164ad2015-11-06 16:28:21 -08003548 * We also sanity check to catch abuse of atomic reserves being used by
3549 * callers that are not in atomic context.
3550 */
3551 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
3552 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
3553 gfp_mask &= ~__GFP_ATOMIC;
3554
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003555retry_cpuset:
3556 compaction_retries = 0;
3557 no_progress_loops = 0;
3558 compact_priority = DEF_COMPACT_PRIORITY;
3559 cpuset_mems_cookie = read_mems_allowed_begin();
Vlastimil Babka96e5cec2017-01-24 15:18:41 -08003560 /*
3561 * We need to recalculate the starting point for the zonelist iterator
3562 * because we might have used different nodemask in the fast path, or
3563 * there was a cpuset modification and we are retrying - otherwise we
3564 * could end up iterating over non-eligible zones endlessly.
3565 */
3566 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
3567 ac->high_zoneidx, ac->nodemask);
3568 if (!ac->preferred_zoneref->zone)
3569 goto nopage;
3570
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003571
Vlastimil Babka31a6c192016-07-28 15:49:13 -07003572 /*
3573 * The fast path uses conservative alloc_flags to succeed only until
3574 * kswapd needs to be woken up, and to avoid the cost of setting up
3575 * alloc_flags precisely. So we do that now.
3576 */
3577 alloc_flags = gfp_to_alloc_flags(gfp_mask);
3578
Mel Gormand0164ad2015-11-06 16:28:21 -08003579 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003580 wake_all_kswapds(order, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003581
Paul Jackson9bf22292005-09-06 15:18:12 -07003582 /*
Vlastimil Babka23771232016-07-28 15:49:16 -07003583 * The adjusted alloc_flags might result in immediate success, so try
3584 * that first
3585 */
3586 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3587 if (page)
3588 goto got_pg;
3589
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003590 /*
3591 * For costly allocations, try direct compaction first, as it's likely
3592 * that we have enough base pages and don't need to reclaim. Don't try
3593 * that for allocations that are allowed to ignore watermarks, as the
3594 * ALLOC_NO_WATERMARKS attempt didn't yet happen.
3595 */
3596 if (can_direct_reclaim && order > PAGE_ALLOC_COSTLY_ORDER &&
3597 !gfp_pfmemalloc_allowed(gfp_mask)) {
3598 page = __alloc_pages_direct_compact(gfp_mask, order,
3599 alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07003600 INIT_COMPACT_PRIORITY,
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003601 &compact_result);
3602 if (page)
3603 goto got_pg;
3604
Vlastimil Babka3eb27712016-07-28 15:49:22 -07003605 /*
3606 * Checks for costly allocations with __GFP_NORETRY, which
3607 * includes THP page fault allocations
3608 */
3609 if (gfp_mask & __GFP_NORETRY) {
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003610 /*
3611 * If compaction is deferred for high-order allocations,
3612 * it is because sync compaction recently failed. If
3613 * this is the case and the caller requested a THP
3614 * allocation, we do not want to heavily disrupt the
3615 * system, so we fail the allocation instead of entering
3616 * direct reclaim.
3617 */
3618 if (compact_result == COMPACT_DEFERRED)
3619 goto nopage;
3620
3621 /*
Vlastimil Babka3eb27712016-07-28 15:49:22 -07003622 * Looks like reclaim/compaction is worth trying, but
3623 * sync compaction could be very expensive, so keep
Vlastimil Babka25160352016-07-28 15:49:25 -07003624 * using async compaction.
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003625 */
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07003626 compact_priority = INIT_COMPACT_PRIORITY;
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003627 }
3628 }
Vlastimil Babka23771232016-07-28 15:49:16 -07003629
3630retry:
3631 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
3632 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
3633 wake_all_kswapds(order, ac);
3634
3635 if (gfp_pfmemalloc_allowed(gfp_mask))
3636 alloc_flags = ALLOC_NO_WATERMARKS;
3637
3638 /*
Mel Gormane46e7b72016-06-03 14:56:01 -07003639 * Reset the zonelist iterators if memory policies can be ignored.
3640 * These allocations are high priority and system rather than user
3641 * orientated.
3642 */
Vlastimil Babka23771232016-07-28 15:49:16 -07003643 if (!(alloc_flags & ALLOC_CPUSET) || (alloc_flags & ALLOC_NO_WATERMARKS)) {
Mel Gormane46e7b72016-06-03 14:56:01 -07003644 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
3645 ac->high_zoneidx, ac->nodemask);
3646 }
3647
Vlastimil Babka23771232016-07-28 15:49:16 -07003648 /* Attempt with potentially adjusted zonelist and alloc_flags */
Vlastimil Babka31a6c192016-07-28 15:49:13 -07003649 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003650 if (page)
3651 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003652
Mel Gormand0164ad2015-11-06 16:28:21 -08003653 /* Caller is not willing to reclaim, we can't balance anything */
3654 if (!can_direct_reclaim) {
David Rientjesaed0a0e2014-01-21 15:51:12 -08003655 /*
Michal Hocko33d53102016-01-14 15:19:05 -08003656 * All existing users of the __GFP_NOFAIL are blockable, so warn
3657 * of any new users that actually allow this type of allocation
3658 * to fail.
David Rientjesaed0a0e2014-01-21 15:51:12 -08003659 */
3660 WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003661 goto nopage;
David Rientjesaed0a0e2014-01-21 15:51:12 -08003662 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003663
Peter Zijlstra341ce062009-06-16 15:32:02 -07003664 /* Avoid recursion of direct reclaim */
Michal Hocko33d53102016-01-14 15:19:05 -08003665 if (current->flags & PF_MEMALLOC) {
3666 /*
3667 * __GFP_NOFAIL request from this context is rather bizarre
3668 * because we cannot reclaim anything and only can loop waiting
3669 * for somebody to do a work for us.
3670 */
3671 if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3672 cond_resched();
3673 goto retry;
3674 }
Peter Zijlstra341ce062009-06-16 15:32:02 -07003675 goto nopage;
Michal Hocko33d53102016-01-14 15:19:05 -08003676 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003677
David Rientjes6583bb62009-07-29 15:02:06 -07003678 /* Avoid allocations with no watermarks from looping endlessly */
3679 if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
3680 goto nopage;
3681
David Rientjes8fe78042014-08-06 16:07:54 -07003682
Mel Gorman11e33f62009-06-16 15:31:57 -07003683 /* Try direct reclaim and then allocating */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003684 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
3685 &did_some_progress);
Mel Gorman11e33f62009-06-16 15:31:57 -07003686 if (page)
3687 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003688
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003689 /* Try direct compaction and then allocating */
3690 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07003691 compact_priority, &compact_result);
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003692 if (page)
3693 goto got_pg;
3694
Johannes Weiner90839052015-06-24 16:57:21 -07003695 /* Do not loop if specifically requested */
3696 if (gfp_mask & __GFP_NORETRY)
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003697 goto nopage;
Johannes Weiner90839052015-06-24 16:57:21 -07003698
Michal Hocko0a0337e2016-05-20 16:57:00 -07003699 /*
3700 * Do not retry costly high order allocations unless they are
3701 * __GFP_REPEAT
3702 */
3703 if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT))
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07003704 goto nopage;
Michal Hocko0a0337e2016-05-20 16:57:00 -07003705
Michal Hocko0a0337e2016-05-20 16:57:00 -07003706 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07003707 did_some_progress > 0, &no_progress_loops))
Michal Hocko0a0337e2016-05-20 16:57:00 -07003708 goto retry;
3709
Michal Hocko33c2d212016-05-20 16:57:06 -07003710 /*
3711 * It doesn't make any sense to retry for the compaction if the order-0
3712 * reclaim is not able to make any progress because the current
3713 * implementation of the compaction depends on the sufficient amount
3714 * of free memory (see __compaction_suitable)
3715 */
3716 if (did_some_progress > 0 &&
Michal Hocko86a294a2016-05-20 16:57:12 -07003717 should_compact_retry(ac, order, alloc_flags,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07003718 compact_result, &compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07003719 &compaction_retries))
Michal Hocko33c2d212016-05-20 16:57:06 -07003720 goto retry;
3721
Vlastimil Babka96e5cec2017-01-24 15:18:41 -08003722 /*
3723 * It's possible we raced with cpuset update so the OOM would be
3724 * premature (see below the nopage: label for full explanation).
3725 */
3726 if (read_mems_allowed_retry(cpuset_mems_cookie))
3727 goto retry_cpuset;
3728
Johannes Weiner90839052015-06-24 16:57:21 -07003729 /* Reclaim has failed us, start killing things */
3730 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
3731 if (page)
3732 goto got_pg;
3733
3734 /* Retry as long as the OOM killer is making progress */
Michal Hocko0a0337e2016-05-20 16:57:00 -07003735 if (did_some_progress) {
3736 no_progress_loops = 0;
Johannes Weiner90839052015-06-24 16:57:21 -07003737 goto retry;
Michal Hocko0a0337e2016-05-20 16:57:00 -07003738 }
Johannes Weiner90839052015-06-24 16:57:21 -07003739
Linus Torvalds1da177e2005-04-16 15:20:36 -07003740nopage:
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003741 /*
Vlastimil Babka96e5cec2017-01-24 15:18:41 -08003742 * When updating a task's mems_allowed or mempolicy nodemask, it is
3743 * possible to race with parallel threads in such a way that our
3744 * allocation can fail while the mask is being updated. If we are about
3745 * to fail, check if the cpuset changed during allocation and if so,
3746 * retry.
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003747 */
3748 if (read_mems_allowed_retry(cpuset_mems_cookie))
3749 goto retry_cpuset;
3750
Michal Hocko7877cdc2016-10-07 17:01:55 -07003751 warn_alloc(gfp_mask,
3752 "page allocation failure: order:%u", order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003753got_pg:
Mel Gorman072bb0a2012-07-31 16:43:58 -07003754 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003755}
Mel Gorman11e33f62009-06-16 15:31:57 -07003756
3757/*
3758 * This is the 'heart' of the zoned buddy allocator.
3759 */
3760struct page *
3761__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
3762 struct zonelist *zonelist, nodemask_t *nodemask)
3763{
Mel Gorman5bb1b162016-05-19 17:13:50 -07003764 struct page *page;
Mel Gormane6cbd7f2016-07-28 15:46:50 -07003765 unsigned int alloc_flags = ALLOC_WMARK_LOW;
Mel Gorman83d4ca82016-05-19 17:13:56 -07003766 gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003767 struct alloc_context ac = {
3768 .high_zoneidx = gfp_zone(gfp_mask),
Mel Gorman682a3382016-05-19 17:13:30 -07003769 .zonelist = zonelist,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003770 .nodemask = nodemask,
3771 .migratetype = gfpflags_to_migratetype(gfp_mask),
3772 };
Mel Gorman11e33f62009-06-16 15:31:57 -07003773
Mel Gorman682a3382016-05-19 17:13:30 -07003774 if (cpusets_enabled()) {
Mel Gorman83d4ca82016-05-19 17:13:56 -07003775 alloc_mask |= __GFP_HARDWALL;
Mel Gorman682a3382016-05-19 17:13:30 -07003776 alloc_flags |= ALLOC_CPUSET;
3777 if (!ac.nodemask)
3778 ac.nodemask = &cpuset_current_mems_allowed;
3779 }
3780
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +10003781 gfp_mask &= gfp_allowed_mask;
3782
Mel Gorman11e33f62009-06-16 15:31:57 -07003783 lockdep_trace_alloc(gfp_mask);
3784
Mel Gormand0164ad2015-11-06 16:28:21 -08003785 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
Mel Gorman11e33f62009-06-16 15:31:57 -07003786
3787 if (should_fail_alloc_page(gfp_mask, order))
3788 return NULL;
3789
3790 /*
3791 * Check the zones suitable for the gfp_mask contain at least one
3792 * valid zone. It's possible to have an empty zonelist as a result
David Rientjes4167e9b2015-04-14 15:46:55 -07003793 * of __GFP_THISNODE and a memoryless node
Mel Gorman11e33f62009-06-16 15:31:57 -07003794 */
3795 if (unlikely(!zonelist->_zonerefs->zone))
3796 return NULL;
3797
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003798 if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE)
Vlastimil Babka21bb9bd2014-10-09 15:26:51 -07003799 alloc_flags |= ALLOC_CMA;
3800
Mel Gormanc9ab0c42015-11-06 16:28:12 -08003801 /* Dirty zone balancing only done in the fast path */
3802 ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE);
3803
Mel Gormane46e7b72016-06-03 14:56:01 -07003804 /*
3805 * The preferred zone is used for statistics but crucially it is
3806 * also used as the starting point for the zonelist iterator. It
3807 * may get reset for allocations that ignore memory policies.
3808 */
Mel Gormanc33d6c02016-05-19 17:14:10 -07003809 ac.preferred_zoneref = first_zones_zonelist(ac.zonelist,
3810 ac.high_zoneidx, ac.nodemask);
Vlastimil Babkaade7afe2017-01-24 15:18:32 -08003811 if (!ac.preferred_zoneref->zone) {
Mel Gorman5bb1b162016-05-19 17:13:50 -07003812 page = NULL;
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003813 /*
3814 * This might be due to race with cpuset_current_mems_allowed
3815 * update, so make sure we retry with original nodemask in the
3816 * slow path.
3817 */
Mel Gorman4fcb0972016-05-19 17:14:01 -07003818 goto no_zone;
Mel Gorman5bb1b162016-05-19 17:13:50 -07003819 }
3820
Mel Gorman5117f452009-06-16 15:31:59 -07003821 /* First allocation attempt */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003822 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
Mel Gorman4fcb0972016-05-19 17:14:01 -07003823 if (likely(page))
3824 goto out;
Andrew Morton91fbdc02015-02-11 15:25:04 -08003825
Vlastimil Babkab678e4f2017-01-24 15:18:38 -08003826no_zone:
Mel Gorman4fcb0972016-05-19 17:14:01 -07003827 /*
3828 * Runtime PM, block IO and its error handling path can deadlock
3829 * because I/O on the device might not complete.
3830 */
3831 alloc_mask = memalloc_noio_flags(gfp_mask);
3832 ac.spread_dirty_pages = false;
Mel Gorman11e33f62009-06-16 15:31:57 -07003833
Mel Gorman47415262016-05-19 17:14:44 -07003834 /*
3835 * Restore the original nodemask if it was potentially replaced with
3836 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
3837 */
Vlastimil Babka96e5cec2017-01-24 15:18:41 -08003838 if (unlikely(ac.nodemask != nodemask))
Mel Gorman47415262016-05-19 17:14:44 -07003839 ac.nodemask = nodemask;
Vlastimil Babkad1656c52017-01-24 15:18:35 -08003840
Mel Gorman4fcb0972016-05-19 17:14:01 -07003841 page = __alloc_pages_slowpath(alloc_mask, order, &ac);
Xishi Qiu23f086f2015-02-11 15:25:07 -08003842
Mel Gorman4fcb0972016-05-19 17:14:01 -07003843out:
Vladimir Davydovc4159a72016-08-08 23:03:12 +03003844 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
3845 unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) {
3846 __free_pages(page, order);
3847 page = NULL;
Vladimir Davydov49491482016-07-26 15:24:24 -07003848 }
3849
Mel Gorman4fcb0972016-05-19 17:14:01 -07003850 if (kmemcheck_enabled && page)
3851 kmemcheck_pagealloc_alloc(page, order, gfp_mask);
3852
3853 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
3854
Mel Gorman11e33f62009-06-16 15:31:57 -07003855 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003856}
Mel Gormand2391712009-06-16 15:31:52 -07003857EXPORT_SYMBOL(__alloc_pages_nodemask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003858
3859/*
3860 * Common helper functions.
3861 */
Harvey Harrison920c7a52008-02-04 22:29:26 -08003862unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003863{
Akinobu Mita945a1112009-09-21 17:01:47 -07003864 struct page *page;
3865
3866 /*
3867 * __get_free_pages() returns a 32-bit address, which cannot represent
3868 * a highmem page
3869 */
3870 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
3871
Linus Torvalds1da177e2005-04-16 15:20:36 -07003872 page = alloc_pages(gfp_mask, order);
3873 if (!page)
3874 return 0;
3875 return (unsigned long) page_address(page);
3876}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003877EXPORT_SYMBOL(__get_free_pages);
3878
Harvey Harrison920c7a52008-02-04 22:29:26 -08003879unsigned long get_zeroed_page(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003880{
Akinobu Mita945a1112009-09-21 17:01:47 -07003881 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003882}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003883EXPORT_SYMBOL(get_zeroed_page);
3884
Harvey Harrison920c7a52008-02-04 22:29:26 -08003885void __free_pages(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003886{
Nick Pigginb5810032005-10-29 18:16:12 -07003887 if (put_page_testzero(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003888 if (order == 0)
Mel Gormanb745bc82014-06-04 16:10:22 -07003889 free_hot_cold_page(page, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003890 else
3891 __free_pages_ok(page, order);
3892 }
3893}
3894
3895EXPORT_SYMBOL(__free_pages);
3896
Harvey Harrison920c7a52008-02-04 22:29:26 -08003897void free_pages(unsigned long addr, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003898{
3899 if (addr != 0) {
Nick Piggin725d7042006-09-25 23:30:55 -07003900 VM_BUG_ON(!virt_addr_valid((void *)addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003901 __free_pages(virt_to_page((void *)addr), order);
3902 }
3903}
3904
3905EXPORT_SYMBOL(free_pages);
3906
Glauber Costa6a1a0d32012-12-18 14:22:00 -08003907/*
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07003908 * Page Fragment:
3909 * An arbitrary-length arbitrary-offset area of memory which resides
3910 * within a 0 or higher order page. Multiple fragments within that page
3911 * are individually refcounted, in the page's reference counter.
3912 *
3913 * The page_frag functions below provide a simple allocation framework for
3914 * page fragments. This is used by the network stack and network device
3915 * drivers to provide a backing region of memory for use as either an
3916 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
3917 */
3918static struct page *__page_frag_refill(struct page_frag_cache *nc,
3919 gfp_t gfp_mask)
3920{
3921 struct page *page = NULL;
3922 gfp_t gfp = gfp_mask;
3923
3924#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
3925 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
3926 __GFP_NOMEMALLOC;
3927 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
3928 PAGE_FRAG_CACHE_MAX_ORDER);
3929 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
3930#endif
3931 if (unlikely(!page))
3932 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
3933
3934 nc->va = page ? page_address(page) : NULL;
3935
3936 return page;
3937}
3938
3939void *__alloc_page_frag(struct page_frag_cache *nc,
3940 unsigned int fragsz, gfp_t gfp_mask)
3941{
3942 unsigned int size = PAGE_SIZE;
3943 struct page *page;
3944 int offset;
3945
3946 if (unlikely(!nc->va)) {
3947refill:
3948 page = __page_frag_refill(nc, gfp_mask);
3949 if (!page)
3950 return NULL;
3951
3952#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
3953 /* if size can vary use size else just use PAGE_SIZE */
3954 size = nc->size;
3955#endif
3956 /* Even if we own the page, we do not use atomic_set().
3957 * This would break get_page_unless_zero() users.
3958 */
Alexander Duyckb59a2e02019-02-15 14:44:12 -08003959 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07003960
3961 /* reset page count bias and offset to start of new frag */
Michal Hocko2f064f32015-08-21 14:11:51 -07003962 nc->pfmemalloc = page_is_pfmemalloc(page);
Alexander Duyckb59a2e02019-02-15 14:44:12 -08003963 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07003964 nc->offset = size;
3965 }
3966
3967 offset = nc->offset - fragsz;
3968 if (unlikely(offset < 0)) {
3969 page = virt_to_page(nc->va);
3970
Joonsoo Kimfe896d12016-03-17 14:19:26 -07003971 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07003972 goto refill;
3973
3974#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
3975 /* if size can vary use size else just use PAGE_SIZE */
3976 size = nc->size;
3977#endif
3978 /* OK, page count is 0, we can safely set it */
Alexander Duyckb59a2e02019-02-15 14:44:12 -08003979 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07003980
3981 /* reset page count bias and offset to start of new frag */
Alexander Duyckb59a2e02019-02-15 14:44:12 -08003982 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07003983 offset = size - fragsz;
3984 }
3985
3986 nc->pagecnt_bias--;
3987 nc->offset = offset;
3988
3989 return nc->va + offset;
3990}
3991EXPORT_SYMBOL(__alloc_page_frag);
3992
3993/*
3994 * Frees a page fragment allocated out of either a compound or order 0 page.
3995 */
3996void __free_page_frag(void *addr)
3997{
3998 struct page *page = virt_to_head_page(addr);
3999
4000 if (unlikely(put_page_testzero(page)))
4001 __free_pages_ok(page, compound_order(page));
4002}
4003EXPORT_SYMBOL(__free_page_frag);
4004
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08004005static void *make_alloc_exact(unsigned long addr, unsigned int order,
4006 size_t size)
Andi Kleenee85c2e2011-05-11 15:13:34 -07004007{
4008 if (addr) {
4009 unsigned long alloc_end = addr + (PAGE_SIZE << order);
4010 unsigned long used = addr + PAGE_ALIGN(size);
4011
4012 split_page(virt_to_page((void *)addr), order);
4013 while (used < alloc_end) {
4014 free_page(used);
4015 used += PAGE_SIZE;
4016 }
4017 }
4018 return (void *)addr;
4019}
4020
Timur Tabi2be0ffe2008-07-23 21:28:11 -07004021/**
4022 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
4023 * @size: the number of bytes to allocate
4024 * @gfp_mask: GFP flags for the allocation
4025 *
4026 * This function is similar to alloc_pages(), except that it allocates the
4027 * minimum number of pages to satisfy the request. alloc_pages() can only
4028 * allocate memory in power-of-two pages.
4029 *
4030 * This function is also limited by MAX_ORDER.
4031 *
4032 * Memory allocated by this function must be released by free_pages_exact().
4033 */
4034void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
4035{
4036 unsigned int order = get_order(size);
4037 unsigned long addr;
4038
4039 addr = __get_free_pages(gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07004040 return make_alloc_exact(addr, order, size);
Timur Tabi2be0ffe2008-07-23 21:28:11 -07004041}
4042EXPORT_SYMBOL(alloc_pages_exact);
4043
4044/**
Andi Kleenee85c2e2011-05-11 15:13:34 -07004045 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
4046 * pages on a node.
Randy Dunlapb5e6ab52011-05-16 13:16:54 -07004047 * @nid: the preferred node ID where memory should be allocated
Andi Kleenee85c2e2011-05-11 15:13:34 -07004048 * @size: the number of bytes to allocate
4049 * @gfp_mask: GFP flags for the allocation
4050 *
4051 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
4052 * back.
Andi Kleenee85c2e2011-05-11 15:13:34 -07004053 */
Fabian Fredericke1931812014-08-06 16:04:59 -07004054void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
Andi Kleenee85c2e2011-05-11 15:13:34 -07004055{
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08004056 unsigned int order = get_order(size);
Andi Kleenee85c2e2011-05-11 15:13:34 -07004057 struct page *p = alloc_pages_node(nid, gfp_mask, order);
4058 if (!p)
4059 return NULL;
4060 return make_alloc_exact((unsigned long)page_address(p), order, size);
4061}
Andi Kleenee85c2e2011-05-11 15:13:34 -07004062
4063/**
Timur Tabi2be0ffe2008-07-23 21:28:11 -07004064 * free_pages_exact - release memory allocated via alloc_pages_exact()
4065 * @virt: the value returned by alloc_pages_exact.
4066 * @size: size of allocation, same value as passed to alloc_pages_exact().
4067 *
4068 * Release the memory allocated by a previous call to alloc_pages_exact.
4069 */
4070void free_pages_exact(void *virt, size_t size)
4071{
4072 unsigned long addr = (unsigned long)virt;
4073 unsigned long end = addr + PAGE_ALIGN(size);
4074
4075 while (addr < end) {
4076 free_page(addr);
4077 addr += PAGE_SIZE;
4078 }
4079}
4080EXPORT_SYMBOL(free_pages_exact);
4081
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08004082/**
4083 * nr_free_zone_pages - count number of pages beyond high watermark
4084 * @offset: The zone index of the highest zone
4085 *
4086 * nr_free_zone_pages() counts the number of counts pages which are beyond the
4087 * high watermark within all zones at or below a given zone index. For each
4088 * zone, the number of pages is calculated as:
Jiang Liu834405c2013-07-03 15:03:04 -07004089 * managed_pages - high_pages
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08004090 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08004091static unsigned long nr_free_zone_pages(int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004092{
Mel Gormandd1a2392008-04-28 02:12:17 -07004093 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07004094 struct zone *zone;
4095
Martin J. Blighe310fd42005-07-29 22:59:18 -07004096 /* Just pick one node, since fallback list is circular */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08004097 unsigned long sum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004098
Mel Gorman0e884602008-04-28 02:12:14 -07004099 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004100
Mel Gorman54a6eb52008-04-28 02:12:16 -07004101 for_each_zone_zonelist(zone, z, zonelist, offset) {
Jiang Liub40da042013-02-22 16:33:52 -08004102 unsigned long size = zone->managed_pages;
Mel Gorman41858962009-06-16 15:32:12 -07004103 unsigned long high = high_wmark_pages(zone);
Martin J. Blighe310fd42005-07-29 22:59:18 -07004104 if (size > high)
4105 sum += size - high;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004106 }
4107
4108 return sum;
4109}
4110
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08004111/**
4112 * nr_free_buffer_pages - count number of pages beyond high watermark
4113 *
4114 * nr_free_buffer_pages() counts the number of pages which are beyond the high
4115 * watermark within ZONE_DMA and ZONE_NORMAL.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004116 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08004117unsigned long nr_free_buffer_pages(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004118{
Al Viroaf4ca452005-10-21 02:55:38 -04004119 return nr_free_zone_pages(gfp_zone(GFP_USER));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004120}
Meelap Shahc2f1a552007-07-17 04:04:39 -07004121EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004122
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08004123/**
4124 * nr_free_pagecache_pages - count number of pages beyond high watermark
4125 *
4126 * nr_free_pagecache_pages() counts the number of pages which are beyond the
4127 * high watermark within all zones.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004128 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08004129unsigned long nr_free_pagecache_pages(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004130{
Mel Gorman2a1e2742007-07-17 04:03:12 -07004131 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004132}
Christoph Lameter08e0f6a2006-09-27 01:50:06 -07004133
4134static inline void show_node(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004135{
Kirill A. Shutemove5adfff2012-12-11 16:00:29 -08004136 if (IS_ENABLED(CONFIG_NUMA))
Andy Whitcroft25ba77c2006-12-06 20:33:03 -08004137 printk("Node %d ", zone_to_nid(zone));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004138}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004139
Igor Redkod02bd272016-03-17 14:19:05 -07004140long si_mem_available(void)
4141{
4142 long available;
4143 unsigned long pagecache;
4144 unsigned long wmark_low = 0;
4145 unsigned long pages[NR_LRU_LISTS];
4146 struct zone *zone;
4147 int lru;
4148
4149 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
Mel Gorman2f95ff92016-08-11 15:32:57 -07004150 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
Igor Redkod02bd272016-03-17 14:19:05 -07004151
4152 for_each_zone(zone)
4153 wmark_low += zone->watermark[WMARK_LOW];
4154
4155 /*
4156 * Estimate the amount of memory available for userspace allocations,
4157 * without causing swapping.
4158 */
4159 available = global_page_state(NR_FREE_PAGES) - totalreserve_pages;
4160
4161 /*
4162 * Not all the page cache can be freed, otherwise the system will
4163 * start swapping. Assume at least half of the page cache, or the
4164 * low watermark worth of cache, needs to stay.
4165 */
4166 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
4167 pagecache -= min(pagecache / 2, wmark_low);
4168 available += pagecache;
4169
4170 /*
4171 * Part of the reclaimable slab consists of items that are in use,
4172 * and cannot be freed. Cap this estimate at the low watermark.
4173 */
4174 available += global_page_state(NR_SLAB_RECLAIMABLE) -
4175 min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
4176
4177 if (available < 0)
4178 available = 0;
4179 return available;
4180}
4181EXPORT_SYMBOL_GPL(si_mem_available);
4182
Linus Torvalds1da177e2005-04-16 15:20:36 -07004183void si_meminfo(struct sysinfo *val)
4184{
4185 val->totalram = totalram_pages;
Mel Gorman11fb9982016-07-28 15:46:20 -07004186 val->sharedram = global_node_page_state(NR_SHMEM);
Christoph Lameterd23ad422007-02-10 01:43:02 -08004187 val->freeram = global_page_state(NR_FREE_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004188 val->bufferram = nr_blockdev_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004189 val->totalhigh = totalhigh_pages;
4190 val->freehigh = nr_free_highpages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004191 val->mem_unit = PAGE_SIZE;
4192}
4193
4194EXPORT_SYMBOL(si_meminfo);
4195
4196#ifdef CONFIG_NUMA
4197void si_meminfo_node(struct sysinfo *val, int nid)
4198{
Jiang Liucdd91a72013-07-03 15:03:27 -07004199 int zone_type; /* needs to be signed */
4200 unsigned long managed_pages = 0;
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07004201 unsigned long managed_highpages = 0;
4202 unsigned long free_highpages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004203 pg_data_t *pgdat = NODE_DATA(nid);
4204
Jiang Liucdd91a72013-07-03 15:03:27 -07004205 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
4206 managed_pages += pgdat->node_zones[zone_type].managed_pages;
4207 val->totalram = managed_pages;
Mel Gorman11fb9982016-07-28 15:46:20 -07004208 val->sharedram = node_page_state(pgdat, NR_SHMEM);
Mel Gorman75ef7182016-07-28 15:45:24 -07004209 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07004210#ifdef CONFIG_HIGHMEM
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07004211 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
4212 struct zone *zone = &pgdat->node_zones[zone_type];
4213
4214 if (is_highmem(zone)) {
4215 managed_highpages += zone->managed_pages;
4216 free_highpages += zone_page_state(zone, NR_FREE_PAGES);
4217 }
4218 }
4219 val->totalhigh = managed_highpages;
4220 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07004221#else
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07004222 val->totalhigh = managed_highpages;
4223 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07004224#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004225 val->mem_unit = PAGE_SIZE;
4226}
4227#endif
4228
David Rientjesddd588b2011-03-22 16:30:46 -07004229/*
David Rientjes7bf02ea2011-05-24 17:11:16 -07004230 * Determine whether the node should be displayed or not, depending on whether
4231 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
David Rientjesddd588b2011-03-22 16:30:46 -07004232 */
David Rientjes7bf02ea2011-05-24 17:11:16 -07004233bool skip_free_areas_node(unsigned int flags, int nid)
David Rientjesddd588b2011-03-22 16:30:46 -07004234{
4235 bool ret = false;
Mel Gormancc9a6c82012-03-21 16:34:11 -07004236 unsigned int cpuset_mems_cookie;
David Rientjesddd588b2011-03-22 16:30:46 -07004237
4238 if (!(flags & SHOW_MEM_FILTER_NODES))
4239 goto out;
4240
Mel Gormancc9a6c82012-03-21 16:34:11 -07004241 do {
Mel Gormand26914d2014-04-03 14:47:24 -07004242 cpuset_mems_cookie = read_mems_allowed_begin();
Mel Gormancc9a6c82012-03-21 16:34:11 -07004243 ret = !node_isset(nid, cpuset_current_mems_allowed);
Mel Gormand26914d2014-04-03 14:47:24 -07004244 } while (read_mems_allowed_retry(cpuset_mems_cookie));
David Rientjesddd588b2011-03-22 16:30:46 -07004245out:
4246 return ret;
4247}
4248
Linus Torvalds1da177e2005-04-16 15:20:36 -07004249#define K(x) ((x) << (PAGE_SHIFT-10))
4250
Rabin Vincent377e4f12012-12-11 16:00:24 -08004251static void show_migration_types(unsigned char type)
4252{
4253 static const char types[MIGRATE_TYPES] = {
4254 [MIGRATE_UNMOVABLE] = 'U',
Rabin Vincent377e4f12012-12-11 16:00:24 -08004255 [MIGRATE_MOVABLE] = 'M',
Vlastimil Babka475a2f92015-12-11 13:40:29 -08004256 [MIGRATE_RECLAIMABLE] = 'E',
4257 [MIGRATE_HIGHATOMIC] = 'H',
Rabin Vincent377e4f12012-12-11 16:00:24 -08004258#ifdef CONFIG_CMA
4259 [MIGRATE_CMA] = 'C',
4260#endif
Minchan Kim194159f2013-02-22 16:33:58 -08004261#ifdef CONFIG_MEMORY_ISOLATION
Rabin Vincent377e4f12012-12-11 16:00:24 -08004262 [MIGRATE_ISOLATE] = 'I',
Minchan Kim194159f2013-02-22 16:33:58 -08004263#endif
Rabin Vincent377e4f12012-12-11 16:00:24 -08004264 };
4265 char tmp[MIGRATE_TYPES + 1];
4266 char *p = tmp;
4267 int i;
4268
4269 for (i = 0; i < MIGRATE_TYPES; i++) {
4270 if (type & (1 << i))
4271 *p++ = types[i];
4272 }
4273
4274 *p = '\0';
Joe Perches1f84a182016-10-27 17:46:29 -07004275 printk(KERN_CONT "(%s) ", tmp);
Rabin Vincent377e4f12012-12-11 16:00:24 -08004276}
4277
Linus Torvalds1da177e2005-04-16 15:20:36 -07004278/*
4279 * Show free area list (used inside shift_scroll-lock stuff)
4280 * We also calculate the percentage fragmentation. We do this by counting the
4281 * memory on each free list with the exception of the first item on the list.
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004282 *
4283 * Bits in @filter:
4284 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
4285 * cpuset.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004286 */
David Rientjes7bf02ea2011-05-24 17:11:16 -07004287void show_free_areas(unsigned int filter)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004288{
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004289 unsigned long free_pcp = 0;
Jes Sorensenc7241912006-09-27 01:50:05 -07004290 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004291 struct zone *zone;
Mel Gorman599d0c92016-07-28 15:45:31 -07004292 pg_data_t *pgdat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004293
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07004294 for_each_populated_zone(zone) {
David Rientjes7bf02ea2011-05-24 17:11:16 -07004295 if (skip_free_areas_node(filter, zone_to_nid(zone)))
David Rientjesddd588b2011-03-22 16:30:46 -07004296 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004297
Konstantin Khlebnikov761b0672015-04-14 15:45:32 -07004298 for_each_online_cpu(cpu)
4299 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004300 }
4301
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07004302 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
4303 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004304 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
4305 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07004306 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004307 " free:%lu free_pcp:%lu free_cma:%lu\n",
Mel Gorman599d0c92016-07-28 15:45:31 -07004308 global_node_page_state(NR_ACTIVE_ANON),
4309 global_node_page_state(NR_INACTIVE_ANON),
4310 global_node_page_state(NR_ISOLATED_ANON),
4311 global_node_page_state(NR_ACTIVE_FILE),
4312 global_node_page_state(NR_INACTIVE_FILE),
4313 global_node_page_state(NR_ISOLATED_FILE),
4314 global_node_page_state(NR_UNEVICTABLE),
Mel Gorman11fb9982016-07-28 15:46:20 -07004315 global_node_page_state(NR_FILE_DIRTY),
4316 global_node_page_state(NR_WRITEBACK),
4317 global_node_page_state(NR_UNSTABLE_NFS),
KOSAKI Motohiro3701b032009-09-21 17:01:29 -07004318 global_page_state(NR_SLAB_RECLAIMABLE),
4319 global_page_state(NR_SLAB_UNRECLAIMABLE),
Mel Gorman50658e22016-07-28 15:46:14 -07004320 global_node_page_state(NR_FILE_MAPPED),
Mel Gorman11fb9982016-07-28 15:46:20 -07004321 global_node_page_state(NR_SHMEM),
Andrew Mortona25700a2007-02-08 14:20:40 -08004322 global_page_state(NR_PAGETABLE),
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07004323 global_page_state(NR_BOUNCE),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004324 global_page_state(NR_FREE_PAGES),
4325 free_pcp,
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07004326 global_page_state(NR_FREE_CMA_PAGES));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004327
Mel Gorman599d0c92016-07-28 15:45:31 -07004328 for_each_online_pgdat(pgdat) {
4329 printk("Node %d"
4330 " active_anon:%lukB"
4331 " inactive_anon:%lukB"
4332 " active_file:%lukB"
4333 " inactive_file:%lukB"
4334 " unevictable:%lukB"
4335 " isolated(anon):%lukB"
4336 " isolated(file):%lukB"
Mel Gorman50658e22016-07-28 15:46:14 -07004337 " mapped:%lukB"
Mel Gorman11fb9982016-07-28 15:46:20 -07004338 " dirty:%lukB"
4339 " writeback:%lukB"
4340 " shmem:%lukB"
4341#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4342 " shmem_thp: %lukB"
4343 " shmem_pmdmapped: %lukB"
4344 " anon_thp: %lukB"
4345#endif
4346 " writeback_tmp:%lukB"
4347 " unstable:%lukB"
Minchan Kim33e077b2016-07-28 15:47:14 -07004348 " pages_scanned:%lu"
Mel Gorman599d0c92016-07-28 15:45:31 -07004349 " all_unreclaimable? %s"
4350 "\n",
4351 pgdat->node_id,
4352 K(node_page_state(pgdat, NR_ACTIVE_ANON)),
4353 K(node_page_state(pgdat, NR_INACTIVE_ANON)),
4354 K(node_page_state(pgdat, NR_ACTIVE_FILE)),
4355 K(node_page_state(pgdat, NR_INACTIVE_FILE)),
4356 K(node_page_state(pgdat, NR_UNEVICTABLE)),
4357 K(node_page_state(pgdat, NR_ISOLATED_ANON)),
4358 K(node_page_state(pgdat, NR_ISOLATED_FILE)),
Mel Gorman50658e22016-07-28 15:46:14 -07004359 K(node_page_state(pgdat, NR_FILE_MAPPED)),
Mel Gorman11fb9982016-07-28 15:46:20 -07004360 K(node_page_state(pgdat, NR_FILE_DIRTY)),
4361 K(node_page_state(pgdat, NR_WRITEBACK)),
Alexander Polakove3b08eb2017-04-07 16:04:45 -07004362 K(node_page_state(pgdat, NR_SHMEM)),
Mel Gorman11fb9982016-07-28 15:46:20 -07004363#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4364 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
4365 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
4366 * HPAGE_PMD_NR),
4367 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
4368#endif
Mel Gorman11fb9982016-07-28 15:46:20 -07004369 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
4370 K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
Minchan Kim33e077b2016-07-28 15:47:14 -07004371 node_page_state(pgdat, NR_PAGES_SCANNED),
Johannes Weiner19a7db12017-05-03 14:51:51 -07004372 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
4373 "yes" : "no");
Mel Gorman599d0c92016-07-28 15:45:31 -07004374 }
4375
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07004376 for_each_populated_zone(zone) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004377 int i;
4378
David Rientjes7bf02ea2011-05-24 17:11:16 -07004379 if (skip_free_areas_node(filter, zone_to_nid(zone)))
David Rientjesddd588b2011-03-22 16:30:46 -07004380 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004381
4382 free_pcp = 0;
4383 for_each_online_cpu(cpu)
4384 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
4385
Linus Torvalds1da177e2005-04-16 15:20:36 -07004386 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07004387 printk(KERN_CONT
4388 "%s"
Linus Torvalds1da177e2005-04-16 15:20:36 -07004389 " free:%lukB"
4390 " min:%lukB"
4391 " low:%lukB"
4392 " high:%lukB"
Minchan Kim71c799f2016-07-28 15:47:26 -07004393 " active_anon:%lukB"
4394 " inactive_anon:%lukB"
4395 " active_file:%lukB"
4396 " inactive_file:%lukB"
4397 " unevictable:%lukB"
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004398 " writepending:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07004399 " present:%lukB"
Jiang Liu9feedc92012-12-12 13:52:12 -08004400 " managed:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004401 " mlocked:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004402 " slab_reclaimable:%lukB"
4403 " slab_unreclaimable:%lukB"
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -07004404 " kernel_stack:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004405 " pagetables:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004406 " bounce:%lukB"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004407 " free_pcp:%lukB"
4408 " local_pcp:%ukB"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07004409 " free_cma:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07004410 "\n",
4411 zone->name,
Mel Gorman88f5acf2011-01-13 15:45:41 -08004412 K(zone_page_state(zone, NR_FREE_PAGES)),
Mel Gorman41858962009-06-16 15:32:12 -07004413 K(min_wmark_pages(zone)),
4414 K(low_wmark_pages(zone)),
4415 K(high_wmark_pages(zone)),
Minchan Kim71c799f2016-07-28 15:47:26 -07004416 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
4417 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
4418 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
4419 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
4420 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004421 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
Linus Torvalds1da177e2005-04-16 15:20:36 -07004422 K(zone->present_pages),
Jiang Liu9feedc92012-12-12 13:52:12 -08004423 K(zone->managed_pages),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004424 K(zone_page_state(zone, NR_MLOCK)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004425 K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
4426 K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
Andy Lutomirskid30dd8b2016-07-28 15:48:14 -07004427 zone_page_state(zone, NR_KERNEL_STACK_KB),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004428 K(zone_page_state(zone, NR_PAGETABLE)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07004429 K(zone_page_state(zone, NR_BOUNCE)),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07004430 K(free_pcp),
4431 K(this_cpu_read(zone->pageset->pcp.count)),
Minchan Kim33e077b2016-07-28 15:47:14 -07004432 K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004433 printk("lowmem_reserve[]:");
4434 for (i = 0; i < MAX_NR_ZONES; i++)
Joe Perches1f84a182016-10-27 17:46:29 -07004435 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
4436 printk(KERN_CONT "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004437 }
4438
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07004439 for_each_populated_zone(zone) {
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08004440 unsigned int order;
4441 unsigned long nr[MAX_ORDER], flags, total = 0;
Rabin Vincent377e4f12012-12-11 16:00:24 -08004442 unsigned char types[MAX_ORDER];
Linus Torvalds1da177e2005-04-16 15:20:36 -07004443
David Rientjes7bf02ea2011-05-24 17:11:16 -07004444 if (skip_free_areas_node(filter, zone_to_nid(zone)))
David Rientjesddd588b2011-03-22 16:30:46 -07004445 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004446 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07004447 printk(KERN_CONT "%s: ", zone->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004448
4449 spin_lock_irqsave(&zone->lock, flags);
4450 for (order = 0; order < MAX_ORDER; order++) {
Rabin Vincent377e4f12012-12-11 16:00:24 -08004451 struct free_area *area = &zone->free_area[order];
4452 int type;
4453
4454 nr[order] = area->nr_free;
Kirill Korotaev8f9de512006-06-23 02:03:50 -07004455 total += nr[order] << order;
Rabin Vincent377e4f12012-12-11 16:00:24 -08004456
4457 types[order] = 0;
4458 for (type = 0; type < MIGRATE_TYPES; type++) {
4459 if (!list_empty(&area->free_list[type]))
4460 types[order] |= 1 << type;
4461 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004462 }
4463 spin_unlock_irqrestore(&zone->lock, flags);
Rabin Vincent377e4f12012-12-11 16:00:24 -08004464 for (order = 0; order < MAX_ORDER; order++) {
Joe Perches1f84a182016-10-27 17:46:29 -07004465 printk(KERN_CONT "%lu*%lukB ",
4466 nr[order], K(1UL) << order);
Rabin Vincent377e4f12012-12-11 16:00:24 -08004467 if (nr[order])
4468 show_migration_types(types[order]);
4469 }
Joe Perches1f84a182016-10-27 17:46:29 -07004470 printk(KERN_CONT "= %lukB\n", K(total));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004471 }
4472
David Rientjes949f7ec2013-04-29 15:07:48 -07004473 hugetlb_show_meminfo();
4474
Mel Gorman11fb9982016-07-28 15:46:20 -07004475 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
Larry Woodmane6f36022008-02-04 22:29:30 -08004476
Linus Torvalds1da177e2005-04-16 15:20:36 -07004477 show_swap_cache_info();
4478}
4479
Mel Gorman19770b32008-04-28 02:12:18 -07004480static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
4481{
4482 zoneref->zone = zone;
4483 zoneref->zone_idx = zone_idx(zone);
4484}
4485
Linus Torvalds1da177e2005-04-16 15:20:36 -07004486/*
4487 * Builds allocation fallback zone lists.
Christoph Lameter1a932052006-01-06 00:11:16 -08004488 *
4489 * Add all populated zones of a node to the zonelist.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004490 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004491static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004492 int nr_zones)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004493{
Christoph Lameter1a932052006-01-06 00:11:16 -08004494 struct zone *zone;
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004495 enum zone_type zone_type = MAX_NR_ZONES;
Christoph Lameter02a68a52006-01-06 00:11:18 -08004496
4497 do {
Christoph Lameter2f6726e2006-09-25 23:31:18 -07004498 zone_type--;
Christoph Lameter070f8032006-01-06 00:11:19 -08004499 zone = pgdat->node_zones + zone_type;
Mel Gorman6aa303d2016-09-01 16:14:55 -07004500 if (managed_zone(zone)) {
Mel Gormandd1a2392008-04-28 02:12:17 -07004501 zoneref_set_zone(zone,
4502 &zonelist->_zonerefs[nr_zones++]);
Christoph Lameter070f8032006-01-06 00:11:19 -08004503 check_highest_zone(zone_type);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004504 }
Christoph Lameter2f6726e2006-09-25 23:31:18 -07004505 } while (zone_type);
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004506
Christoph Lameter070f8032006-01-06 00:11:19 -08004507 return nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004508}
4509
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004510
4511/*
4512 * zonelist_order:
4513 * 0 = automatic detection of better ordering.
4514 * 1 = order by ([node] distance, -zonetype)
4515 * 2 = order by (-zonetype, [node] distance)
4516 *
4517 * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
4518 * the same zonelist. So only NUMA can configure this param.
4519 */
4520#define ZONELIST_ORDER_DEFAULT 0
4521#define ZONELIST_ORDER_NODE 1
4522#define ZONELIST_ORDER_ZONE 2
4523
4524/* zonelist order in the kernel.
4525 * set_zonelist_order() will set this to NODE or ZONE.
4526 */
4527static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
4528static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
4529
4530
Linus Torvalds1da177e2005-04-16 15:20:36 -07004531#ifdef CONFIG_NUMA
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004532/* The value user specified ....changed by config */
4533static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
4534/* string for sysctl */
4535#define NUMA_ZONELIST_ORDER_LEN 16
4536char numa_zonelist_order[16] = "default";
4537
4538/*
4539 * interface for configure zonelist ordering.
4540 * command line option "numa_zonelist_order"
4541 * = "[dD]efault - default, automatic configuration.
4542 * = "[nN]ode - order by node locality, then by zone within node
4543 * = "[zZ]one - order by zone, then by locality within zone
4544 */
4545
4546static int __parse_numa_zonelist_order(char *s)
4547{
4548 if (*s == 'd' || *s == 'D') {
4549 user_zonelist_order = ZONELIST_ORDER_DEFAULT;
4550 } else if (*s == 'n' || *s == 'N') {
4551 user_zonelist_order = ZONELIST_ORDER_NODE;
4552 } else if (*s == 'z' || *s == 'Z') {
4553 user_zonelist_order = ZONELIST_ORDER_ZONE;
4554 } else {
Joe Perches11705322016-03-17 14:19:50 -07004555 pr_warn("Ignoring invalid numa_zonelist_order value: %s\n", s);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004556 return -EINVAL;
4557 }
4558 return 0;
4559}
4560
4561static __init int setup_numa_zonelist_order(char *s)
4562{
Volodymyr G. Lukiianykecb256f2011-01-13 15:46:26 -08004563 int ret;
4564
4565 if (!s)
4566 return 0;
4567
4568 ret = __parse_numa_zonelist_order(s);
4569 if (ret == 0)
4570 strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
4571
4572 return ret;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004573}
4574early_param("numa_zonelist_order", setup_numa_zonelist_order);
4575
4576/*
4577 * sysctl handler for numa_zonelist_order
4578 */
Joe Perchescccad5b2014-06-06 14:38:09 -07004579int numa_zonelist_order_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07004580 void __user *buffer, size_t *length,
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004581 loff_t *ppos)
4582{
4583 char saved_string[NUMA_ZONELIST_ORDER_LEN];
4584 int ret;
Andi Kleen443c6f12009-12-23 21:00:47 +01004585 static DEFINE_MUTEX(zl_order_mutex);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004586
Andi Kleen443c6f12009-12-23 21:00:47 +01004587 mutex_lock(&zl_order_mutex);
Chen Gangdacbde02013-07-03 15:02:35 -07004588 if (write) {
4589 if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) {
4590 ret = -EINVAL;
4591 goto out;
4592 }
4593 strcpy(saved_string, (char *)table->data);
4594 }
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07004595 ret = proc_dostring(table, write, buffer, length, ppos);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004596 if (ret)
Andi Kleen443c6f12009-12-23 21:00:47 +01004597 goto out;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004598 if (write) {
4599 int oldval = user_zonelist_order;
Chen Gangdacbde02013-07-03 15:02:35 -07004600
4601 ret = __parse_numa_zonelist_order((char *)table->data);
4602 if (ret) {
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004603 /*
4604 * bogus value. restore saved string
4605 */
Chen Gangdacbde02013-07-03 15:02:35 -07004606 strncpy((char *)table->data, saved_string,
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004607 NUMA_ZONELIST_ORDER_LEN);
4608 user_zonelist_order = oldval;
Haicheng Li4eaf3f62010-05-24 14:32:52 -07004609 } else if (oldval != user_zonelist_order) {
4610 mutex_lock(&zonelists_mutex);
Oscar Salvador23feab12020-08-18 13:00:46 +02004611 build_all_zonelists(NULL, NULL, false);
Haicheng Li4eaf3f62010-05-24 14:32:52 -07004612 mutex_unlock(&zonelists_mutex);
4613 }
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004614 }
Andi Kleen443c6f12009-12-23 21:00:47 +01004615out:
4616 mutex_unlock(&zl_order_mutex);
4617 return ret;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004618}
4619
4620
Christoph Lameter62bc62a2009-06-16 15:32:15 -07004621#define MAX_NODE_LOAD (nr_online_nodes)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004622static int node_load[MAX_NUMNODES];
4623
Linus Torvalds1da177e2005-04-16 15:20:36 -07004624/**
Pavel Pisa4dc3b162005-05-01 08:59:25 -07004625 * find_next_best_node - find the next node that should appear in a given node's fallback list
Linus Torvalds1da177e2005-04-16 15:20:36 -07004626 * @node: node whose fallback list we're appending
4627 * @used_node_mask: nodemask_t of already used nodes
4628 *
4629 * We use a number of factors to determine which is the next node that should
4630 * appear on a given node's fallback list. The node should not have appeared
4631 * already in @node's fallback list, and it should be the next closest node
4632 * according to the distance array (which contains arbitrary distance values
4633 * from each node to each node in the system), and should also prefer nodes
4634 * with no CPUs, since presumably they'll have very little allocation pressure
4635 * on them otherwise.
4636 * It returns -1 if no node is found.
4637 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004638static int find_next_best_node(int node, nodemask_t *used_node_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004639{
Linus Torvalds4cf808e2006-02-17 20:38:21 +01004640 int n, val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004641 int min_val = INT_MAX;
David Rientjes00ef2d22013-02-22 16:35:36 -08004642 int best_node = NUMA_NO_NODE;
Rusty Russella70f7302009-03-13 14:49:46 +10304643 const struct cpumask *tmp = cpumask_of_node(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004644
Linus Torvalds4cf808e2006-02-17 20:38:21 +01004645 /* Use the local node if we haven't already */
4646 if (!node_isset(node, *used_node_mask)) {
4647 node_set(node, *used_node_mask);
4648 return node;
4649 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004650
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08004651 for_each_node_state(n, N_MEMORY) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004652
4653 /* Don't want a node to appear more than once */
4654 if (node_isset(n, *used_node_mask))
4655 continue;
4656
Linus Torvalds1da177e2005-04-16 15:20:36 -07004657 /* Use the distance array to find the distance */
4658 val = node_distance(node, n);
4659
Linus Torvalds4cf808e2006-02-17 20:38:21 +01004660 /* Penalize nodes under us ("prefer the next node") */
4661 val += (n < node);
4662
Linus Torvalds1da177e2005-04-16 15:20:36 -07004663 /* Give preference to headless and unused nodes */
Rusty Russella70f7302009-03-13 14:49:46 +10304664 tmp = cpumask_of_node(n);
4665 if (!cpumask_empty(tmp))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004666 val += PENALTY_FOR_NODE_WITH_CPUS;
4667
4668 /* Slight preference for less loaded node */
4669 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
4670 val += node_load[n];
4671
4672 if (val < min_val) {
4673 min_val = val;
4674 best_node = n;
4675 }
4676 }
4677
4678 if (best_node >= 0)
4679 node_set(best_node, *used_node_mask);
4680
4681 return best_node;
4682}
4683
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004684
4685/*
4686 * Build zonelists ordered by node and zones within node.
4687 * This results in maximum locality--normal zone overflows into local
4688 * DMA zone, if any--but risks exhausting DMA zone.
4689 */
4690static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004691{
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004692 int j;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004693 struct zonelist *zonelist;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004694
Aneesh Kumar K.Vc9634cf2016-10-07 16:59:12 -07004695 zonelist = &pgdat->node_zonelists[ZONELIST_FALLBACK];
Mel Gormandd1a2392008-04-28 02:12:17 -07004696 for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
Mel Gorman54a6eb52008-04-28 02:12:16 -07004697 ;
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004698 j = build_zonelists_node(NODE_DATA(node), zonelist, j);
Mel Gormandd1a2392008-04-28 02:12:17 -07004699 zonelist->_zonerefs[j].zone = NULL;
4700 zonelist->_zonerefs[j].zone_idx = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004701}
4702
4703/*
Christoph Lameter523b9452007-10-16 01:25:37 -07004704 * Build gfp_thisnode zonelists
4705 */
4706static void build_thisnode_zonelists(pg_data_t *pgdat)
4707{
Christoph Lameter523b9452007-10-16 01:25:37 -07004708 int j;
4709 struct zonelist *zonelist;
4710
Aneesh Kumar K.Vc9634cf2016-10-07 16:59:12 -07004711 zonelist = &pgdat->node_zonelists[ZONELIST_NOFALLBACK];
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004712 j = build_zonelists_node(pgdat, zonelist, 0);
Mel Gormandd1a2392008-04-28 02:12:17 -07004713 zonelist->_zonerefs[j].zone = NULL;
4714 zonelist->_zonerefs[j].zone_idx = 0;
Christoph Lameter523b9452007-10-16 01:25:37 -07004715}
4716
4717/*
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004718 * Build zonelists ordered by zone and nodes within zones.
4719 * This results in conserving DMA zone[s] until all Normal memory is
4720 * exhausted, but results in overflowing to remote node while memory
4721 * may still exist in local DMA zone.
4722 */
4723static int node_order[MAX_NUMNODES];
4724
4725static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
4726{
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004727 int pos, j, node;
4728 int zone_type; /* needs to be signed */
4729 struct zone *z;
4730 struct zonelist *zonelist;
4731
Aneesh Kumar K.Vc9634cf2016-10-07 16:59:12 -07004732 zonelist = &pgdat->node_zonelists[ZONELIST_FALLBACK];
Mel Gorman54a6eb52008-04-28 02:12:16 -07004733 pos = 0;
4734 for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
4735 for (j = 0; j < nr_nodes; j++) {
4736 node = node_order[j];
4737 z = &NODE_DATA(node)->node_zones[zone_type];
Mel Gorman6aa303d2016-09-01 16:14:55 -07004738 if (managed_zone(z)) {
Mel Gormandd1a2392008-04-28 02:12:17 -07004739 zoneref_set_zone(z,
4740 &zonelist->_zonerefs[pos++]);
Mel Gorman54a6eb52008-04-28 02:12:16 -07004741 check_highest_zone(zone_type);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004742 }
4743 }
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004744 }
Mel Gormandd1a2392008-04-28 02:12:17 -07004745 zonelist->_zonerefs[pos].zone = NULL;
4746 zonelist->_zonerefs[pos].zone_idx = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004747}
4748
Mel Gorman31939132014-10-09 15:28:30 -07004749#if defined(CONFIG_64BIT)
4750/*
4751 * Devices that require DMA32/DMA are relatively rare and do not justify a
4752 * penalty to every machine in case the specialised case applies. Default
4753 * to Node-ordering on 64-bit NUMA machines
4754 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004755static int default_zonelist_order(void)
4756{
Mel Gorman31939132014-10-09 15:28:30 -07004757 return ZONELIST_ORDER_NODE;
4758}
4759#else
4760/*
4761 * On 32-bit, the Normal zone needs to be preserved for allocations accessible
4762 * by the kernel. If processes running on node 0 deplete the low memory zone
4763 * then reclaim will occur more frequency increasing stalls and potentially
4764 * be easier to OOM if a large percentage of the zone is under writeback or
4765 * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set.
4766 * Hence, default to zone ordering on 32-bit.
4767 */
4768static int default_zonelist_order(void)
4769{
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004770 return ZONELIST_ORDER_ZONE;
4771}
Mel Gorman31939132014-10-09 15:28:30 -07004772#endif /* CONFIG_64BIT */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004773
4774static void set_zonelist_order(void)
4775{
4776 if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
4777 current_zonelist_order = default_zonelist_order();
4778 else
4779 current_zonelist_order = user_zonelist_order;
4780}
4781
4782static void build_zonelists(pg_data_t *pgdat)
4783{
Yaowei Baic00eb152016-01-14 15:19:00 -08004784 int i, node, load;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004785 nodemask_t used_mask;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004786 int local_node, prev_node;
4787 struct zonelist *zonelist;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08004788 unsigned int order = current_zonelist_order;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004789
4790 /* initialize zonelists */
Christoph Lameter523b9452007-10-16 01:25:37 -07004791 for (i = 0; i < MAX_ZONELISTS; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004792 zonelist = pgdat->node_zonelists + i;
Mel Gormandd1a2392008-04-28 02:12:17 -07004793 zonelist->_zonerefs[0].zone = NULL;
4794 zonelist->_zonerefs[0].zone_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004795 }
4796
4797 /* NUMA-aware ordering of nodes */
4798 local_node = pgdat->node_id;
Christoph Lameter62bc62a2009-06-16 15:32:15 -07004799 load = nr_online_nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004800 prev_node = local_node;
4801 nodes_clear(used_mask);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004802
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004803 memset(node_order, 0, sizeof(node_order));
Yaowei Baic00eb152016-01-14 15:19:00 -08004804 i = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004805
Linus Torvalds1da177e2005-04-16 15:20:36 -07004806 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
4807 /*
4808 * We don't want to pressure a particular node.
4809 * So adding penalty to the first node in same
4810 * distance group to make it round-robin.
4811 */
David Rientjes957f8222012-10-08 16:33:24 -07004812 if (node_distance(local_node, node) !=
4813 node_distance(local_node, prev_node))
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004814 node_load[node] = load;
4815
Linus Torvalds1da177e2005-04-16 15:20:36 -07004816 prev_node = node;
4817 load--;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004818 if (order == ZONELIST_ORDER_NODE)
4819 build_zonelists_in_node_order(pgdat, node);
4820 else
Yaowei Baic00eb152016-01-14 15:19:00 -08004821 node_order[i++] = node; /* remember order */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004822 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004823
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004824 if (order == ZONELIST_ORDER_ZONE) {
4825 /* calculate node order -- i.e., DMA last! */
Yaowei Baic00eb152016-01-14 15:19:00 -08004826 build_zonelists_in_zone_order(pgdat, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004827 }
Christoph Lameter523b9452007-10-16 01:25:37 -07004828
4829 build_thisnode_zonelists(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004830}
4831
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07004832#ifdef CONFIG_HAVE_MEMORYLESS_NODES
4833/*
4834 * Return node id of node used for "local" allocations.
4835 * I.e., first node id of first zone in arg node's generic zonelist.
4836 * Used for initializing percpu 'numa_mem', which is used primarily
4837 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
4838 */
4839int local_memory_node(int node)
4840{
Mel Gormanc33d6c02016-05-19 17:14:10 -07004841 struct zoneref *z;
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07004842
Mel Gormanc33d6c02016-05-19 17:14:10 -07004843 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07004844 gfp_zone(GFP_KERNEL),
Mel Gormanc33d6c02016-05-19 17:14:10 -07004845 NULL);
4846 return z->zone->node;
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07004847}
4848#endif
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004849
Joonsoo Kim6423aa82016-08-10 16:27:49 -07004850static void setup_min_unmapped_ratio(void);
4851static void setup_min_slab_ratio(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004852#else /* CONFIG_NUMA */
4853
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004854static void set_zonelist_order(void)
4855{
4856 current_zonelist_order = ZONELIST_ORDER_ZONE;
4857}
4858
4859static void build_zonelists(pg_data_t *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004860{
Christoph Lameter19655d32006-09-25 23:31:19 -07004861 int node, local_node;
Mel Gorman54a6eb52008-04-28 02:12:16 -07004862 enum zone_type j;
4863 struct zonelist *zonelist;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004864
4865 local_node = pgdat->node_id;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004866
Aneesh Kumar K.Vc9634cf2016-10-07 16:59:12 -07004867 zonelist = &pgdat->node_zonelists[ZONELIST_FALLBACK];
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004868 j = build_zonelists_node(pgdat, zonelist, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004869
Mel Gorman54a6eb52008-04-28 02:12:16 -07004870 /*
4871 * Now we build the zonelist so that it contains the zones
4872 * of all the other nodes.
4873 * We don't want to pressure a particular node, so when
4874 * building the zones for node N, we make sure that the
4875 * zones coming right after the local ones are those from
4876 * node N+1 (modulo N)
4877 */
4878 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
4879 if (!node_online(node))
4880 continue;
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004881 j = build_zonelists_node(NODE_DATA(node), zonelist, j);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004882 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07004883 for (node = 0; node < local_node; node++) {
4884 if (!node_online(node))
4885 continue;
Zhang Yanfeibc732f12013-07-08 16:00:06 -07004886 j = build_zonelists_node(NODE_DATA(node), zonelist, j);
Mel Gorman54a6eb52008-04-28 02:12:16 -07004887 }
4888
Mel Gormandd1a2392008-04-28 02:12:17 -07004889 zonelist->_zonerefs[j].zone = NULL;
4890 zonelist->_zonerefs[j].zone_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004891}
4892
4893#endif /* CONFIG_NUMA */
4894
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004895/*
4896 * Boot pageset table. One per cpu which is going to be used for all
4897 * zones and all nodes. The parameters will be set in such a way
4898 * that an item put on a list will immediately be handed over to
4899 * the buddy list. This is safe since pageset manipulation is done
4900 * with interrupts disabled.
4901 *
4902 * The boot_pagesets must be kept even after bootup is complete for
4903 * unused processors and/or zones. They do play a role for bootstrapping
4904 * hotplugged processors.
4905 *
4906 * zoneinfo_show() and maybe other functions do
4907 * not check if the processor is online before following the pageset pointer.
4908 * Other parts of the kernel may not check if the zone is available.
4909 */
4910static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
4911static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
Haicheng Li1f522502010-05-24 14:32:51 -07004912static void setup_zone_pageset(struct zone *zone);
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004913
Haicheng Li4eaf3f62010-05-24 14:32:52 -07004914/*
4915 * Global mutex to protect against size modification of zonelists
4916 * as well as to serialize pageset setup for the new populated zone.
4917 */
4918DEFINE_MUTEX(zonelists_mutex);
4919
Rusty Russell9b1a4d32008-07-28 12:16:30 -05004920/* return values int ....just for stop_machine() */
Jiang Liu4ed7e022012-07-31 16:43:35 -07004921static int __build_all_zonelists(void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004922{
Yasunori Goto68113782006-06-23 02:03:11 -07004923 int nid;
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004924 int cpu;
Jiang Liu9adb62a2012-07-31 16:43:28 -07004925 pg_data_t *self = data;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08004926
Bo Liu7f9cfb32009-08-18 14:11:19 -07004927#ifdef CONFIG_NUMA
4928 memset(node_load, 0, sizeof(node_load));
4929#endif
Jiang Liu9adb62a2012-07-31 16:43:28 -07004930
4931 if (self && !node_online(self->node_id)) {
4932 build_zonelists(self);
Jiang Liu9adb62a2012-07-31 16:43:28 -07004933 }
4934
Paul Jackson9276b1bc2006-12-06 20:31:48 -08004935 for_each_online_node(nid) {
Christoph Lameter7ea15302007-10-16 01:25:29 -07004936 pg_data_t *pgdat = NODE_DATA(nid);
4937
4938 build_zonelists(pgdat);
Paul Jackson9276b1bc2006-12-06 20:31:48 -08004939 }
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004940
4941 /*
4942 * Initialize the boot_pagesets that are going to be used
4943 * for bootstrapping processors. The real pagesets for
4944 * each zone will be allocated later when the per cpu
4945 * allocator is available.
4946 *
4947 * boot_pagesets are used also for bootstrapping offline
4948 * cpus if the system is already booted because the pagesets
4949 * are needed to initialize allocators on a specific cpu too.
4950 * F.e. the percpu allocator needs the page allocator which
4951 * needs the percpu allocator in order to allocate its pagesets
4952 * (a chicken-egg dilemma).
4953 */
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07004954 for_each_possible_cpu(cpu) {
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004955 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
4956
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07004957#ifdef CONFIG_HAVE_MEMORYLESS_NODES
4958 /*
4959 * We now know the "local memory node" for each node--
4960 * i.e., the node of the first zone in the generic zonelist.
4961 * Set up numa_mem percpu variable for on-line cpus. During
4962 * boot, only the boot cpu should be on-line; we'll init the
4963 * secondary cpus' numa_mem as they come on-line. During
4964 * node/memory hotplug, we'll fixup all on-line cpus.
4965 */
4966 if (cpu_online(cpu))
4967 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
4968#endif
4969 }
4970
Yasunori Goto68113782006-06-23 02:03:11 -07004971 return 0;
4972}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004973
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08004974static noinline void __init
4975build_all_zonelists_init(void)
4976{
4977 __build_all_zonelists(NULL);
4978 mminit_verify_zonelist();
4979 cpuset_init_current_mems_allowed();
4980}
4981
Haicheng Li4eaf3f62010-05-24 14:32:52 -07004982/*
4983 * Called with zonelists_mutex held always
4984 * unless system_state == SYSTEM_BOOTING.
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08004985 *
4986 * __ref due to (1) call of __meminit annotated setup_zone_pageset
4987 * [we're only called with non-NULL zone through __meminit paths] and
4988 * (2) call of __init annotated helper build_all_zonelists_init
4989 * [protected by SYSTEM_BOOTING].
Haicheng Li4eaf3f62010-05-24 14:32:52 -07004990 */
Oscar Salvador23feab12020-08-18 13:00:46 +02004991void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone,
4992 bool hotplug_context)
Yasunori Goto68113782006-06-23 02:03:11 -07004993{
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07004994 set_zonelist_order();
4995
Oscar Salvador23feab12020-08-18 13:00:46 +02004996 if (system_state == SYSTEM_BOOTING && !hotplug_context) {
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08004997 build_all_zonelists_init();
Yasunori Goto68113782006-06-23 02:03:11 -07004998 } else {
KAMEZAWA Hiroyukie9959f02010-11-24 12:57:09 -08004999#ifdef CONFIG_MEMORY_HOTPLUG
Jiang Liu9adb62a2012-07-31 16:43:28 -07005000 if (zone)
5001 setup_zone_pageset(zone);
KAMEZAWA Hiroyukie9959f02010-11-24 12:57:09 -08005002#endif
Cody P Schaferdd1895e2013-07-03 15:01:36 -07005003 /* we have to stop all cpus to guarantee there is no user
5004 of zonelist */
Jiang Liu9adb62a2012-07-31 16:43:28 -07005005 stop_machine(__build_all_zonelists, pgdat, NULL);
Yasunori Goto68113782006-06-23 02:03:11 -07005006 /* cpuset refresh routine should be here */
5007 }
Andrew Mortonbd1e22b2006-06-23 02:03:47 -07005008 vm_total_pages = nr_free_pagecache_pages();
Mel Gorman9ef9acb2007-10-16 01:25:54 -07005009 /*
5010 * Disable grouping by mobility if the number of pages in the
5011 * system is too low to allow the mechanism to work. It would be
5012 * more accurate, but expensive to check per-zone. This check is
5013 * made on memory-hotadd so a system can start with mobility
5014 * disabled and enable it later
5015 */
Mel Gormand9c23402007-10-16 01:26:01 -07005016 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
Mel Gorman9ef9acb2007-10-16 01:25:54 -07005017 page_group_by_mobility_disabled = 1;
5018 else
5019 page_group_by_mobility_disabled = 0;
5020
Joe Perches756a025f02016-03-17 14:19:47 -07005021 pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n",
5022 nr_online_nodes,
5023 zonelist_order_name[current_zonelist_order],
5024 page_group_by_mobility_disabled ? "off" : "on",
5025 vm_total_pages);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005026#ifdef CONFIG_NUMA
Anton Blanchardf88dfff2014-12-10 15:42:53 -08005027 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005028#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005029}
5030
5031/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07005032 * Initially all pages are reserved - free ones are freed
5033 * up by free_all_bootmem() once the early boot process is
5034 * done. Non-atomic initialization, single-pass.
5035 */
Matt Tolentinoc09b4242006-01-17 07:03:44 +01005036void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
Dave Hansena2f3aa022007-01-10 23:15:30 -08005037 unsigned long start_pfn, enum memmap_context context)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005038{
Dan Williams4b94ffd2016-01-15 16:56:22 -08005039 struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn));
Andy Whitcroft29751f62005-06-23 00:08:00 -07005040 unsigned long end_pfn = start_pfn + size;
Dan Williams4b94ffd2016-01-15 16:56:22 -08005041 pg_data_t *pgdat = NODE_DATA(nid);
Andy Whitcroft29751f62005-06-23 00:08:00 -07005042 unsigned long pfn;
Mel Gorman3a80a7f2015-06-30 14:57:02 -07005043 unsigned long nr_initialised = 0;
Taku Izumi342332e2016-03-15 14:55:22 -07005044#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5045 struct memblock_region *r = NULL, *tmp;
5046#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005047
Hugh Dickins22b31ee2009-01-06 14:40:09 -08005048 if (highest_memmap_pfn < end_pfn - 1)
5049 highest_memmap_pfn = end_pfn - 1;
5050
Dan Williams4b94ffd2016-01-15 16:56:22 -08005051 /*
5052 * Honor reservation requested by the driver for this ZONE_DEVICE
5053 * memory
5054 */
5055 if (altmap && start_pfn == altmap->base_pfn)
5056 start_pfn += altmap->reserve;
5057
Greg Ungerercbe8dd42006-01-12 01:05:24 -08005058 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
Dave Hansena2f3aa022007-01-10 23:15:30 -08005059 /*
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07005060 * There can be holes in boot-time mem_map[]s handed to this
5061 * function. They do not exist on hotplugged memory.
Dave Hansena2f3aa022007-01-10 23:15:30 -08005062 */
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07005063 if (context != MEMMAP_EARLY)
5064 goto not_early;
5065
5066 if (!early_pfn_valid(pfn))
5067 continue;
5068 if (!early_pfn_in_nid(pfn, nid))
5069 continue;
5070 if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised))
5071 break;
Taku Izumi342332e2016-03-15 14:55:22 -07005072
5073#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07005074 /*
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07005075 * Check given memblock attribute by firmware which can affect
5076 * kernel memory layout. If zone==ZONE_MOVABLE but memory is
5077 * mirrored, it's an overlapped memmap init. skip it.
5078 */
5079 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
5080 if (!r || pfn >= memblock_region_memory_end_pfn(r)) {
5081 for_each_memblock(memory, tmp)
5082 if (pfn < memblock_region_memory_end_pfn(tmp))
5083 break;
5084 r = tmp;
Taku Izumi342332e2016-03-15 14:55:22 -07005085 }
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07005086 if (pfn >= memblock_region_memory_base_pfn(r) &&
5087 memblock_is_mirror(r)) {
5088 /* already initialized as NORMAL */
5089 pfn = memblock_region_memory_end_pfn(r);
5090 continue;
5091 }
Dave Hansena2f3aa022007-01-10 23:15:30 -08005092 }
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07005093#endif
Mel Gormanac5d2532015-06-30 14:57:20 -07005094
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07005095not_early:
Mel Gormanac5d2532015-06-30 14:57:20 -07005096 /*
5097 * Mark the block movable so that blocks are reserved for
5098 * movable at startup. This will force kernel allocations
5099 * to reserve their blocks rather than leaking throughout
5100 * the address space during boot when many long-lived
Mel Gorman974a7862015-11-06 16:28:34 -08005101 * kernel allocations are made.
Mel Gormanac5d2532015-06-30 14:57:20 -07005102 *
5103 * bitmap is created for zone's valid pfn range. but memmap
5104 * can be created for invalid pages (for alignment)
5105 * check here not to call set_pageblock_migratetype() against
5106 * pfn out of zone.
5107 */
5108 if (!(pfn & (pageblock_nr_pages - 1))) {
5109 struct page *page = pfn_to_page(pfn);
5110
5111 __init_single_page(page, pfn, zone, nid);
5112 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5113 } else {
5114 __init_single_pfn(pfn, zone, nid);
5115 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005116 }
5117}
5118
Andi Kleen1e548de2008-02-04 22:29:26 -08005119static void __meminit zone_init_free_lists(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005120{
Mel Gorman7aeb09f2014-06-04 16:10:21 -07005121 unsigned int order, t;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07005122 for_each_migratetype_order(order, t) {
5123 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005124 zone->free_area[order].nr_free = 0;
5125 }
5126}
5127
5128#ifndef __HAVE_ARCH_MEMMAP_INIT
5129#define memmap_init(size, nid, zone, start_pfn) \
Dave Hansena2f3aa022007-01-10 23:15:30 -08005130 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005131#endif
5132
David Rientjes7cd2b0a2014-06-23 13:22:04 -07005133static int zone_batchsize(struct zone *zone)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005134{
David Howells3a6be872009-05-06 16:03:03 -07005135#ifdef CONFIG_MMU
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005136 int batch;
5137
5138 /*
5139 * The per-cpu-pages pools are set to around 1000th of the
Seth, Rohitba56e912005-10-29 18:15:47 -07005140 * size of the zone. But no more than 1/2 of a meg.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005141 *
5142 * OK, so we don't know how big the cache is. So guess.
5143 */
Jiang Liub40da042013-02-22 16:33:52 -08005144 batch = zone->managed_pages / 1024;
Seth, Rohitba56e912005-10-29 18:15:47 -07005145 if (batch * PAGE_SIZE > 512 * 1024)
5146 batch = (512 * 1024) / PAGE_SIZE;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005147 batch /= 4; /* We effectively *= 4 below */
5148 if (batch < 1)
5149 batch = 1;
5150
5151 /*
Nick Piggin0ceaacc2005-12-04 13:55:25 +11005152 * Clamp the batch to a 2^n - 1 value. Having a power
5153 * of 2 value was found to be more likely to have
5154 * suboptimal cache aliasing properties in some cases.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005155 *
Nick Piggin0ceaacc2005-12-04 13:55:25 +11005156 * For example if 2 tasks are alternately allocating
5157 * batches of pages, one task can end up with a lot
5158 * of pages of one half of the possible page colors
5159 * and the other with pages of the other colors.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005160 */
David Howells91552032009-05-06 16:03:02 -07005161 batch = rounddown_pow_of_two(batch + batch/2) - 1;
Seth, Rohitba56e912005-10-29 18:15:47 -07005162
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005163 return batch;
David Howells3a6be872009-05-06 16:03:03 -07005164
5165#else
5166 /* The deferral and batching of frees should be suppressed under NOMMU
5167 * conditions.
5168 *
5169 * The problem is that NOMMU needs to be able to allocate large chunks
5170 * of contiguous memory as there's no hardware page translation to
5171 * assemble apparent contiguous memory from discontiguous pages.
5172 *
5173 * Queueing large contiguous runs of pages for batching, however,
5174 * causes the pages to actually be freed in smaller chunks. As there
5175 * can be a significant delay between the individual batches being
5176 * recycled, this leads to the once large chunks of space being
5177 * fragmented and becoming unavailable for high-order allocations.
5178 */
5179 return 0;
5180#endif
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005181}
5182
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07005183/*
5184 * pcp->high and pcp->batch values are related and dependent on one another:
5185 * ->batch must never be higher then ->high.
5186 * The following function updates them in a safe manner without read side
5187 * locking.
5188 *
5189 * Any new users of pcp->batch and pcp->high should ensure they can cope with
5190 * those fields changing asynchronously (acording the the above rule).
5191 *
5192 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
5193 * outside of boot time (or some other assurance that no concurrent updaters
5194 * exist).
5195 */
5196static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
5197 unsigned long batch)
5198{
5199 /* start with a fail safe value for batch */
5200 pcp->batch = 1;
5201 smp_wmb();
5202
5203 /* Update high, then batch, in order */
5204 pcp->high = high;
5205 smp_wmb();
5206
5207 pcp->batch = batch;
5208}
5209
Cody P Schafer36640332013-07-03 15:01:40 -07005210/* a companion to pageset_set_high() */
Cody P Schafer4008bab2013-07-03 15:01:28 -07005211static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
5212{
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07005213 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
Cody P Schafer4008bab2013-07-03 15:01:28 -07005214}
5215
Cody P Schafer88c90db2013-07-03 15:01:35 -07005216static void pageset_init(struct per_cpu_pageset *p)
Christoph Lameter2caaad42005-06-21 17:15:00 -07005217{
5218 struct per_cpu_pages *pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07005219 int migratetype;
Christoph Lameter2caaad42005-06-21 17:15:00 -07005220
Magnus Damm1c6fe942005-10-26 01:58:59 -07005221 memset(p, 0, sizeof(*p));
5222
Christoph Lameter3dfa5722008-02-04 22:29:19 -08005223 pcp = &p->pcp;
Christoph Lameter2caaad42005-06-21 17:15:00 -07005224 pcp->count = 0;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07005225 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
5226 INIT_LIST_HEAD(&pcp->lists[migratetype]);
Christoph Lameter2caaad42005-06-21 17:15:00 -07005227}
5228
Cody P Schafer88c90db2013-07-03 15:01:35 -07005229static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
5230{
5231 pageset_init(p);
5232 pageset_set_batch(p, batch);
5233}
5234
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005235/*
Cody P Schafer36640332013-07-03 15:01:40 -07005236 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005237 * to the value high for the pageset p.
5238 */
Cody P Schafer36640332013-07-03 15:01:40 -07005239static void pageset_set_high(struct per_cpu_pageset *p,
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005240 unsigned long high)
5241{
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07005242 unsigned long batch = max(1UL, high / 4);
5243 if ((high / 4) > (PAGE_SHIFT * 8))
5244 batch = PAGE_SHIFT * 8;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005245
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07005246 pageset_update(&p->pcp, high, batch);
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005247}
5248
David Rientjes7cd2b0a2014-06-23 13:22:04 -07005249static void pageset_set_high_and_batch(struct zone *zone,
5250 struct per_cpu_pageset *pcp)
Cody P Schafer56cef2b2013-07-03 15:01:38 -07005251{
Cody P Schafer56cef2b2013-07-03 15:01:38 -07005252 if (percpu_pagelist_fraction)
Cody P Schafer36640332013-07-03 15:01:40 -07005253 pageset_set_high(pcp,
Cody P Schafer56cef2b2013-07-03 15:01:38 -07005254 (zone->managed_pages /
5255 percpu_pagelist_fraction));
5256 else
5257 pageset_set_batch(pcp, zone_batchsize(zone));
5258}
5259
Cody P Schafer169f6c12013-07-03 15:01:41 -07005260static void __meminit zone_pageset_init(struct zone *zone, int cpu)
5261{
5262 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
5263
5264 pageset_init(pcp);
5265 pageset_set_high_and_batch(zone, pcp);
5266}
5267
Jiang Liu4ed7e022012-07-31 16:43:35 -07005268static void __meminit setup_zone_pageset(struct zone *zone)
Wu Fengguang319774e2010-05-24 14:32:49 -07005269{
5270 int cpu;
Wu Fengguang319774e2010-05-24 14:32:49 -07005271 zone->pageset = alloc_percpu(struct per_cpu_pageset);
Cody P Schafer56cef2b2013-07-03 15:01:38 -07005272 for_each_possible_cpu(cpu)
5273 zone_pageset_init(zone, cpu);
Wu Fengguang319774e2010-05-24 14:32:49 -07005274}
5275
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005276/*
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005277 * Allocate per cpu pagesets and initialize them.
5278 * Before this call only boot pagesets were available.
Christoph Lameter2caaad42005-06-21 17:15:00 -07005279 */
Al Viro78d99552005-12-15 09:18:25 +00005280void __init setup_per_cpu_pageset(void)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005281{
Mel Gormanb4911ea2016-08-04 15:31:49 -07005282 struct pglist_data *pgdat;
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005283 struct zone *zone;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005284
Wu Fengguang319774e2010-05-24 14:32:49 -07005285 for_each_populated_zone(zone)
5286 setup_zone_pageset(zone);
Mel Gormanb4911ea2016-08-04 15:31:49 -07005287
5288 for_each_online_pgdat(pgdat)
5289 pgdat->per_cpu_nodestats =
5290 alloc_percpu(struct per_cpu_nodestat);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07005291}
5292
Matt Tolentinoc09b4242006-01-17 07:03:44 +01005293static __meminit void zone_pcp_init(struct zone *zone)
Dave Hansened8ece22005-10-29 18:16:50 -07005294{
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005295 /*
5296 * per cpu subsystem is not up at this point. The following code
5297 * relies on the ability of the linker to provide the
5298 * offset of a (static) per cpu variable into the per cpu area.
5299 */
5300 zone->pageset = &boot_pageset;
Dave Hansened8ece22005-10-29 18:16:50 -07005301
Xishi Qiub38a8722013-11-12 15:07:20 -08005302 if (populated_zone(zone))
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005303 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
5304 zone->name, zone->present_pages,
5305 zone_batchsize(zone));
Dave Hansened8ece22005-10-29 18:16:50 -07005306}
5307
Jiang Liu4ed7e022012-07-31 16:43:35 -07005308int __meminit init_currently_empty_zone(struct zone *zone,
Yasunori Goto718127c2006-06-23 02:03:10 -07005309 unsigned long zone_start_pfn,
Yaowei Baib171e402015-11-05 18:47:06 -08005310 unsigned long size)
Dave Hansened8ece22005-10-29 18:16:50 -07005311{
5312 struct pglist_data *pgdat = zone->zone_pgdat;
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07005313
Dave Hansened8ece22005-10-29 18:16:50 -07005314 pgdat->nr_zones = zone_idx(zone) + 1;
5315
Dave Hansened8ece22005-10-29 18:16:50 -07005316 zone->zone_start_pfn = zone_start_pfn;
5317
Mel Gorman708614e2008-07-23 21:26:51 -07005318 mminit_dprintk(MMINIT_TRACE, "memmap_init",
5319 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
5320 pgdat->node_id,
5321 (unsigned long)zone_idx(zone),
5322 zone_start_pfn, (zone_start_pfn + size));
5323
Andi Kleen1e548de2008-02-04 22:29:26 -08005324 zone_init_free_lists(zone);
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07005325 zone->initialized = 1;
Yasunori Goto718127c2006-06-23 02:03:10 -07005326
5327 return 0;
Dave Hansened8ece22005-10-29 18:16:50 -07005328}
5329
Tejun Heo0ee332c2011-12-08 10:22:09 -08005330#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Mel Gormanc7132162006-09-27 01:49:43 -07005331#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
Mel Gorman8a942fd2015-06-30 14:56:55 -07005332
Mel Gormanc7132162006-09-27 01:49:43 -07005333/*
5334 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
Mel Gormanc7132162006-09-27 01:49:43 -07005335 */
Mel Gorman8a942fd2015-06-30 14:56:55 -07005336int __meminit __early_pfn_to_nid(unsigned long pfn,
5337 struct mminit_pfnnid_cache *state)
Mel Gormanc7132162006-09-27 01:49:43 -07005338{
Tejun Heoc13291a2011-07-12 10:46:30 +02005339 unsigned long start_pfn, end_pfn;
Yinghai Lue76b63f2013-09-11 14:22:17 -07005340 int nid;
Russ Anderson7c243c72013-04-29 15:07:59 -07005341
Mel Gorman8a942fd2015-06-30 14:56:55 -07005342 if (state->last_start <= pfn && pfn < state->last_end)
5343 return state->last_nid;
Mel Gormanc7132162006-09-27 01:49:43 -07005344
Yinghai Lue76b63f2013-09-11 14:22:17 -07005345 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
5346 if (nid != -1) {
Mel Gorman8a942fd2015-06-30 14:56:55 -07005347 state->last_start = start_pfn;
5348 state->last_end = end_pfn;
5349 state->last_nid = nid;
Yinghai Lue76b63f2013-09-11 14:22:17 -07005350 }
5351
5352 return nid;
Mel Gormanc7132162006-09-27 01:49:43 -07005353}
5354#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
5355
Mel Gormanc7132162006-09-27 01:49:43 -07005356/**
Santosh Shilimkar67828322014-01-21 15:50:25 -08005357 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005358 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
Santosh Shilimkar67828322014-01-21 15:50:25 -08005359 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
Mel Gormanc7132162006-09-27 01:49:43 -07005360 *
Zhang Zhen7d018172014-06-04 16:10:53 -07005361 * If an architecture guarantees that all ranges registered contain no holes
5362 * and may be freed, this this function may be used instead of calling
5363 * memblock_free_early_nid() manually.
Mel Gormanc7132162006-09-27 01:49:43 -07005364 */
Tejun Heoc13291a2011-07-12 10:46:30 +02005365void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07005366{
Tejun Heoc13291a2011-07-12 10:46:30 +02005367 unsigned long start_pfn, end_pfn;
5368 int i, this_nid;
Mel Gormanc7132162006-09-27 01:49:43 -07005369
Tejun Heoc13291a2011-07-12 10:46:30 +02005370 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
5371 start_pfn = min(start_pfn, max_low_pfn);
5372 end_pfn = min(end_pfn, max_low_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07005373
Tejun Heoc13291a2011-07-12 10:46:30 +02005374 if (start_pfn < end_pfn)
Santosh Shilimkar67828322014-01-21 15:50:25 -08005375 memblock_free_early_nid(PFN_PHYS(start_pfn),
5376 (end_pfn - start_pfn) << PAGE_SHIFT,
5377 this_nid);
Mel Gormanc7132162006-09-27 01:49:43 -07005378 }
5379}
5380
5381/**
5382 * sparse_memory_present_with_active_regions - Call memory_present for each active range
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005383 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
Mel Gormanc7132162006-09-27 01:49:43 -07005384 *
Zhang Zhen7d018172014-06-04 16:10:53 -07005385 * If an architecture guarantees that all ranges registered contain no holes and may
5386 * be freed, this function may be used instead of calling memory_present() manually.
Mel Gormanc7132162006-09-27 01:49:43 -07005387 */
5388void __init sparse_memory_present_with_active_regions(int nid)
5389{
Tejun Heoc13291a2011-07-12 10:46:30 +02005390 unsigned long start_pfn, end_pfn;
5391 int i, this_nid;
Mel Gormanc7132162006-09-27 01:49:43 -07005392
Tejun Heoc13291a2011-07-12 10:46:30 +02005393 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
5394 memory_present(this_nid, start_pfn, end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07005395}
5396
5397/**
5398 * get_pfn_range_for_nid - Return the start and end page frames for a node
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005399 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
5400 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
5401 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
Mel Gormanc7132162006-09-27 01:49:43 -07005402 *
5403 * It returns the start and end page frame of a node based on information
Zhang Zhen7d018172014-06-04 16:10:53 -07005404 * provided by memblock_set_node(). If called for a node
Mel Gormanc7132162006-09-27 01:49:43 -07005405 * with no available memory, a warning is printed and the start and end
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005406 * PFNs will be 0.
Mel Gormanc7132162006-09-27 01:49:43 -07005407 */
Yasunori Gotoa3142c82007-05-08 00:23:07 -07005408void __meminit get_pfn_range_for_nid(unsigned int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07005409 unsigned long *start_pfn, unsigned long *end_pfn)
5410{
Tejun Heoc13291a2011-07-12 10:46:30 +02005411 unsigned long this_start_pfn, this_end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07005412 int i;
Tejun Heoc13291a2011-07-12 10:46:30 +02005413
Mel Gormanc7132162006-09-27 01:49:43 -07005414 *start_pfn = -1UL;
5415 *end_pfn = 0;
5416
Tejun Heoc13291a2011-07-12 10:46:30 +02005417 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
5418 *start_pfn = min(*start_pfn, this_start_pfn);
5419 *end_pfn = max(*end_pfn, this_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07005420 }
5421
Christoph Lameter633c0662007-10-16 01:25:37 -07005422 if (*start_pfn == -1UL)
Mel Gormanc7132162006-09-27 01:49:43 -07005423 *start_pfn = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07005424}
5425
5426/*
Mel Gorman2a1e2742007-07-17 04:03:12 -07005427 * This finds a zone that can be used for ZONE_MOVABLE pages. The
5428 * assumption is made that zones within a node are ordered in monotonic
5429 * increasing memory addresses so that the "highest" populated zone is used
5430 */
Adrian Bunkb69a7282008-07-23 21:28:12 -07005431static void __init find_usable_zone_for_movable(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07005432{
5433 int zone_index;
5434 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
5435 if (zone_index == ZONE_MOVABLE)
5436 continue;
5437
5438 if (arch_zone_highest_possible_pfn[zone_index] >
5439 arch_zone_lowest_possible_pfn[zone_index])
5440 break;
5441 }
5442
5443 VM_BUG_ON(zone_index == -1);
5444 movable_zone = zone_index;
5445}
5446
5447/*
5448 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
Lucas De Marchi25985ed2011-03-30 22:57:33 -03005449 * because it is sized independent of architecture. Unlike the other zones,
Mel Gorman2a1e2742007-07-17 04:03:12 -07005450 * the starting point for ZONE_MOVABLE is not fixed. It may be different
5451 * in each node depending on the size of each node and how evenly kernelcore
5452 * is distributed. This helper function adjusts the zone ranges
5453 * provided by the architecture for a given node by using the end of the
5454 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
5455 * zones within a node are in order of monotonic increases memory addresses
5456 */
Adrian Bunkb69a7282008-07-23 21:28:12 -07005457static void __meminit adjust_zone_range_for_zone_movable(int nid,
Mel Gorman2a1e2742007-07-17 04:03:12 -07005458 unsigned long zone_type,
5459 unsigned long node_start_pfn,
5460 unsigned long node_end_pfn,
5461 unsigned long *zone_start_pfn,
5462 unsigned long *zone_end_pfn)
5463{
5464 /* Only adjust if ZONE_MOVABLE is on this node */
5465 if (zone_movable_pfn[nid]) {
5466 /* Size ZONE_MOVABLE */
5467 if (zone_type == ZONE_MOVABLE) {
5468 *zone_start_pfn = zone_movable_pfn[nid];
5469 *zone_end_pfn = min(node_end_pfn,
5470 arch_zone_highest_possible_pfn[movable_zone]);
5471
Xishi Qiue506b992016-10-07 16:58:06 -07005472 /* Adjust for ZONE_MOVABLE starting within this range */
5473 } else if (!mirrored_kernelcore &&
5474 *zone_start_pfn < zone_movable_pfn[nid] &&
5475 *zone_end_pfn > zone_movable_pfn[nid]) {
5476 *zone_end_pfn = zone_movable_pfn[nid];
5477
Mel Gorman2a1e2742007-07-17 04:03:12 -07005478 /* Check if this whole range is within ZONE_MOVABLE */
5479 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
5480 *zone_start_pfn = *zone_end_pfn;
5481 }
5482}
5483
5484/*
Mel Gormanc7132162006-09-27 01:49:43 -07005485 * Return the number of pages a zone spans in a node, including holes
5486 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
5487 */
Paul Mundt6ea6e682007-07-15 23:38:20 -07005488static unsigned long __meminit zone_spanned_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07005489 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005490 unsigned long node_start_pfn,
5491 unsigned long node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07005492 unsigned long *zone_start_pfn,
5493 unsigned long *zone_end_pfn,
Mel Gormanc7132162006-09-27 01:49:43 -07005494 unsigned long *ignored)
5495{
Linxu Fang121100e2019-05-13 17:19:17 -07005496 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
5497 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Xishi Qiub5685e92015-09-08 15:04:16 -07005498 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07005499 if (!node_start_pfn && !node_end_pfn)
5500 return 0;
5501
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005502 /* Get the start and end of the zone */
Linxu Fang121100e2019-05-13 17:19:17 -07005503 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
5504 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman2a1e2742007-07-17 04:03:12 -07005505 adjust_zone_range_for_zone_movable(nid, zone_type,
5506 node_start_pfn, node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07005507 zone_start_pfn, zone_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07005508
5509 /* Check that this node has pages within the zone's required range */
Taku Izumid91749c2016-03-15 14:55:18 -07005510 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07005511 return 0;
5512
5513 /* Move the zone boundaries inside the node if necessary */
Taku Izumid91749c2016-03-15 14:55:18 -07005514 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
5515 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07005516
5517 /* Return the spanned pages */
Taku Izumid91749c2016-03-15 14:55:18 -07005518 return *zone_end_pfn - *zone_start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07005519}
5520
5521/*
5522 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005523 * then all holes in the requested range will be accounted for.
Mel Gormanc7132162006-09-27 01:49:43 -07005524 */
Yinghai Lu32996252009-12-15 17:59:02 -08005525unsigned long __meminit __absent_pages_in_range(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07005526 unsigned long range_start_pfn,
5527 unsigned long range_end_pfn)
5528{
Tejun Heo96e907d2011-07-12 10:46:29 +02005529 unsigned long nr_absent = range_end_pfn - range_start_pfn;
5530 unsigned long start_pfn, end_pfn;
5531 int i;
Mel Gormanc7132162006-09-27 01:49:43 -07005532
Tejun Heo96e907d2011-07-12 10:46:29 +02005533 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
5534 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
5535 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
5536 nr_absent -= end_pfn - start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07005537 }
Tejun Heo96e907d2011-07-12 10:46:29 +02005538 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07005539}
5540
5541/**
5542 * absent_pages_in_range - Return number of page frames in holes within a range
5543 * @start_pfn: The start PFN to start searching for holes
5544 * @end_pfn: The end PFN to stop searching for holes
5545 *
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005546 * It returns the number of pages frames in memory holes within a range.
Mel Gormanc7132162006-09-27 01:49:43 -07005547 */
5548unsigned long __init absent_pages_in_range(unsigned long start_pfn,
5549 unsigned long end_pfn)
5550{
5551 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
5552}
5553
5554/* Return the number of page frames in holes in a zone on a node */
Paul Mundt6ea6e682007-07-15 23:38:20 -07005555static unsigned long __meminit zone_absent_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07005556 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005557 unsigned long node_start_pfn,
5558 unsigned long node_end_pfn,
Mel Gormanc7132162006-09-27 01:49:43 -07005559 unsigned long *ignored)
5560{
Tejun Heo96e907d2011-07-12 10:46:29 +02005561 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
5562 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Mel Gorman9c7cd682006-09-27 01:49:58 -07005563 unsigned long zone_start_pfn, zone_end_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07005564 unsigned long nr_absent;
Mel Gorman9c7cd682006-09-27 01:49:58 -07005565
Xishi Qiub5685e92015-09-08 15:04:16 -07005566 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07005567 if (!node_start_pfn && !node_end_pfn)
5568 return 0;
5569
Tejun Heo96e907d2011-07-12 10:46:29 +02005570 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
5571 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman9c7cd682006-09-27 01:49:58 -07005572
Mel Gorman2a1e2742007-07-17 04:03:12 -07005573 adjust_zone_range_for_zone_movable(nid, zone_type,
5574 node_start_pfn, node_end_pfn,
5575 &zone_start_pfn, &zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07005576 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
5577
5578 /*
5579 * ZONE_MOVABLE handling.
5580 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
5581 * and vice versa.
5582 */
Xishi Qiue506b992016-10-07 16:58:06 -07005583 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
5584 unsigned long start_pfn, end_pfn;
5585 struct memblock_region *r;
Taku Izumi342332e2016-03-15 14:55:22 -07005586
Xishi Qiue506b992016-10-07 16:58:06 -07005587 for_each_memblock(memory, r) {
5588 start_pfn = clamp(memblock_region_memory_base_pfn(r),
5589 zone_start_pfn, zone_end_pfn);
5590 end_pfn = clamp(memblock_region_memory_end_pfn(r),
5591 zone_start_pfn, zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07005592
Xishi Qiue506b992016-10-07 16:58:06 -07005593 if (zone_type == ZONE_MOVABLE &&
5594 memblock_is_mirror(r))
5595 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07005596
Xishi Qiue506b992016-10-07 16:58:06 -07005597 if (zone_type == ZONE_NORMAL &&
5598 !memblock_is_mirror(r))
5599 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07005600 }
5601 }
5602
5603 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07005604}
Mel Gorman0e0b8642006-09-27 01:49:56 -07005605
Tejun Heo0ee332c2011-12-08 10:22:09 -08005606#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Paul Mundt6ea6e682007-07-15 23:38:20 -07005607static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07005608 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005609 unsigned long node_start_pfn,
5610 unsigned long node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07005611 unsigned long *zone_start_pfn,
5612 unsigned long *zone_end_pfn,
Mel Gormanc7132162006-09-27 01:49:43 -07005613 unsigned long *zones_size)
5614{
Taku Izumid91749c2016-03-15 14:55:18 -07005615 unsigned int zone;
5616
5617 *zone_start_pfn = node_start_pfn;
5618 for (zone = 0; zone < zone_type; zone++)
5619 *zone_start_pfn += zones_size[zone];
5620
5621 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type];
5622
Mel Gormanc7132162006-09-27 01:49:43 -07005623 return zones_size[zone_type];
5624}
5625
Paul Mundt6ea6e682007-07-15 23:38:20 -07005626static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07005627 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005628 unsigned long node_start_pfn,
5629 unsigned long node_end_pfn,
Mel Gormanc7132162006-09-27 01:49:43 -07005630 unsigned long *zholes_size)
5631{
5632 if (!zholes_size)
5633 return 0;
5634
5635 return zholes_size[zone_type];
5636}
Yinghai Lu20e69262013-03-01 14:51:27 -08005637
Tejun Heo0ee332c2011-12-08 10:22:09 -08005638#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -07005639
Yasunori Gotoa3142c82007-05-08 00:23:07 -07005640static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005641 unsigned long node_start_pfn,
5642 unsigned long node_end_pfn,
5643 unsigned long *zones_size,
5644 unsigned long *zholes_size)
Mel Gormanc7132162006-09-27 01:49:43 -07005645{
Gu Zhengfebd5942015-06-24 16:57:02 -07005646 unsigned long realtotalpages = 0, totalpages = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07005647 enum zone_type i;
5648
Gu Zhengfebd5942015-06-24 16:57:02 -07005649 for (i = 0; i < MAX_NR_ZONES; i++) {
5650 struct zone *zone = pgdat->node_zones + i;
Taku Izumid91749c2016-03-15 14:55:18 -07005651 unsigned long zone_start_pfn, zone_end_pfn;
Gu Zhengfebd5942015-06-24 16:57:02 -07005652 unsigned long size, real_size;
Mel Gormanc7132162006-09-27 01:49:43 -07005653
Gu Zhengfebd5942015-06-24 16:57:02 -07005654 size = zone_spanned_pages_in_node(pgdat->node_id, i,
5655 node_start_pfn,
5656 node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07005657 &zone_start_pfn,
5658 &zone_end_pfn,
Gu Zhengfebd5942015-06-24 16:57:02 -07005659 zones_size);
5660 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005661 node_start_pfn, node_end_pfn,
5662 zholes_size);
Taku Izumid91749c2016-03-15 14:55:18 -07005663 if (size)
5664 zone->zone_start_pfn = zone_start_pfn;
5665 else
5666 zone->zone_start_pfn = 0;
Gu Zhengfebd5942015-06-24 16:57:02 -07005667 zone->spanned_pages = size;
5668 zone->present_pages = real_size;
5669
5670 totalpages += size;
5671 realtotalpages += real_size;
5672 }
5673
5674 pgdat->node_spanned_pages = totalpages;
Mel Gormanc7132162006-09-27 01:49:43 -07005675 pgdat->node_present_pages = realtotalpages;
5676 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
5677 realtotalpages);
5678}
5679
Mel Gorman835c1342007-10-16 01:25:47 -07005680#ifndef CONFIG_SPARSEMEM
5681/*
5682 * Calculate the size of the zone->blockflags rounded to an unsigned long
Mel Gormand9c23402007-10-16 01:26:01 -07005683 * Start by making sure zonesize is a multiple of pageblock_order by rounding
5684 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
Mel Gorman835c1342007-10-16 01:25:47 -07005685 * round what is now in bits to nearest long in bits, then return it in
5686 * bytes.
5687 */
Linus Torvalds7c455122013-02-18 09:58:02 -08005688static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
Mel Gorman835c1342007-10-16 01:25:47 -07005689{
5690 unsigned long usemapsize;
5691
Linus Torvalds7c455122013-02-18 09:58:02 -08005692 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
Mel Gormand9c23402007-10-16 01:26:01 -07005693 usemapsize = roundup(zonesize, pageblock_nr_pages);
5694 usemapsize = usemapsize >> pageblock_order;
Mel Gorman835c1342007-10-16 01:25:47 -07005695 usemapsize *= NR_PAGEBLOCK_BITS;
5696 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
5697
5698 return usemapsize / 8;
5699}
5700
5701static void __init setup_usemap(struct pglist_data *pgdat,
Linus Torvalds7c455122013-02-18 09:58:02 -08005702 struct zone *zone,
5703 unsigned long zone_start_pfn,
5704 unsigned long zonesize)
Mel Gorman835c1342007-10-16 01:25:47 -07005705{
Linus Torvalds7c455122013-02-18 09:58:02 -08005706 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
Mel Gorman835c1342007-10-16 01:25:47 -07005707 zone->pageblock_flags = NULL;
Julia Lawall58a01a42009-01-06 14:39:28 -08005708 if (usemapsize)
Santosh Shilimkar67828322014-01-21 15:50:25 -08005709 zone->pageblock_flags =
5710 memblock_virt_alloc_node_nopanic(usemapsize,
5711 pgdat->node_id);
Mel Gorman835c1342007-10-16 01:25:47 -07005712}
5713#else
Linus Torvalds7c455122013-02-18 09:58:02 -08005714static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
5715 unsigned long zone_start_pfn, unsigned long zonesize) {}
Mel Gorman835c1342007-10-16 01:25:47 -07005716#endif /* CONFIG_SPARSEMEM */
5717
Mel Gormand9c23402007-10-16 01:26:01 -07005718#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Mel Gormanba72cb82007-11-28 16:21:13 -08005719
Mel Gormand9c23402007-10-16 01:26:01 -07005720/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
Chen Gang15ca2202013-09-11 14:20:27 -07005721void __paginginit set_pageblock_order(void)
Mel Gormand9c23402007-10-16 01:26:01 -07005722{
Andrew Morton955c1cd2012-05-29 15:06:31 -07005723 unsigned int order;
5724
Mel Gormand9c23402007-10-16 01:26:01 -07005725 /* Check that pageblock_nr_pages has not already been setup */
5726 if (pageblock_order)
5727 return;
5728
Andrew Morton955c1cd2012-05-29 15:06:31 -07005729 if (HPAGE_SHIFT > PAGE_SHIFT)
5730 order = HUGETLB_PAGE_ORDER;
5731 else
5732 order = MAX_ORDER - 1;
5733
Mel Gormand9c23402007-10-16 01:26:01 -07005734 /*
5735 * Assume the largest contiguous order of interest is a huge page.
Andrew Morton955c1cd2012-05-29 15:06:31 -07005736 * This value may be variable depending on boot parameters on IA64 and
5737 * powerpc.
Mel Gormand9c23402007-10-16 01:26:01 -07005738 */
5739 pageblock_order = order;
5740}
5741#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
5742
Mel Gormanba72cb82007-11-28 16:21:13 -08005743/*
5744 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
Andrew Morton955c1cd2012-05-29 15:06:31 -07005745 * is unused as pageblock_order is set at compile-time. See
5746 * include/linux/pageblock-flags.h for the values of pageblock_order based on
5747 * the kernel config
Mel Gormanba72cb82007-11-28 16:21:13 -08005748 */
Chen Gang15ca2202013-09-11 14:20:27 -07005749void __paginginit set_pageblock_order(void)
Mel Gormanba72cb82007-11-28 16:21:13 -08005750{
Mel Gormanba72cb82007-11-28 16:21:13 -08005751}
Mel Gormand9c23402007-10-16 01:26:01 -07005752
5753#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
5754
Jiang Liu01cefae2012-12-12 13:52:19 -08005755static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
5756 unsigned long present_pages)
5757{
5758 unsigned long pages = spanned_pages;
5759
5760 /*
5761 * Provide a more accurate estimation if there are holes within
5762 * the zone and SPARSEMEM is in use. If there are holes within the
5763 * zone, each populated memory region may cost us one or two extra
5764 * memmap pages due to alignment because memmap pages for each
5765 * populated regions may not naturally algined on page boundary.
5766 * So the (present_pages >> 4) heuristic is a tradeoff for that.
5767 */
5768 if (spanned_pages > present_pages + (present_pages >> 4) &&
5769 IS_ENABLED(CONFIG_SPARSEMEM))
5770 pages = present_pages;
5771
5772 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
5773}
5774
Linus Torvalds1da177e2005-04-16 15:20:36 -07005775/*
5776 * Set up the zone data structures:
5777 * - mark all pages reserved
5778 * - mark all memory queues empty
5779 * - clear the memory bitmaps
Minchan Kim6527af52012-07-31 16:46:16 -07005780 *
5781 * NOTE: pgdat should get zeroed by caller.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005782 */
Wei Yang7f3eb552015-09-08 14:59:50 -07005783static void __paginginit free_area_init_core(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005784{
Christoph Lameter2f1b6242006-09-25 23:31:13 -07005785 enum zone_type j;
Dave Hansened8ece22005-10-29 18:16:50 -07005786 int nid = pgdat->node_id;
Yasunori Goto718127c2006-06-23 02:03:10 -07005787 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005788
Dave Hansen208d54e2005-10-29 18:16:52 -07005789 pgdat_resize_init(pgdat);
Andrea Arcangeli8177a422012-03-23 20:56:34 +01005790#ifdef CONFIG_NUMA_BALANCING
5791 spin_lock_init(&pgdat->numabalancing_migrate_lock);
5792 pgdat->numabalancing_migrate_nr_pages = 0;
5793 pgdat->numabalancing_migrate_next_window = jiffies;
5794#endif
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08005795#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5796 spin_lock_init(&pgdat->split_queue_lock);
5797 INIT_LIST_HEAD(&pgdat->split_queue);
5798 pgdat->split_queue_len = 0;
5799#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005800 init_waitqueue_head(&pgdat->kswapd_wait);
Mel Gorman55150612012-07-31 16:44:35 -07005801 init_waitqueue_head(&pgdat->pfmemalloc_wait);
Vlastimil Babka698b1b32016-03-17 14:18:08 -07005802#ifdef CONFIG_COMPACTION
5803 init_waitqueue_head(&pgdat->kcompactd_wait);
5804#endif
Joonsoo Kimeefa8642014-12-12 16:55:46 -08005805 pgdat_page_ext_init(pgdat);
Mel Gormana52633d2016-07-28 15:45:28 -07005806 spin_lock_init(&pgdat->lru_lock);
Mel Gormana9dd0a82016-07-28 15:46:02 -07005807 lruvec_init(node_lruvec(pgdat));
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01005808
Linus Torvalds1da177e2005-04-16 15:20:36 -07005809 for (j = 0; j < MAX_NR_ZONES; j++) {
5810 struct zone *zone = pgdat->node_zones + j;
Jiang Liu9feedc92012-12-12 13:52:12 -08005811 unsigned long size, realsize, freesize, memmap_pages;
Taku Izumid91749c2016-03-15 14:55:18 -07005812 unsigned long zone_start_pfn = zone->zone_start_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005813
Gu Zhengfebd5942015-06-24 16:57:02 -07005814 size = zone->spanned_pages;
5815 realsize = freesize = zone->present_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005816
Mel Gorman0e0b8642006-09-27 01:49:56 -07005817 /*
Jiang Liu9feedc92012-12-12 13:52:12 -08005818 * Adjust freesize so that it accounts for how much memory
Mel Gorman0e0b8642006-09-27 01:49:56 -07005819 * is used by this zone for memmap. This affects the watermark
5820 * and per-cpu initialisations
5821 */
Jiang Liu01cefae2012-12-12 13:52:19 -08005822 memmap_pages = calc_memmap_size(size, realsize);
Zhong Hongboba914f42014-12-12 16:56:21 -08005823 if (!is_highmem_idx(j)) {
5824 if (freesize >= memmap_pages) {
5825 freesize -= memmap_pages;
5826 if (memmap_pages)
5827 printk(KERN_DEBUG
5828 " %s zone: %lu pages used for memmap\n",
5829 zone_names[j], memmap_pages);
5830 } else
Joe Perches11705322016-03-17 14:19:50 -07005831 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
Zhong Hongboba914f42014-12-12 16:56:21 -08005832 zone_names[j], memmap_pages, freesize);
5833 }
Mel Gorman0e0b8642006-09-27 01:49:56 -07005834
Christoph Lameter62672762007-02-10 01:43:07 -08005835 /* Account for reserved pages */
Jiang Liu9feedc92012-12-12 13:52:12 -08005836 if (j == 0 && freesize > dma_reserve) {
5837 freesize -= dma_reserve;
Yinghai Lud903ef92008-10-18 20:27:06 -07005838 printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
Christoph Lameter62672762007-02-10 01:43:07 -08005839 zone_names[0], dma_reserve);
Mel Gorman0e0b8642006-09-27 01:49:56 -07005840 }
5841
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005842 if (!is_highmem_idx(j))
Jiang Liu9feedc92012-12-12 13:52:12 -08005843 nr_kernel_pages += freesize;
Jiang Liu01cefae2012-12-12 13:52:19 -08005844 /* Charge for highmem memmap if there are enough kernel pages */
5845 else if (nr_kernel_pages > memmap_pages * 2)
5846 nr_kernel_pages -= memmap_pages;
Jiang Liu9feedc92012-12-12 13:52:12 -08005847 nr_all_pages += freesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005848
Jiang Liu9feedc92012-12-12 13:52:12 -08005849 /*
5850 * Set an approximate value for lowmem here, it will be adjusted
5851 * when the bootmem allocator frees pages into the buddy system.
5852 * And all highmem pages will be managed by the buddy system.
5853 */
5854 zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
Christoph Lameter96146342006-07-03 00:24:13 -07005855#ifdef CONFIG_NUMA
Christoph Lameterd5f541e2006-09-27 01:50:08 -07005856 zone->node = nid;
Christoph Lameter96146342006-07-03 00:24:13 -07005857#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005858 zone->name = zone_names[j];
Linus Torvalds1da177e2005-04-16 15:20:36 -07005859 zone->zone_pgdat = pgdat;
Mel Gormana52633d2016-07-28 15:45:28 -07005860 spin_lock_init(&zone->lock);
5861 zone_seqlock_init(zone);
Dave Hansened8ece22005-10-29 18:16:50 -07005862 zone_pcp_init(zone);
Johannes Weiner81c0a2b2013-09-11 14:20:47 -07005863
Linus Torvalds1da177e2005-04-16 15:20:36 -07005864 if (!size)
5865 continue;
5866
Andrew Morton955c1cd2012-05-29 15:06:31 -07005867 set_pageblock_order();
Linus Torvalds7c455122013-02-18 09:58:02 -08005868 setup_usemap(pgdat, zone, zone_start_pfn, size);
Yaowei Baib171e402015-11-05 18:47:06 -08005869 ret = init_currently_empty_zone(zone, zone_start_pfn, size);
Yasunori Goto718127c2006-06-23 02:03:10 -07005870 BUG_ON(ret);
Heiko Carstens76cdd582008-05-14 16:05:52 -07005871 memmap_init(size, nid, j, zone_start_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005872 }
5873}
5874
Fabian Frederickbd721ea2016-08-02 14:03:33 -07005875static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005876{
Tony Luckb0aeba72015-11-10 10:09:47 -08005877 unsigned long __maybe_unused start = 0;
Laura Abbotta1c34a32015-11-05 18:48:46 -08005878 unsigned long __maybe_unused offset = 0;
5879
Linus Torvalds1da177e2005-04-16 15:20:36 -07005880 /* Skip empty nodes */
5881 if (!pgdat->node_spanned_pages)
5882 return;
5883
Andy Whitcroftd41dee32005-06-23 00:07:54 -07005884#ifdef CONFIG_FLAT_NODE_MEM_MAP
Tony Luckb0aeba72015-11-10 10:09:47 -08005885 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
5886 offset = pgdat->node_start_pfn - start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005887 /* ia64 gets its own node_mem_map, before this, without bootmem */
5888 if (!pgdat->node_mem_map) {
Tony Luckb0aeba72015-11-10 10:09:47 -08005889 unsigned long size, end;
Andy Whitcroftd41dee32005-06-23 00:07:54 -07005890 struct page *map;
5891
Bob Piccoe984bb42006-05-20 15:00:31 -07005892 /*
5893 * The zone's endpoints aren't required to be MAX_ORDER
5894 * aligned but the node_mem_map endpoints must be in order
5895 * for the buddy allocator to function correctly.
5896 */
Cody P Schafer108bcc92013-02-22 16:35:23 -08005897 end = pgdat_end_pfn(pgdat);
Bob Piccoe984bb42006-05-20 15:00:31 -07005898 end = ALIGN(end, MAX_ORDER_NR_PAGES);
5899 size = (end - start) * sizeof(struct page);
Dave Hansen6f167ec2005-06-23 00:07:39 -07005900 map = alloc_remap(pgdat->node_id, size);
5901 if (!map)
Santosh Shilimkar67828322014-01-21 15:50:25 -08005902 map = memblock_virt_alloc_node_nopanic(size,
5903 pgdat->node_id);
Laura Abbotta1c34a32015-11-05 18:48:46 -08005904 pgdat->node_mem_map = map + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005905 }
Roman Zippel12d810c2007-05-31 00:40:54 -07005906#ifndef CONFIG_NEED_MULTIPLE_NODES
Linus Torvalds1da177e2005-04-16 15:20:36 -07005907 /*
5908 * With no DISCONTIG, the global mem_map is just set as node 0's
5909 */
Mel Gormanc7132162006-09-27 01:49:43 -07005910 if (pgdat == NODE_DATA(0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005911 mem_map = NODE_DATA(0)->node_mem_map;
Laura Abbotta1c34a32015-11-05 18:48:46 -08005912#if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
Mel Gormanc7132162006-09-27 01:49:43 -07005913 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
Laura Abbotta1c34a32015-11-05 18:48:46 -08005914 mem_map -= offset;
Tejun Heo0ee332c2011-12-08 10:22:09 -08005915#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -07005916 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005917#endif
Andy Whitcroftd41dee32005-06-23 00:07:54 -07005918#endif /* CONFIG_FLAT_NODE_MEM_MAP */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005919}
5920
Johannes Weiner9109fb72008-07-23 21:27:20 -07005921void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
5922 unsigned long node_start_pfn, unsigned long *zholes_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005923{
Johannes Weiner9109fb72008-07-23 21:27:20 -07005924 pg_data_t *pgdat = NODE_DATA(nid);
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005925 unsigned long start_pfn = 0;
5926 unsigned long end_pfn = 0;
Johannes Weiner9109fb72008-07-23 21:27:20 -07005927
Minchan Kim88fdf752012-07-31 16:46:14 -07005928 /* pg_data_t should be reset to zero when it's allocated */
Mel Gorman38087d92016-07-28 15:45:49 -07005929 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
Minchan Kim88fdf752012-07-31 16:46:14 -07005930
Linus Torvalds1da177e2005-04-16 15:20:36 -07005931 pgdat->node_id = nid;
5932 pgdat->node_start_pfn = node_start_pfn;
Mel Gorman75ef7182016-07-28 15:45:24 -07005933 pgdat->per_cpu_nodestats = NULL;
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005934#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5935 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
Juergen Gross8d29e182015-02-11 15:26:01 -08005936 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
Zhen Lei4ada0c52015-09-08 15:04:19 -07005937 (u64)start_pfn << PAGE_SHIFT,
5938 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
Taku Izumid91749c2016-03-15 14:55:18 -07005939#else
5940 start_pfn = node_start_pfn;
Zhang Yanfei7960aed2013-07-08 15:59:52 -07005941#endif
5942 calculate_node_totalpages(pgdat, start_pfn, end_pfn,
5943 zones_size, zholes_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005944
5945 alloc_node_mem_map(pgdat);
Yinghai Lue8c27ac2008-06-01 13:15:22 -07005946#ifdef CONFIG_FLAT_NODE_MEM_MAP
5947 printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
5948 nid, (unsigned long)pgdat,
5949 (unsigned long)pgdat->node_mem_map);
5950#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005951
Michal Hocko292f70c2017-06-02 14:46:49 -07005952 reset_deferred_meminit(pgdat);
Wei Yang7f3eb552015-09-08 14:59:50 -07005953 free_area_init_core(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005954}
5955
Tejun Heo0ee332c2011-12-08 10:22:09 -08005956#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Miklos Szeredi418508c2007-05-23 13:57:55 -07005957
5958#if MAX_NUMNODES > 1
5959/*
5960 * Figure out the number of possible node ids.
5961 */
Cody P Schaferf9872ca2013-04-29 15:08:01 -07005962void __init setup_nr_node_ids(void)
Miklos Szeredi418508c2007-05-23 13:57:55 -07005963{
Wei Yang904a9552015-09-08 14:59:48 -07005964 unsigned int highest;
Miklos Szeredi418508c2007-05-23 13:57:55 -07005965
Wei Yang904a9552015-09-08 14:59:48 -07005966 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
Miklos Szeredi418508c2007-05-23 13:57:55 -07005967 nr_node_ids = highest + 1;
5968}
Miklos Szeredi418508c2007-05-23 13:57:55 -07005969#endif
5970
Mel Gormanc7132162006-09-27 01:49:43 -07005971/**
Tejun Heo1e019792011-07-12 09:45:34 +02005972 * node_map_pfn_alignment - determine the maximum internode alignment
5973 *
5974 * This function should be called after node map is populated and sorted.
5975 * It calculates the maximum power of two alignment which can distinguish
5976 * all the nodes.
5977 *
5978 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
5979 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
5980 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
5981 * shifted, 1GiB is enough and this function will indicate so.
5982 *
5983 * This is used to test whether pfn -> nid mapping of the chosen memory
5984 * model has fine enough granularity to avoid incorrect mapping for the
5985 * populated node map.
5986 *
5987 * Returns the determined alignment in pfn's. 0 if there is no alignment
5988 * requirement (single node).
5989 */
5990unsigned long __init node_map_pfn_alignment(void)
5991{
5992 unsigned long accl_mask = 0, last_end = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02005993 unsigned long start, end, mask;
Tejun Heo1e019792011-07-12 09:45:34 +02005994 int last_nid = -1;
Tejun Heoc13291a2011-07-12 10:46:30 +02005995 int i, nid;
Tejun Heo1e019792011-07-12 09:45:34 +02005996
Tejun Heoc13291a2011-07-12 10:46:30 +02005997 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
Tejun Heo1e019792011-07-12 09:45:34 +02005998 if (!start || last_nid < 0 || last_nid == nid) {
5999 last_nid = nid;
6000 last_end = end;
6001 continue;
6002 }
6003
6004 /*
6005 * Start with a mask granular enough to pin-point to the
6006 * start pfn and tick off bits one-by-one until it becomes
6007 * too coarse to separate the current node from the last.
6008 */
6009 mask = ~((1 << __ffs(start)) - 1);
6010 while (mask && last_end <= (start & (mask << 1)))
6011 mask <<= 1;
6012
6013 /* accumulate all internode masks */
6014 accl_mask |= mask;
6015 }
6016
6017 /* convert mask to number of pages */
6018 return ~accl_mask + 1;
6019}
6020
Mel Gormana6af2bc2007-02-10 01:42:57 -08006021/* Find the lowest pfn for a node */
Adrian Bunkb69a7282008-07-23 21:28:12 -07006022static unsigned long __init find_min_pfn_for_node(int nid)
Mel Gormanc7132162006-09-27 01:49:43 -07006023{
Mel Gormana6af2bc2007-02-10 01:42:57 -08006024 unsigned long min_pfn = ULONG_MAX;
Tejun Heoc13291a2011-07-12 10:46:30 +02006025 unsigned long start_pfn;
6026 int i;
Mel Gorman1abbfb42006-11-23 12:01:41 +00006027
Tejun Heoc13291a2011-07-12 10:46:30 +02006028 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
6029 min_pfn = min(min_pfn, start_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006030
Mel Gormana6af2bc2007-02-10 01:42:57 -08006031 if (min_pfn == ULONG_MAX) {
Joe Perches11705322016-03-17 14:19:50 -07006032 pr_warn("Could not find start_pfn for node %d\n", nid);
Mel Gormana6af2bc2007-02-10 01:42:57 -08006033 return 0;
6034 }
6035
6036 return min_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006037}
6038
6039/**
6040 * find_min_pfn_with_active_regions - Find the minimum PFN registered
6041 *
6042 * It returns the minimum PFN based on information provided via
Zhang Zhen7d018172014-06-04 16:10:53 -07006043 * memblock_set_node().
Mel Gormanc7132162006-09-27 01:49:43 -07006044 */
6045unsigned long __init find_min_pfn_with_active_regions(void)
6046{
6047 return find_min_pfn_for_node(MAX_NUMNODES);
6048}
6049
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006050/*
6051 * early_calculate_totalpages()
6052 * Sum pages in active regions for movable zone.
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006053 * Populate N_MEMORY for calculating usable_nodes.
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006054 */
Adrian Bunk484f51f2007-10-16 01:26:03 -07006055static unsigned long __init early_calculate_totalpages(void)
Mel Gorman7e63efe2007-07-17 04:03:15 -07006056{
Mel Gorman7e63efe2007-07-17 04:03:15 -07006057 unsigned long totalpages = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02006058 unsigned long start_pfn, end_pfn;
6059 int i, nid;
Mel Gorman7e63efe2007-07-17 04:03:15 -07006060
Tejun Heoc13291a2011-07-12 10:46:30 +02006061 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
6062 unsigned long pages = end_pfn - start_pfn;
6063
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006064 totalpages += pages;
6065 if (pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006066 node_set_state(nid, N_MEMORY);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006067 }
Pintu Kumarb8af2942013-09-11 14:20:34 -07006068 return totalpages;
Mel Gorman7e63efe2007-07-17 04:03:15 -07006069}
6070
Mel Gorman2a1e2742007-07-17 04:03:12 -07006071/*
6072 * Find the PFN the Movable zone begins in each node. Kernel memory
6073 * is spread evenly between nodes as long as the nodes have enough
6074 * memory. When they don't, some nodes will have more kernelcore than
6075 * others
6076 */
Kautuk Consulb224ef82012-03-21 16:34:15 -07006077static void __init find_zone_movable_pfns_for_nodes(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07006078{
6079 int i, nid;
6080 unsigned long usable_startpfn;
6081 unsigned long kernelcore_node, kernelcore_remaining;
Yinghai Lu66918dc2009-06-30 11:41:37 -07006082 /* save the state before borrow the nodemask */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006083 nodemask_t saved_node_state = node_states[N_MEMORY];
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006084 unsigned long totalpages = early_calculate_totalpages();
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006085 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
Emil Medve136199f2014-04-07 15:37:52 -07006086 struct memblock_region *r;
Tang Chenb2f3eeb2014-01-21 15:49:38 -08006087
6088 /* Need to find movable_zone earlier when movable_node is specified. */
6089 find_usable_zone_for_movable();
Mel Gorman2a1e2742007-07-17 04:03:12 -07006090
Mel Gorman7e63efe2007-07-17 04:03:15 -07006091 /*
Tang Chenb2f3eeb2014-01-21 15:49:38 -08006092 * If movable_node is specified, ignore kernelcore and movablecore
6093 * options.
6094 */
6095 if (movable_node_is_enabled()) {
Emil Medve136199f2014-04-07 15:37:52 -07006096 for_each_memblock(memory, r) {
6097 if (!memblock_is_hotpluggable(r))
Tang Chenb2f3eeb2014-01-21 15:49:38 -08006098 continue;
6099
Emil Medve136199f2014-04-07 15:37:52 -07006100 nid = r->nid;
Tang Chenb2f3eeb2014-01-21 15:49:38 -08006101
Emil Medve136199f2014-04-07 15:37:52 -07006102 usable_startpfn = PFN_DOWN(r->base);
Tang Chenb2f3eeb2014-01-21 15:49:38 -08006103 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
6104 min(usable_startpfn, zone_movable_pfn[nid]) :
6105 usable_startpfn;
6106 }
6107
6108 goto out2;
6109 }
6110
6111 /*
Taku Izumi342332e2016-03-15 14:55:22 -07006112 * If kernelcore=mirror is specified, ignore movablecore option
6113 */
6114 if (mirrored_kernelcore) {
6115 bool mem_below_4gb_not_mirrored = false;
6116
6117 for_each_memblock(memory, r) {
6118 if (memblock_is_mirror(r))
6119 continue;
6120
6121 nid = r->nid;
6122
6123 usable_startpfn = memblock_region_memory_base_pfn(r);
6124
6125 if (usable_startpfn < 0x100000) {
6126 mem_below_4gb_not_mirrored = true;
6127 continue;
6128 }
6129
6130 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
6131 min(usable_startpfn, zone_movable_pfn[nid]) :
6132 usable_startpfn;
6133 }
6134
6135 if (mem_below_4gb_not_mirrored)
6136 pr_warn("This configuration results in unmirrored kernel memory.");
6137
6138 goto out2;
6139 }
6140
6141 /*
Tang Chenb2f3eeb2014-01-21 15:49:38 -08006142 * If movablecore=nn[KMG] was specified, calculate what size of
Mel Gorman7e63efe2007-07-17 04:03:15 -07006143 * kernelcore that corresponds so that memory usable for
6144 * any allocation type is evenly spread. If both kernelcore
6145 * and movablecore are specified, then the value of kernelcore
6146 * will be used for required_kernelcore if it's greater than
6147 * what movablecore would have allowed.
6148 */
6149 if (required_movablecore) {
Mel Gorman7e63efe2007-07-17 04:03:15 -07006150 unsigned long corepages;
6151
6152 /*
6153 * Round-up so that ZONE_MOVABLE is at least as large as what
6154 * was requested by the user
6155 */
6156 required_movablecore =
6157 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
Xishi Qiu9fd745d2015-11-05 18:48:11 -08006158 required_movablecore = min(totalpages, required_movablecore);
Mel Gorman7e63efe2007-07-17 04:03:15 -07006159 corepages = totalpages - required_movablecore;
6160
6161 required_kernelcore = max(required_kernelcore, corepages);
6162 }
6163
Xishi Qiubde304b2015-11-05 18:48:56 -08006164 /*
6165 * If kernelcore was not specified or kernelcore size is larger
6166 * than totalpages, there is no ZONE_MOVABLE.
6167 */
6168 if (!required_kernelcore || required_kernelcore >= totalpages)
Yinghai Lu66918dc2009-06-30 11:41:37 -07006169 goto out;
Mel Gorman2a1e2742007-07-17 04:03:12 -07006170
6171 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
Mel Gorman2a1e2742007-07-17 04:03:12 -07006172 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
6173
6174restart:
6175 /* Spread kernelcore memory as evenly as possible throughout nodes */
6176 kernelcore_node = required_kernelcore / usable_nodes;
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006177 for_each_node_state(nid, N_MEMORY) {
Tejun Heoc13291a2011-07-12 10:46:30 +02006178 unsigned long start_pfn, end_pfn;
6179
Mel Gorman2a1e2742007-07-17 04:03:12 -07006180 /*
6181 * Recalculate kernelcore_node if the division per node
6182 * now exceeds what is necessary to satisfy the requested
6183 * amount of memory for the kernel
6184 */
6185 if (required_kernelcore < kernelcore_node)
6186 kernelcore_node = required_kernelcore / usable_nodes;
6187
6188 /*
6189 * As the map is walked, we track how much memory is usable
6190 * by the kernel using kernelcore_remaining. When it is
6191 * 0, the rest of the node is usable by ZONE_MOVABLE
6192 */
6193 kernelcore_remaining = kernelcore_node;
6194
6195 /* Go through each range of PFNs within this node */
Tejun Heoc13291a2011-07-12 10:46:30 +02006196 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
Mel Gorman2a1e2742007-07-17 04:03:12 -07006197 unsigned long size_pages;
6198
Tejun Heoc13291a2011-07-12 10:46:30 +02006199 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
Mel Gorman2a1e2742007-07-17 04:03:12 -07006200 if (start_pfn >= end_pfn)
6201 continue;
6202
6203 /* Account for what is only usable for kernelcore */
6204 if (start_pfn < usable_startpfn) {
6205 unsigned long kernel_pages;
6206 kernel_pages = min(end_pfn, usable_startpfn)
6207 - start_pfn;
6208
6209 kernelcore_remaining -= min(kernel_pages,
6210 kernelcore_remaining);
6211 required_kernelcore -= min(kernel_pages,
6212 required_kernelcore);
6213
6214 /* Continue if range is now fully accounted */
6215 if (end_pfn <= usable_startpfn) {
6216
6217 /*
6218 * Push zone_movable_pfn to the end so
6219 * that if we have to rebalance
6220 * kernelcore across nodes, we will
6221 * not double account here
6222 */
6223 zone_movable_pfn[nid] = end_pfn;
6224 continue;
6225 }
6226 start_pfn = usable_startpfn;
6227 }
6228
6229 /*
6230 * The usable PFN range for ZONE_MOVABLE is from
6231 * start_pfn->end_pfn. Calculate size_pages as the
6232 * number of pages used as kernelcore
6233 */
6234 size_pages = end_pfn - start_pfn;
6235 if (size_pages > kernelcore_remaining)
6236 size_pages = kernelcore_remaining;
6237 zone_movable_pfn[nid] = start_pfn + size_pages;
6238
6239 /*
6240 * Some kernelcore has been met, update counts and
6241 * break if the kernelcore for this node has been
Pintu Kumarb8af2942013-09-11 14:20:34 -07006242 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07006243 */
6244 required_kernelcore -= min(required_kernelcore,
6245 size_pages);
6246 kernelcore_remaining -= size_pages;
6247 if (!kernelcore_remaining)
6248 break;
6249 }
6250 }
6251
6252 /*
6253 * If there is still required_kernelcore, we do another pass with one
6254 * less node in the count. This will push zone_movable_pfn[nid] further
6255 * along on the nodes that still have memory until kernelcore is
Pintu Kumarb8af2942013-09-11 14:20:34 -07006256 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07006257 */
6258 usable_nodes--;
6259 if (usable_nodes && required_kernelcore > usable_nodes)
6260 goto restart;
6261
Tang Chenb2f3eeb2014-01-21 15:49:38 -08006262out2:
Mel Gorman2a1e2742007-07-17 04:03:12 -07006263 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
6264 for (nid = 0; nid < MAX_NUMNODES; nid++)
6265 zone_movable_pfn[nid] =
6266 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
Yinghai Lu66918dc2009-06-30 11:41:37 -07006267
Yinghai Lu20e69262013-03-01 14:51:27 -08006268out:
Yinghai Lu66918dc2009-06-30 11:41:37 -07006269 /* restore the node_state */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006270 node_states[N_MEMORY] = saved_node_state;
Mel Gorman2a1e2742007-07-17 04:03:12 -07006271}
6272
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006273/* Any regular or high memory on that node ? */
6274static void check_for_memory(pg_data_t *pgdat, int nid)
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006275{
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006276 enum zone_type zone_type;
6277
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006278 if (N_MEMORY == N_NORMAL_MEMORY)
6279 return;
6280
6281 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006282 struct zone *zone = &pgdat->node_zones[zone_type];
Xishi Qiub38a8722013-11-12 15:07:20 -08006283 if (populated_zone(zone)) {
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006284 node_set_state(nid, N_HIGH_MEMORY);
6285 if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
6286 zone_type <= ZONE_NORMAL)
6287 node_set_state(nid, N_NORMAL_MEMORY);
Bob Liud0048b02012-01-12 17:19:07 -08006288 break;
6289 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006290 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006291}
6292
Mel Gormanc7132162006-09-27 01:49:43 -07006293/**
6294 * free_area_init_nodes - Initialise all pg_data_t and zone data
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006295 * @max_zone_pfn: an array of max PFNs for each zone
Mel Gormanc7132162006-09-27 01:49:43 -07006296 *
6297 * This will call free_area_init_node() for each active node in the system.
Zhang Zhen7d018172014-06-04 16:10:53 -07006298 * Using the page ranges provided by memblock_set_node(), the size of each
Mel Gormanc7132162006-09-27 01:49:43 -07006299 * zone in each node and their holes is calculated. If the maximum PFN
6300 * between two adjacent zones match, it is assumed that the zone is empty.
6301 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
6302 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
6303 * starts where the previous one ended. For example, ZONE_DMA32 starts
6304 * at arch_max_dma_pfn.
6305 */
6306void __init free_area_init_nodes(unsigned long *max_zone_pfn)
6307{
Tejun Heoc13291a2011-07-12 10:46:30 +02006308 unsigned long start_pfn, end_pfn;
6309 int i, nid;
Mel Gormana6af2bc2007-02-10 01:42:57 -08006310
Mel Gormanc7132162006-09-27 01:49:43 -07006311 /* Record where the zone boundaries are */
6312 memset(arch_zone_lowest_possible_pfn, 0,
6313 sizeof(arch_zone_lowest_possible_pfn));
6314 memset(arch_zone_highest_possible_pfn, 0,
6315 sizeof(arch_zone_highest_possible_pfn));
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07006316
6317 start_pfn = find_min_pfn_with_active_regions();
6318
6319 for (i = 0; i < MAX_NR_ZONES; i++) {
Mel Gorman2a1e2742007-07-17 04:03:12 -07006320 if (i == ZONE_MOVABLE)
6321 continue;
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07006322
6323 end_pfn = max(max_zone_pfn[i], start_pfn);
6324 arch_zone_lowest_possible_pfn[i] = start_pfn;
6325 arch_zone_highest_possible_pfn[i] = end_pfn;
6326
6327 start_pfn = end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006328 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07006329 arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
6330 arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
6331
6332 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
6333 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
Kautuk Consulb224ef82012-03-21 16:34:15 -07006334 find_zone_movable_pfns_for_nodes();
Mel Gormanc7132162006-09-27 01:49:43 -07006335
Mel Gormanc7132162006-09-27 01:49:43 -07006336 /* Print out the zone ranges */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006337 pr_info("Zone ranges:\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07006338 for (i = 0; i < MAX_NR_ZONES; i++) {
6339 if (i == ZONE_MOVABLE)
6340 continue;
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006341 pr_info(" %-8s ", zone_names[i]);
David Rientjes72f0ba02010-03-05 13:42:14 -08006342 if (arch_zone_lowest_possible_pfn[i] ==
6343 arch_zone_highest_possible_pfn[i])
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006344 pr_cont("empty\n");
David Rientjes72f0ba02010-03-05 13:42:14 -08006345 else
Juergen Gross8d29e182015-02-11 15:26:01 -08006346 pr_cont("[mem %#018Lx-%#018Lx]\n",
6347 (u64)arch_zone_lowest_possible_pfn[i]
6348 << PAGE_SHIFT,
6349 ((u64)arch_zone_highest_possible_pfn[i]
Bjorn Helgaasa62e2f42012-05-29 15:06:30 -07006350 << PAGE_SHIFT) - 1);
Mel Gorman2a1e2742007-07-17 04:03:12 -07006351 }
6352
6353 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006354 pr_info("Movable zone start for each node\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07006355 for (i = 0; i < MAX_NUMNODES; i++) {
6356 if (zone_movable_pfn[i])
Juergen Gross8d29e182015-02-11 15:26:01 -08006357 pr_info(" Node %d: %#018Lx\n", i,
6358 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
Mel Gorman2a1e2742007-07-17 04:03:12 -07006359 }
Mel Gormanc7132162006-09-27 01:49:43 -07006360
Wanpeng Lif2d52fe2012-10-08 16:32:24 -07006361 /* Print out the early node map */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006362 pr_info("Early memory node ranges\n");
Tejun Heoc13291a2011-07-12 10:46:30 +02006363 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
Juergen Gross8d29e182015-02-11 15:26:01 -08006364 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
6365 (u64)start_pfn << PAGE_SHIFT,
6366 ((u64)end_pfn << PAGE_SHIFT) - 1);
Mel Gormanc7132162006-09-27 01:49:43 -07006367
6368 /* Initialise every node */
Mel Gorman708614e2008-07-23 21:26:51 -07006369 mminit_verify_pageflags_layout();
Christoph Lameter8ef82862007-02-20 13:57:52 -08006370 setup_nr_node_ids();
Mel Gormanc7132162006-09-27 01:49:43 -07006371 for_each_online_node(nid) {
6372 pg_data_t *pgdat = NODE_DATA(nid);
Johannes Weiner9109fb72008-07-23 21:27:20 -07006373 free_area_init_node(nid, NULL,
Mel Gormanc7132162006-09-27 01:49:43 -07006374 find_min_pfn_for_node(nid), NULL);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07006375
6376 /* Any memory on that node */
6377 if (pgdat->node_present_pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006378 node_set_state(nid, N_MEMORY);
6379 check_for_memory(pgdat, nid);
Mel Gormanc7132162006-09-27 01:49:43 -07006380 }
6381}
Mel Gorman2a1e2742007-07-17 04:03:12 -07006382
Mel Gorman7e63efe2007-07-17 04:03:15 -07006383static int __init cmdline_parse_core(char *p, unsigned long *core)
Mel Gorman2a1e2742007-07-17 04:03:12 -07006384{
6385 unsigned long long coremem;
6386 if (!p)
6387 return -EINVAL;
6388
6389 coremem = memparse(p, &p);
Mel Gorman7e63efe2007-07-17 04:03:15 -07006390 *core = coremem >> PAGE_SHIFT;
Mel Gorman2a1e2742007-07-17 04:03:12 -07006391
Mel Gorman7e63efe2007-07-17 04:03:15 -07006392 /* Paranoid check that UL is enough for the coremem value */
Mel Gorman2a1e2742007-07-17 04:03:12 -07006393 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
6394
6395 return 0;
6396}
Mel Gormaned7ed362007-07-17 04:03:14 -07006397
Mel Gorman7e63efe2007-07-17 04:03:15 -07006398/*
6399 * kernelcore=size sets the amount of memory for use for allocations that
6400 * cannot be reclaimed or migrated.
6401 */
6402static int __init cmdline_parse_kernelcore(char *p)
6403{
Taku Izumi342332e2016-03-15 14:55:22 -07006404 /* parse kernelcore=mirror */
6405 if (parse_option_str(p, "mirror")) {
6406 mirrored_kernelcore = true;
6407 return 0;
6408 }
6409
Mel Gorman7e63efe2007-07-17 04:03:15 -07006410 return cmdline_parse_core(p, &required_kernelcore);
6411}
6412
6413/*
6414 * movablecore=size sets the amount of memory for use for allocations that
6415 * can be reclaimed or migrated.
6416 */
6417static int __init cmdline_parse_movablecore(char *p)
6418{
6419 return cmdline_parse_core(p, &required_movablecore);
6420}
6421
Mel Gormaned7ed362007-07-17 04:03:14 -07006422early_param("kernelcore", cmdline_parse_kernelcore);
Mel Gorman7e63efe2007-07-17 04:03:15 -07006423early_param("movablecore", cmdline_parse_movablecore);
Mel Gormaned7ed362007-07-17 04:03:14 -07006424
Tejun Heo0ee332c2011-12-08 10:22:09 -08006425#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -07006426
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07006427void adjust_managed_page_count(struct page *page, long count)
6428{
6429 spin_lock(&managed_page_count_lock);
6430 page_zone(page)->managed_pages += count;
6431 totalram_pages += count;
Jiang Liu3dcc0572013-07-03 15:03:21 -07006432#ifdef CONFIG_HIGHMEM
6433 if (PageHighMem(page))
6434 totalhigh_pages += count;
6435#endif
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07006436 spin_unlock(&managed_page_count_lock);
6437}
Jiang Liu3dcc0572013-07-03 15:03:21 -07006438EXPORT_SYMBOL(adjust_managed_page_count);
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07006439
Jiang Liu11199692013-07-03 15:02:48 -07006440unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
Jiang Liu69afade2013-04-29 15:06:21 -07006441{
Jiang Liu11199692013-07-03 15:02:48 -07006442 void *pos;
6443 unsigned long pages = 0;
Jiang Liu69afade2013-04-29 15:06:21 -07006444
Jiang Liu11199692013-07-03 15:02:48 -07006445 start = (void *)PAGE_ALIGN((unsigned long)start);
6446 end = (void *)((unsigned long)end & PAGE_MASK);
6447 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
Jiang Liudbe67df2013-07-03 15:02:51 -07006448 if ((unsigned int)poison <= 0xFF)
Jiang Liu11199692013-07-03 15:02:48 -07006449 memset(pos, poison, PAGE_SIZE);
6450 free_reserved_page(virt_to_page(pos));
Jiang Liu69afade2013-04-29 15:06:21 -07006451 }
6452
6453 if (pages && s)
Josh Poimboeuf7b95b742016-10-25 09:51:14 -05006454 pr_info("Freeing %s memory: %ldK\n",
6455 s, pages << (PAGE_SHIFT - 10));
Jiang Liu69afade2013-04-29 15:06:21 -07006456
6457 return pages;
6458}
Jiang Liu11199692013-07-03 15:02:48 -07006459EXPORT_SYMBOL(free_reserved_area);
Jiang Liu69afade2013-04-29 15:06:21 -07006460
Jiang Liucfa11e02013-04-29 15:07:00 -07006461#ifdef CONFIG_HIGHMEM
6462void free_highmem_page(struct page *page)
6463{
6464 __free_reserved_page(page);
6465 totalram_pages++;
Jiang Liu7b4b2a02013-07-03 15:03:11 -07006466 page_zone(page)->managed_pages++;
Jiang Liucfa11e02013-04-29 15:07:00 -07006467 totalhigh_pages++;
6468}
6469#endif
6470
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07006471
6472void __init mem_init_print_info(const char *str)
6473{
6474 unsigned long physpages, codesize, datasize, rosize, bss_size;
6475 unsigned long init_code_size, init_data_size;
6476
6477 physpages = get_num_physpages();
6478 codesize = _etext - _stext;
6479 datasize = _edata - _sdata;
6480 rosize = __end_rodata - __start_rodata;
6481 bss_size = __bss_stop - __bss_start;
6482 init_data_size = __init_end - __init_begin;
6483 init_code_size = _einittext - _sinittext;
6484
6485 /*
6486 * Detect special cases and adjust section sizes accordingly:
6487 * 1) .init.* may be embedded into .data sections
6488 * 2) .init.text.* may be out of [__init_begin, __init_end],
6489 * please refer to arch/tile/kernel/vmlinux.lds.S.
6490 * 3) .rodata.* may be embedded into .text or .data sections.
6491 */
6492#define adj_init_size(start, end, size, pos, adj) \
Pintu Kumarb8af2942013-09-11 14:20:34 -07006493 do { \
6494 if (start <= pos && pos < end && size > adj) \
6495 size -= adj; \
6496 } while (0)
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07006497
6498 adj_init_size(__init_begin, __init_end, init_data_size,
6499 _sinittext, init_code_size);
6500 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
6501 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
6502 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
6503 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
6504
6505#undef adj_init_size
6506
Joe Perches756a025f02016-03-17 14:19:47 -07006507 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07006508#ifdef CONFIG_HIGHMEM
Joe Perches756a025f02016-03-17 14:19:47 -07006509 ", %luK highmem"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07006510#endif
Joe Perches756a025f02016-03-17 14:19:47 -07006511 "%s%s)\n",
6512 nr_free_pages() << (PAGE_SHIFT - 10),
6513 physpages << (PAGE_SHIFT - 10),
6514 codesize >> 10, datasize >> 10, rosize >> 10,
6515 (init_data_size + init_code_size) >> 10, bss_size >> 10,
6516 (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
6517 totalcma_pages << (PAGE_SHIFT - 10),
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07006518#ifdef CONFIG_HIGHMEM
Joe Perches756a025f02016-03-17 14:19:47 -07006519 totalhigh_pages << (PAGE_SHIFT - 10),
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07006520#endif
Joe Perches756a025f02016-03-17 14:19:47 -07006521 str ? ", " : "", str ? str : "");
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07006522}
6523
Mel Gorman0e0b8642006-09-27 01:49:56 -07006524/**
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006525 * set_dma_reserve - set the specified number of pages reserved in the first zone
6526 * @new_dma_reserve: The number of pages to mark reserved
Mel Gorman0e0b8642006-09-27 01:49:56 -07006527 *
Yaowei Bai013110a2015-09-08 15:04:10 -07006528 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
Mel Gorman0e0b8642006-09-27 01:49:56 -07006529 * In the DMA zone, a significant percentage may be consumed by kernel image
6530 * and other unfreeable allocations which can skew the watermarks badly. This
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006531 * function may optionally be used to account for unfreeable pages in the
6532 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
6533 * smaller per-cpu batchsize.
Mel Gorman0e0b8642006-09-27 01:49:56 -07006534 */
6535void __init set_dma_reserve(unsigned long new_dma_reserve)
6536{
6537 dma_reserve = new_dma_reserve;
6538}
6539
Linus Torvalds1da177e2005-04-16 15:20:36 -07006540void __init free_area_init(unsigned long *zones_size)
6541{
Johannes Weiner9109fb72008-07-23 21:27:20 -07006542 free_area_init_node(0, zones_size,
Linus Torvalds1da177e2005-04-16 15:20:36 -07006543 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
6544}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006545
Linus Torvalds1da177e2005-04-16 15:20:36 -07006546static int page_alloc_cpu_notify(struct notifier_block *self,
6547 unsigned long action, void *hcpu)
6548{
6549 int cpu = (unsigned long)hcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006550
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006551 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
Konstantin Khlebnikovf0cb3c72012-03-21 16:34:06 -07006552 lru_add_drain_cpu(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08006553 drain_pages(cpu);
6554
6555 /*
6556 * Spill the event counters of the dead processor
6557 * into the current processors event counters.
6558 * This artificially elevates the count of the current
6559 * processor.
6560 */
Christoph Lameterf8891e52006-06-30 01:55:45 -07006561 vm_events_fold_cpu(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08006562
6563 /*
6564 * Zero the differential counters of the dead processor
6565 * so that the vm statistics are consistent.
6566 *
6567 * This is only okay since the processor is dead and cannot
6568 * race with what we are doing.
6569 */
Christoph Lameter2bb921e2013-09-11 14:21:30 -07006570 cpu_vm_stats_fold(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006571 }
6572 return NOTIFY_OK;
6573}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006574
6575void __init page_alloc_init(void)
6576{
6577 hotcpu_notifier(page_alloc_cpu_notify, 0);
6578}
6579
6580/*
Yaowei Bai34b10062015-09-08 15:04:13 -07006581 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006582 * or min_free_kbytes changes.
6583 */
6584static void calculate_totalreserve_pages(void)
6585{
6586 struct pglist_data *pgdat;
6587 unsigned long reserve_pages = 0;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07006588 enum zone_type i, j;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006589
6590 for_each_online_pgdat(pgdat) {
Mel Gorman281e3722016-07-28 15:46:11 -07006591
6592 pgdat->totalreserve_pages = 0;
6593
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006594 for (i = 0; i < MAX_NR_ZONES; i++) {
6595 struct zone *zone = pgdat->node_zones + i;
Mel Gorman3484b2d2014-08-06 16:07:14 -07006596 long max = 0;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006597
6598 /* Find valid and maximum lowmem_reserve in the zone */
6599 for (j = i; j < MAX_NR_ZONES; j++) {
6600 if (zone->lowmem_reserve[j] > max)
6601 max = zone->lowmem_reserve[j];
6602 }
6603
Mel Gorman41858962009-06-16 15:32:12 -07006604 /* we treat the high watermark as reserved pages. */
6605 max += high_wmark_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006606
Jiang Liub40da042013-02-22 16:33:52 -08006607 if (max > zone->managed_pages)
6608 max = zone->managed_pages;
Johannes Weinera8d01432016-01-14 15:20:15 -08006609
Mel Gorman281e3722016-07-28 15:46:11 -07006610 pgdat->totalreserve_pages += max;
Johannes Weinera8d01432016-01-14 15:20:15 -08006611
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006612 reserve_pages += max;
6613 }
6614 }
6615 totalreserve_pages = reserve_pages;
6616}
6617
6618/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07006619 * setup_per_zone_lowmem_reserve - called whenever
Yaowei Bai34b10062015-09-08 15:04:13 -07006620 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
Linus Torvalds1da177e2005-04-16 15:20:36 -07006621 * has a correct pages reserved value, so an adequate number of
6622 * pages are left in the zone after a successful __alloc_pages().
6623 */
6624static void setup_per_zone_lowmem_reserve(void)
6625{
6626 struct pglist_data *pgdat;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07006627 enum zone_type j, idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006628
KAMEZAWA Hiroyukiec936fc2006-03-27 01:15:59 -08006629 for_each_online_pgdat(pgdat) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006630 for (j = 0; j < MAX_NR_ZONES; j++) {
6631 struct zone *zone = pgdat->node_zones + j;
Jiang Liub40da042013-02-22 16:33:52 -08006632 unsigned long managed_pages = zone->managed_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006633
6634 zone->lowmem_reserve[j] = 0;
6635
Christoph Lameter2f6726e2006-09-25 23:31:18 -07006636 idx = j;
6637 while (idx) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006638 struct zone *lower_zone;
6639
Christoph Lameter2f6726e2006-09-25 23:31:18 -07006640 idx--;
6641
Linus Torvalds1da177e2005-04-16 15:20:36 -07006642 if (sysctl_lowmem_reserve_ratio[idx] < 1)
6643 sysctl_lowmem_reserve_ratio[idx] = 1;
6644
6645 lower_zone = pgdat->node_zones + idx;
Jiang Liub40da042013-02-22 16:33:52 -08006646 lower_zone->lowmem_reserve[j] = managed_pages /
Linus Torvalds1da177e2005-04-16 15:20:36 -07006647 sysctl_lowmem_reserve_ratio[idx];
Jiang Liub40da042013-02-22 16:33:52 -08006648 managed_pages += lower_zone->managed_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006649 }
6650 }
6651 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006652
6653 /* update totalreserve_pages */
6654 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006655}
6656
Mel Gormancfd3da12011-04-25 21:36:42 +00006657static void __setup_per_zone_wmarks(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006658{
6659 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
6660 unsigned long lowmem_pages = 0;
6661 struct zone *zone;
6662 unsigned long flags;
6663
6664 /* Calculate total number of !ZONE_HIGHMEM pages */
6665 for_each_zone(zone) {
6666 if (!is_highmem(zone))
Jiang Liub40da042013-02-22 16:33:52 -08006667 lowmem_pages += zone->managed_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006668 }
6669
6670 for_each_zone(zone) {
Andrew Mortonac924c62006-05-15 09:43:59 -07006671 u64 tmp;
6672
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07006673 spin_lock_irqsave(&zone->lock, flags);
Jiang Liub40da042013-02-22 16:33:52 -08006674 tmp = (u64)pages_min * zone->managed_pages;
Andrew Mortonac924c62006-05-15 09:43:59 -07006675 do_div(tmp, lowmem_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006676 if (is_highmem(zone)) {
6677 /*
Nick Piggin669ed172005-11-13 16:06:45 -08006678 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
6679 * need highmem pages, so cap pages_min to a small
6680 * value here.
6681 *
Mel Gorman41858962009-06-16 15:32:12 -07006682 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
Yaowei Bai42ff2702015-04-14 15:47:14 -07006683 * deltas control asynch page reclaim, and so should
Nick Piggin669ed172005-11-13 16:06:45 -08006684 * not be capped for highmem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006685 */
Andrew Morton90ae8d62013-02-22 16:32:22 -08006686 unsigned long min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006687
Jiang Liub40da042013-02-22 16:33:52 -08006688 min_pages = zone->managed_pages / 1024;
Andrew Morton90ae8d62013-02-22 16:32:22 -08006689 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
Mel Gorman41858962009-06-16 15:32:12 -07006690 zone->watermark[WMARK_MIN] = min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006691 } else {
Nick Piggin669ed172005-11-13 16:06:45 -08006692 /*
6693 * If it's a lowmem zone, reserve a number of pages
Linus Torvalds1da177e2005-04-16 15:20:36 -07006694 * proportionate to the zone's size.
6695 */
Mel Gorman41858962009-06-16 15:32:12 -07006696 zone->watermark[WMARK_MIN] = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006697 }
6698
Johannes Weiner795ae7a2016-03-17 14:19:14 -07006699 /*
6700 * Set the kswapd watermarks distance according to the
6701 * scale factor in proportion to available memory, but
6702 * ensure a minimum size on small systems.
6703 */
6704 tmp = max_t(u64, tmp >> 2,
6705 mult_frac(zone->managed_pages,
6706 watermark_scale_factor, 10000));
6707
6708 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
6709 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
Marek Szyprowski49f223a2012-01-25 12:49:24 +01006710
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07006711 spin_unlock_irqrestore(&zone->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006712 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07006713
6714 /* update totalreserve_pages */
6715 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006716}
6717
Mel Gormancfd3da12011-04-25 21:36:42 +00006718/**
6719 * setup_per_zone_wmarks - called when min_free_kbytes changes
6720 * or when memory is hot-{added|removed}
6721 *
6722 * Ensures that the watermark[min,low,high] values for each zone are set
6723 * correctly with respect to min_free_kbytes.
6724 */
6725void setup_per_zone_wmarks(void)
6726{
6727 mutex_lock(&zonelists_mutex);
6728 __setup_per_zone_wmarks();
6729 mutex_unlock(&zonelists_mutex);
6730}
6731
Randy Dunlap55a44622009-09-21 17:01:20 -07006732/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07006733 * Initialise min_free_kbytes.
6734 *
6735 * For small machines we want it small (128k min). For large machines
6736 * we want it large (64MB max). But it is not linear, because network
6737 * bandwidth does not increase linearly with machine size. We use
6738 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07006739 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
Linus Torvalds1da177e2005-04-16 15:20:36 -07006740 * min_free_kbytes = sqrt(lowmem_kbytes * 16)
6741 *
6742 * which yields
6743 *
6744 * 16MB: 512k
6745 * 32MB: 724k
6746 * 64MB: 1024k
6747 * 128MB: 1448k
6748 * 256MB: 2048k
6749 * 512MB: 2896k
6750 * 1024MB: 4096k
6751 * 2048MB: 5792k
6752 * 4096MB: 8192k
6753 * 8192MB: 11584k
6754 * 16384MB: 16384k
6755 */
KOSAKI Motohiro1b79acc2011-05-24 17:11:32 -07006756int __meminit init_per_zone_wmark_min(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006757{
6758 unsigned long lowmem_kbytes;
Michal Hocko5f127332013-07-08 16:00:40 -07006759 int new_min_free_kbytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006760
6761 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
Michal Hocko5f127332013-07-08 16:00:40 -07006762 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006763
Michal Hocko5f127332013-07-08 16:00:40 -07006764 if (new_min_free_kbytes > user_min_free_kbytes) {
6765 min_free_kbytes = new_min_free_kbytes;
6766 if (min_free_kbytes < 128)
6767 min_free_kbytes = 128;
6768 if (min_free_kbytes > 65536)
6769 min_free_kbytes = 65536;
6770 } else {
6771 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
6772 new_min_free_kbytes, user_min_free_kbytes);
6773 }
Minchan Kimbc75d332009-06-16 15:32:48 -07006774 setup_per_zone_wmarks();
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -07006775 refresh_zone_stat_thresholds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006776 setup_per_zone_lowmem_reserve();
Joonsoo Kim6423aa82016-08-10 16:27:49 -07006777
6778#ifdef CONFIG_NUMA
6779 setup_min_unmapped_ratio();
6780 setup_min_slab_ratio();
6781#endif
6782
Linus Torvalds1da177e2005-04-16 15:20:36 -07006783 return 0;
6784}
Jason Baronbc22af72016-05-05 16:22:12 -07006785core_initcall(init_per_zone_wmark_min)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006786
6787/*
Pintu Kumarb8af2942013-09-11 14:20:34 -07006788 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
Linus Torvalds1da177e2005-04-16 15:20:36 -07006789 * that we can call two helper functions whenever min_free_kbytes
6790 * changes.
6791 */
Joe Perchescccad5b2014-06-06 14:38:09 -07006792int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07006793 void __user *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006794{
Han Pingtianda8c7572014-01-23 15:53:17 -08006795 int rc;
6796
6797 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6798 if (rc)
6799 return rc;
6800
Michal Hocko5f127332013-07-08 16:00:40 -07006801 if (write) {
6802 user_min_free_kbytes = min_free_kbytes;
Minchan Kimbc75d332009-06-16 15:32:48 -07006803 setup_per_zone_wmarks();
Michal Hocko5f127332013-07-08 16:00:40 -07006804 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006805 return 0;
6806}
6807
Johannes Weiner795ae7a2016-03-17 14:19:14 -07006808int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
6809 void __user *buffer, size_t *length, loff_t *ppos)
6810{
6811 int rc;
6812
6813 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6814 if (rc)
6815 return rc;
6816
6817 if (write)
6818 setup_per_zone_wmarks();
6819
6820 return 0;
6821}
6822
Christoph Lameter96146342006-07-03 00:24:13 -07006823#ifdef CONFIG_NUMA
Joonsoo Kim6423aa82016-08-10 16:27:49 -07006824static void setup_min_unmapped_ratio(void)
Christoph Lameter96146342006-07-03 00:24:13 -07006825{
Joonsoo Kim6423aa82016-08-10 16:27:49 -07006826 pg_data_t *pgdat;
Christoph Lameter96146342006-07-03 00:24:13 -07006827 struct zone *zone;
Christoph Lameter96146342006-07-03 00:24:13 -07006828
Mel Gormana5f5f912016-07-28 15:46:32 -07006829 for_each_online_pgdat(pgdat)
Joonsoo Kim81cbcbc2016-08-10 16:27:46 -07006830 pgdat->min_unmapped_pages = 0;
Mel Gormana5f5f912016-07-28 15:46:32 -07006831
Christoph Lameter96146342006-07-03 00:24:13 -07006832 for_each_zone(zone)
Mel Gormana5f5f912016-07-28 15:46:32 -07006833 zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages *
Christoph Lameter96146342006-07-03 00:24:13 -07006834 sysctl_min_unmapped_ratio) / 100;
Christoph Lameter96146342006-07-03 00:24:13 -07006835}
Christoph Lameter0ff38492006-09-25 23:31:52 -07006836
Joonsoo Kim6423aa82016-08-10 16:27:49 -07006837
6838int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07006839 void __user *buffer, size_t *length, loff_t *ppos)
Christoph Lameter0ff38492006-09-25 23:31:52 -07006840{
Christoph Lameter0ff38492006-09-25 23:31:52 -07006841 int rc;
6842
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07006843 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
Christoph Lameter0ff38492006-09-25 23:31:52 -07006844 if (rc)
6845 return rc;
6846
Joonsoo Kim6423aa82016-08-10 16:27:49 -07006847 setup_min_unmapped_ratio();
6848
6849 return 0;
6850}
6851
6852static void setup_min_slab_ratio(void)
6853{
6854 pg_data_t *pgdat;
6855 struct zone *zone;
6856
Mel Gormana5f5f912016-07-28 15:46:32 -07006857 for_each_online_pgdat(pgdat)
6858 pgdat->min_slab_pages = 0;
6859
Christoph Lameter0ff38492006-09-25 23:31:52 -07006860 for_each_zone(zone)
Mel Gormana5f5f912016-07-28 15:46:32 -07006861 zone->zone_pgdat->min_slab_pages += (zone->managed_pages *
Christoph Lameter0ff38492006-09-25 23:31:52 -07006862 sysctl_min_slab_ratio) / 100;
Joonsoo Kim6423aa82016-08-10 16:27:49 -07006863}
6864
6865int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
6866 void __user *buffer, size_t *length, loff_t *ppos)
6867{
6868 int rc;
6869
6870 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6871 if (rc)
6872 return rc;
6873
6874 setup_min_slab_ratio();
6875
Christoph Lameter0ff38492006-09-25 23:31:52 -07006876 return 0;
6877}
Christoph Lameter96146342006-07-03 00:24:13 -07006878#endif
6879
Linus Torvalds1da177e2005-04-16 15:20:36 -07006880/*
6881 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
6882 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
6883 * whenever sysctl_lowmem_reserve_ratio changes.
6884 *
6885 * The reserve ratio obviously has absolutely no relation with the
Mel Gorman41858962009-06-16 15:32:12 -07006886 * minimum watermarks. The lowmem reserve ratio can only make sense
Linus Torvalds1da177e2005-04-16 15:20:36 -07006887 * if in function of the boot time zone sizes.
6888 */
Joe Perchescccad5b2014-06-06 14:38:09 -07006889int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07006890 void __user *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006891{
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07006892 proc_dointvec_minmax(table, write, buffer, length, ppos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006893 setup_per_zone_lowmem_reserve();
6894 return 0;
6895}
6896
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006897/*
6898 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
Pintu Kumarb8af2942013-09-11 14:20:34 -07006899 * cpu. It is the fraction of total pages in each zone that a hot per cpu
6900 * pagelist can have before it gets flushed back to buddy allocator.
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006901 */
Joe Perchescccad5b2014-06-06 14:38:09 -07006902int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07006903 void __user *buffer, size_t *length, loff_t *ppos)
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006904{
6905 struct zone *zone;
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006906 int old_percpu_pagelist_fraction;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006907 int ret;
6908
Cody P Schaferc8e251f2013-07-03 15:01:29 -07006909 mutex_lock(&pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006910 old_percpu_pagelist_fraction = percpu_pagelist_fraction;
6911
6912 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
6913 if (!write || ret < 0)
6914 goto out;
6915
6916 /* Sanity checking to avoid pcp imbalance */
6917 if (percpu_pagelist_fraction &&
6918 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
6919 percpu_pagelist_fraction = old_percpu_pagelist_fraction;
6920 ret = -EINVAL;
6921 goto out;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006922 }
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006923
6924 /* No change? */
6925 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
6926 goto out;
6927
6928 for_each_populated_zone(zone) {
6929 unsigned int cpu;
6930
6931 for_each_possible_cpu(cpu)
6932 pageset_set_high_and_batch(zone,
6933 per_cpu_ptr(zone->pageset, cpu));
6934 }
6935out:
Cody P Schaferc8e251f2013-07-03 15:01:29 -07006936 mutex_unlock(&pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006937 return ret;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006938}
6939
Rasmus Villemoesa9919c72015-06-24 16:56:28 -07006940#ifdef CONFIG_NUMA
David S. Millerf034b5d2006-08-24 03:08:07 -07006941int hashdist = HASHDIST_DEFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006942
Linus Torvalds1da177e2005-04-16 15:20:36 -07006943static int __init set_hashdist(char *str)
6944{
6945 if (!str)
6946 return 0;
6947 hashdist = simple_strtoul(str, &str, 0);
6948 return 1;
6949}
6950__setup("hashdist=", set_hashdist);
6951#endif
6952
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07006953#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
6954/*
6955 * Returns the number of pages that arch has reserved but
6956 * is not known to alloc_large_system_hash().
6957 */
6958static unsigned long __init arch_reserved_kernel_pages(void)
6959{
6960 return 0;
6961}
6962#endif
6963
Linus Torvalds1da177e2005-04-16 15:20:36 -07006964/*
6965 * allocate a large system hash table from bootmem
6966 * - it is assumed that the hash table must contain an exact power-of-2
6967 * quantity of entries
6968 * - limit is the number of hash buckets, not the total allocation size
6969 */
6970void *__init alloc_large_system_hash(const char *tablename,
6971 unsigned long bucketsize,
6972 unsigned long numentries,
6973 int scale,
6974 int flags,
6975 unsigned int *_hash_shift,
6976 unsigned int *_hash_mask,
Tim Bird31fe62b2012-05-23 13:33:35 +00006977 unsigned long low_limit,
6978 unsigned long high_limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006979{
Tim Bird31fe62b2012-05-23 13:33:35 +00006980 unsigned long long max = high_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006981 unsigned long log2qty, size;
6982 void *table = NULL;
6983
6984 /* allow the kernel cmdline to have a say */
6985 if (!numentries) {
6986 /* round applicable memory size up to nearest megabyte */
Andrew Morton04903662006-12-06 20:37:33 -08006987 numentries = nr_kernel_pages;
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07006988 numentries -= arch_reserved_kernel_pages();
Jerry Zhoua7e83312013-09-11 14:20:26 -07006989
6990 /* It isn't necessary when PAGE_SIZE >= 1MB */
6991 if (PAGE_SHIFT < 20)
6992 numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006993
6994 /* limit to 1 bucket per 2^scale bytes of low memory */
6995 if (scale > PAGE_SHIFT)
6996 numentries >>= (scale - PAGE_SHIFT);
6997 else
6998 numentries <<= (PAGE_SHIFT - scale);
Paul Mundt9ab37b82007-01-05 16:36:30 -08006999
7000 /* Make sure we've got at least a 0-order allocation.. */
Jan Beulich2c85f512009-09-21 17:03:07 -07007001 if (unlikely(flags & HASH_SMALL)) {
7002 /* Makes no sense without HASH_EARLY */
7003 WARN_ON(!(flags & HASH_EARLY));
7004 if (!(numentries >> *_hash_shift)) {
7005 numentries = 1UL << *_hash_shift;
7006 BUG_ON(!numentries);
7007 }
7008 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
Paul Mundt9ab37b82007-01-05 16:36:30 -08007009 numentries = PAGE_SIZE / bucketsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007010 }
John Hawkes6e692ed2006-03-25 03:08:02 -08007011 numentries = roundup_pow_of_two(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007012
7013 /* limit allocation size to 1/16 total memory by default */
7014 if (max == 0) {
7015 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
7016 do_div(max, bucketsize);
7017 }
Dimitri Sivanich074b8512012-02-08 12:39:07 -08007018 max = min(max, 0x80000000ULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007019
Tim Bird31fe62b2012-05-23 13:33:35 +00007020 if (numentries < low_limit)
7021 numentries = low_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007022 if (numentries > max)
7023 numentries = max;
7024
David Howellsf0d1b0b2006-12-08 02:37:49 -08007025 log2qty = ilog2(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007026
7027 do {
7028 size = bucketsize << log2qty;
7029 if (flags & HASH_EARLY)
Santosh Shilimkar67828322014-01-21 15:50:25 -08007030 table = memblock_virt_alloc_nopanic(size, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007031 else if (hashdist)
7032 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
7033 else {
Eric Dumazet1037b832007-07-15 23:38:05 -07007034 /*
7035 * If bucketsize is not a power-of-two, we may free
Mel Gormana1dd2682009-06-16 15:32:19 -07007036 * some pages at the end of hash table which
7037 * alloc_pages_exact() automatically does
Eric Dumazet1037b832007-07-15 23:38:05 -07007038 */
Catalin Marinas264ef8a2009-07-07 10:33:01 +01007039 if (get_order(size) < MAX_ORDER) {
Mel Gormana1dd2682009-06-16 15:32:19 -07007040 table = alloc_pages_exact(size, GFP_ATOMIC);
Catalin Marinas264ef8a2009-07-07 10:33:01 +01007041 kmemleak_alloc(table, size, 1, GFP_ATOMIC);
7042 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007043 }
7044 } while (!table && size > PAGE_SIZE && --log2qty);
7045
7046 if (!table)
7047 panic("Failed to allocate %s hash table\n", tablename);
7048
Joe Perches11705322016-03-17 14:19:50 -07007049 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
7050 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007051
7052 if (_hash_shift)
7053 *_hash_shift = log2qty;
7054 if (_hash_mask)
7055 *_hash_mask = (1 << log2qty) - 1;
7056
7057 return table;
7058}
KAMEZAWA Hiroyukia117e662006-03-27 01:15:25 -08007059
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07007060/*
Minchan Kim80934512012-07-31 16:43:01 -07007061 * This function checks whether pageblock includes unmovable pages or not.
7062 * If @count is not zero, it is okay to include less @count unmovable pages
7063 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07007064 * PageLRU check without isolation or lru_lock could race so that
Minchan Kim80934512012-07-31 16:43:01 -07007065 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
7066 * expect this function should be exact.
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07007067 */
Wen Congyangb023f462012-12-11 16:00:45 -08007068bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
7069 bool skip_hwpoisoned_pages)
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007070{
7071 unsigned long pfn, iter, found;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01007072 int mt;
7073
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007074 /*
7075 * For avoiding noise data, lru_add_drain_all() should be called
Minchan Kim80934512012-07-31 16:43:01 -07007076 * If ZONE_MOVABLE, the zone never contains unmovable pages
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007077 */
7078 if (zone_idx(zone) == ZONE_MOVABLE)
Minchan Kim80934512012-07-31 16:43:01 -07007079 return false;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01007080 mt = get_pageblock_migratetype(page);
7081 if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
Minchan Kim80934512012-07-31 16:43:01 -07007082 return false;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007083
7084 pfn = page_to_pfn(page);
7085 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
7086 unsigned long check = pfn + iter;
7087
Namhyung Kim29723fc2011-02-25 14:44:25 -08007088 if (!pfn_valid_within(check))
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007089 continue;
Namhyung Kim29723fc2011-02-25 14:44:25 -08007090
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007091 page = pfn_to_page(check);
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07007092
7093 /*
7094 * Hugepages are not in LRU lists, but they're movable.
7095 * We need not scan over tail pages bacause we don't
7096 * handle each tail page individually in migration.
7097 */
7098 if (PageHuge(page)) {
7099 iter = round_up(iter + 1, 1<<compound_order(page)) - 1;
7100 continue;
7101 }
7102
Minchan Kim97d255c2012-07-31 16:42:59 -07007103 /*
7104 * We can't use page_count without pin a page
7105 * because another CPU can free compound page.
7106 * This check already skips compound tails of THP
Joonsoo Kim0139aa72016-05-19 17:10:49 -07007107 * because their page->_refcount is zero at all time.
Minchan Kim97d255c2012-07-31 16:42:59 -07007108 */
Joonsoo Kimfe896d12016-03-17 14:19:26 -07007109 if (!page_ref_count(page)) {
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007110 if (PageBuddy(page))
7111 iter += (1 << page_order(page)) - 1;
7112 continue;
7113 }
Minchan Kim97d255c2012-07-31 16:42:59 -07007114
Wen Congyangb023f462012-12-11 16:00:45 -08007115 /*
7116 * The HWPoisoned page may be not in buddy system, and
7117 * page_count() is not 0.
7118 */
7119 if (skip_hwpoisoned_pages && PageHWPoison(page))
7120 continue;
7121
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007122 if (!PageLRU(page))
7123 found++;
7124 /*
Johannes Weiner6b4f7792014-12-12 16:56:13 -08007125 * If there are RECLAIMABLE pages, we need to check
7126 * it. But now, memory offline itself doesn't call
7127 * shrink_node_slabs() and it still to be fixed.
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007128 */
7129 /*
7130 * If the page is not RAM, page_count()should be 0.
7131 * we don't need more check. This is an _used_ not-movable page.
7132 *
7133 * The problematic thing here is PG_reserved pages. PG_reserved
7134 * is set to both of a memory hole page and a _used_ kernel
7135 * page at boot.
7136 */
7137 if (found > count)
Minchan Kim80934512012-07-31 16:43:01 -07007138 return true;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007139 }
Minchan Kim80934512012-07-31 16:43:01 -07007140 return false;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07007141}
7142
7143bool is_pageblock_removable_nolock(struct page *page)
7144{
Michal Hocko656a0702012-01-20 14:33:58 -08007145 struct zone *zone;
7146 unsigned long pfn;
Michal Hocko687875f2012-01-20 14:33:55 -08007147
7148 /*
7149 * We have to be careful here because we are iterating over memory
7150 * sections which are not zone aware so we might end up outside of
7151 * the zone but still within the section.
Michal Hocko656a0702012-01-20 14:33:58 -08007152 * We have to take care about the node as well. If the node is offline
7153 * its NODE_DATA will be NULL - see page_zone.
Michal Hocko687875f2012-01-20 14:33:55 -08007154 */
Michal Hocko656a0702012-01-20 14:33:58 -08007155 if (!node_online(page_to_nid(page)))
7156 return false;
7157
7158 zone = page_zone(page);
7159 pfn = page_to_pfn(page);
Cody P Schafer108bcc92013-02-22 16:35:23 -08007160 if (!zone_spans_pfn(zone, pfn))
Michal Hocko687875f2012-01-20 14:33:55 -08007161 return false;
7162
Wen Congyangb023f462012-12-11 16:00:45 -08007163 return !has_unmovable_pages(zone, page, 0, true);
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07007164}
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07007165
Vlastimil Babka080fe202016-02-05 15:36:41 -08007166#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007167
7168static unsigned long pfn_max_align_down(unsigned long pfn)
7169{
7170 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
7171 pageblock_nr_pages) - 1);
7172}
7173
7174static unsigned long pfn_max_align_up(unsigned long pfn)
7175{
7176 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
7177 pageblock_nr_pages));
7178}
7179
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007180/* [start, end) must belong to a single zone. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07007181static int __alloc_contig_migrate_range(struct compact_control *cc,
7182 unsigned long start, unsigned long end)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007183{
7184 /* This function is based on compact_zone() from compaction.c. */
Minchan Kimbeb51ea2012-10-08 16:33:51 -07007185 unsigned long nr_reclaimed;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007186 unsigned long pfn = start;
7187 unsigned int tries = 0;
7188 int ret = 0;
7189
Marek Szyprowskibe49a6e2012-12-12 13:51:19 -08007190 migrate_prep();
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007191
Mel Gormanbb13ffe2012-10-08 16:32:41 -07007192 while (pfn < end || !list_empty(&cc->migratepages)) {
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007193 if (fatal_signal_pending(current)) {
7194 ret = -EINTR;
7195 break;
7196 }
7197
Mel Gormanbb13ffe2012-10-08 16:32:41 -07007198 if (list_empty(&cc->migratepages)) {
7199 cc->nr_migratepages = 0;
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07007200 pfn = isolate_migratepages_range(cc, pfn, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007201 if (!pfn) {
7202 ret = -EINTR;
7203 break;
7204 }
7205 tries = 0;
7206 } else if (++tries == 5) {
7207 ret = ret < 0 ? ret : -EBUSY;
7208 break;
7209 }
7210
Minchan Kimbeb51ea2012-10-08 16:33:51 -07007211 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
7212 &cc->migratepages);
7213 cc->nr_migratepages -= nr_reclaimed;
Minchan Kim02c6de82012-10-08 16:31:55 -07007214
Hugh Dickins9c620e22013-02-22 16:35:14 -08007215 ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
David Rientjese0b9dae2014-06-04 16:08:28 -07007216 NULL, 0, cc->mode, MR_CMA);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007217 }
Srinivas Pandruvada2a6f5122013-02-22 16:32:09 -08007218 if (ret < 0) {
7219 putback_movable_pages(&cc->migratepages);
7220 return ret;
7221 }
7222 return 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007223}
7224
7225/**
7226 * alloc_contig_range() -- tries to allocate given range of pages
7227 * @start: start PFN to allocate
7228 * @end: one-past-the-last PFN to allocate
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02007229 * @migratetype: migratetype of the underlaying pageblocks (either
7230 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
7231 * in range must have the same migratetype and it must
7232 * be either of the two.
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007233 *
7234 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
7235 * aligned, however it's the caller's responsibility to guarantee that
7236 * we are the only thread that changes migrate type of pageblocks the
7237 * pages fall in.
7238 *
7239 * The PFN range must belong to a single zone.
7240 *
7241 * Returns zero on success or negative error code. On success all
7242 * pages which PFN is in [start, end) are allocated for the caller and
7243 * need to be freed with free_contig_range().
7244 */
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02007245int alloc_contig_range(unsigned long start, unsigned long end,
7246 unsigned migratetype)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007247{
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007248 unsigned long outer_start, outer_end;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08007249 unsigned int order;
7250 int ret = 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007251
Mel Gormanbb13ffe2012-10-08 16:32:41 -07007252 struct compact_control cc = {
7253 .nr_migratepages = 0,
7254 .order = -1,
7255 .zone = page_zone(pfn_to_page(start)),
David Rientjese0b9dae2014-06-04 16:08:28 -07007256 .mode = MIGRATE_SYNC,
Mel Gormanbb13ffe2012-10-08 16:32:41 -07007257 .ignore_skip_hint = true,
7258 };
7259 INIT_LIST_HEAD(&cc.migratepages);
7260
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007261 /*
7262 * What we do here is we mark all pageblocks in range as
7263 * MIGRATE_ISOLATE. Because pageblock and max order pages may
7264 * have different sizes, and due to the way page allocator
7265 * work, we align the range to biggest of the two pages so
7266 * that page allocator won't try to merge buddies from
7267 * different pageblocks and change MIGRATE_ISOLATE to some
7268 * other migration type.
7269 *
7270 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
7271 * migrate the pages from an unaligned range (ie. pages that
7272 * we are interested in). This will put all the pages in
7273 * range back to page allocator as MIGRATE_ISOLATE.
7274 *
7275 * When this is done, we take the pages in range from page
7276 * allocator removing them from the buddy system. This way
7277 * page allocator will never consider using them.
7278 *
7279 * This lets us mark the pageblocks back as
7280 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
7281 * aligned range but not in the unaligned, original range are
7282 * put back to page allocator so that buddy can use them.
7283 */
7284
7285 ret = start_isolate_page_range(pfn_max_align_down(start),
Wen Congyangb023f462012-12-11 16:00:45 -08007286 pfn_max_align_up(end), migratetype,
7287 false);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007288 if (ret)
Bob Liu86a595f2012-10-25 13:37:56 -07007289 return ret;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007290
Joonsoo Kim8ef58492016-01-14 15:18:45 -08007291 /*
7292 * In case of -EBUSY, we'd like to know which page causes problem.
Mike Kravetz436f19a2017-11-29 16:10:01 -08007293 * So, just fall through. test_pages_isolated() has a tracepoint
7294 * which will report the busy page.
7295 *
7296 * It is possible that busy pages could become available before
7297 * the call to test_pages_isolated, and the range will actually be
7298 * allocated. So, if we fall through be sure to clear ret so that
7299 * -EBUSY is not accidentally used or returned to caller.
Joonsoo Kim8ef58492016-01-14 15:18:45 -08007300 */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07007301 ret = __alloc_contig_migrate_range(&cc, start, end);
Joonsoo Kim8ef58492016-01-14 15:18:45 -08007302 if (ret && ret != -EBUSY)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007303 goto done;
Mike Kravetz436f19a2017-11-29 16:10:01 -08007304 ret =0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007305
7306 /*
7307 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
7308 * aligned blocks that are marked as MIGRATE_ISOLATE. What's
7309 * more, all pages in [start, end) are free in page allocator.
7310 * What we are going to do is to allocate all pages from
7311 * [start, end) (that is remove them from page allocator).
7312 *
7313 * The only problem is that pages at the beginning and at the
7314 * end of interesting range may be not aligned with pages that
7315 * page allocator holds, ie. they can be part of higher order
7316 * pages. Because of this, we reserve the bigger range and
7317 * once this is done free the pages we are not interested in.
7318 *
7319 * We don't have to hold zone->lock here because the pages are
7320 * isolated thus they won't get removed from buddy.
7321 */
7322
7323 lru_add_drain_all();
Vlastimil Babka510f5502014-12-10 15:43:07 -08007324 drain_all_pages(cc.zone);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007325
7326 order = 0;
7327 outer_start = start;
7328 while (!PageBuddy(pfn_to_page(outer_start))) {
7329 if (++order >= MAX_ORDER) {
Joonsoo Kim8ef58492016-01-14 15:18:45 -08007330 outer_start = start;
7331 break;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007332 }
7333 outer_start &= ~0UL << order;
7334 }
7335
Joonsoo Kim8ef58492016-01-14 15:18:45 -08007336 if (outer_start != start) {
7337 order = page_order(pfn_to_page(outer_start));
7338
7339 /*
7340 * outer_start page could be small order buddy page and
7341 * it doesn't include start page. Adjust outer_start
7342 * in this case to report failed page properly
7343 * on tracepoint in test_pages_isolated()
7344 */
7345 if (outer_start + (1UL << order) <= start)
7346 outer_start = start;
7347 }
7348
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007349 /* Make sure the range is really isolated. */
Wen Congyangb023f462012-12-11 16:00:45 -08007350 if (test_pages_isolated(outer_start, end, false)) {
Jonathan Toppinsb56cd772017-08-10 15:23:35 -07007351 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
Michal Nazarewiczdae803e2014-11-13 15:19:27 -08007352 __func__, outer_start, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007353 ret = -EBUSY;
7354 goto done;
7355 }
7356
Marek Szyprowski49f223a2012-01-25 12:49:24 +01007357 /* Grab isolated pages from freelists. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07007358 outer_end = isolate_freepages_range(&cc, outer_start, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007359 if (!outer_end) {
7360 ret = -EBUSY;
7361 goto done;
7362 }
7363
7364 /* Free head and tail (if any) */
7365 if (start != outer_start)
7366 free_contig_range(outer_start, start - outer_start);
7367 if (end != outer_end)
7368 free_contig_range(end, outer_end - end);
7369
7370done:
7371 undo_isolate_page_range(pfn_max_align_down(start),
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02007372 pfn_max_align_up(end), migratetype);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007373 return ret;
7374}
7375
7376void free_contig_range(unsigned long pfn, unsigned nr_pages)
7377{
Marek Szyprowskibcc2b022012-12-20 15:05:18 -08007378 unsigned int count = 0;
7379
7380 for (; nr_pages--; pfn++) {
7381 struct page *page = pfn_to_page(pfn);
7382
7383 count += page_count(page) != 1;
7384 __free_page(page);
7385 }
7386 WARN(count != 0, "%d pages are still in use!\n", count);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01007387}
7388#endif
7389
Jiang Liu4ed7e022012-07-31 16:43:35 -07007390#ifdef CONFIG_MEMORY_HOTPLUG
Cody P Schafer0a647f32013-07-03 15:01:33 -07007391/*
7392 * The zone indicated has a new number of managed_pages; batch sizes and percpu
7393 * page high values need to be recalulated.
7394 */
Jiang Liu4ed7e022012-07-31 16:43:35 -07007395void __meminit zone_pcp_update(struct zone *zone)
7396{
Cody P Schafer0a647f32013-07-03 15:01:33 -07007397 unsigned cpu;
Cody P Schaferc8e251f2013-07-03 15:01:29 -07007398 mutex_lock(&pcp_batch_high_lock);
Cody P Schafer0a647f32013-07-03 15:01:33 -07007399 for_each_possible_cpu(cpu)
Cody P Schafer169f6c12013-07-03 15:01:41 -07007400 pageset_set_high_and_batch(zone,
7401 per_cpu_ptr(zone->pageset, cpu));
Cody P Schaferc8e251f2013-07-03 15:01:29 -07007402 mutex_unlock(&pcp_batch_high_lock);
Jiang Liu4ed7e022012-07-31 16:43:35 -07007403}
7404#endif
7405
Jiang Liu340175b2012-07-31 16:43:32 -07007406void zone_pcp_reset(struct zone *zone)
7407{
7408 unsigned long flags;
Minchan Kim5a883812012-10-08 16:33:39 -07007409 int cpu;
7410 struct per_cpu_pageset *pset;
Jiang Liu340175b2012-07-31 16:43:32 -07007411
7412 /* avoid races with drain_pages() */
7413 local_irq_save(flags);
7414 if (zone->pageset != &boot_pageset) {
Minchan Kim5a883812012-10-08 16:33:39 -07007415 for_each_online_cpu(cpu) {
7416 pset = per_cpu_ptr(zone->pageset, cpu);
7417 drain_zonestat(zone, pset);
7418 }
Jiang Liu340175b2012-07-31 16:43:32 -07007419 free_percpu(zone->pageset);
7420 zone->pageset = &boot_pageset;
7421 }
7422 local_irq_restore(flags);
7423}
7424
Wen Congyang6dcd73d2012-12-11 16:01:01 -08007425#ifdef CONFIG_MEMORY_HOTREMOVE
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07007426/*
Joonsoo Kimb9eb6312016-05-19 17:12:06 -07007427 * All pages in the range must be in a single zone and isolated
7428 * before calling this.
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07007429 */
7430void
7431__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
7432{
7433 struct page *page;
7434 struct zone *zone;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07007435 unsigned int order, i;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07007436 unsigned long pfn;
7437 unsigned long flags;
7438 /* find the first valid pfn */
7439 for (pfn = start_pfn; pfn < end_pfn; pfn++)
7440 if (pfn_valid(pfn))
7441 break;
7442 if (pfn == end_pfn)
7443 return;
7444 zone = page_zone(pfn_to_page(pfn));
7445 spin_lock_irqsave(&zone->lock, flags);
7446 pfn = start_pfn;
7447 while (pfn < end_pfn) {
7448 if (!pfn_valid(pfn)) {
7449 pfn++;
7450 continue;
7451 }
7452 page = pfn_to_page(pfn);
Wen Congyangb023f462012-12-11 16:00:45 -08007453 /*
7454 * The HWPoisoned page may be not in buddy system, and
7455 * page_count() is not 0.
7456 */
7457 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
7458 pfn++;
7459 SetPageReserved(page);
7460 continue;
7461 }
7462
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07007463 BUG_ON(page_count(page));
7464 BUG_ON(!PageBuddy(page));
7465 order = page_order(page);
7466#ifdef CONFIG_DEBUG_VM
Joe Perches11705322016-03-17 14:19:50 -07007467 pr_info("remove from free list %lx %d %lx\n",
7468 pfn, 1 << order, end_pfn);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07007469#endif
7470 list_del(&page->lru);
7471 rmv_page_order(page);
7472 zone->free_area[order].nr_free--;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07007473 for (i = 0; i < (1 << order); i++)
7474 SetPageReserved((page+i));
7475 pfn += (1 << order);
7476 }
7477 spin_unlock_irqrestore(&zone->lock, flags);
7478}
7479#endif
Wu Fengguang8d22ba12009-12-16 12:19:58 +01007480
Wu Fengguang8d22ba12009-12-16 12:19:58 +01007481bool is_free_buddy_page(struct page *page)
7482{
7483 struct zone *zone = page_zone(page);
7484 unsigned long pfn = page_to_pfn(page);
7485 unsigned long flags;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07007486 unsigned int order;
Wu Fengguang8d22ba12009-12-16 12:19:58 +01007487
7488 spin_lock_irqsave(&zone->lock, flags);
7489 for (order = 0; order < MAX_ORDER; order++) {
7490 struct page *page_head = page - (pfn & ((1 << order) - 1));
7491
7492 if (PageBuddy(page_head) && page_order(page_head) >= order)
7493 break;
7494 }
7495 spin_unlock_irqrestore(&zone->lock, flags);
7496
7497 return order < MAX_ORDER;
7498}